ubuntu-dev-tools/syncpackage

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2008-2010 Martin Pitt <martin.pitt@canonical.com>
#               2010 Benjamin Drung <bdrung@ubuntu.com>
#
# ##################################################################
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 3.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# See file /usr/share/common-licenses/GPL-3 for more details.
#
# ##################################################################

import apt_pkg
import hashlib
import optparse
import os
import re
import shutil
import subprocess
import sys
import urllib

# ubuntu-dev-tools modules
from ubuntutools.requestsync.mail import getDebianSrcPkg as ubuntutools_requestsync_mail_getDebianSrcPkg
from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg
from ubuntutools.lp import udtexceptions
from ubuntutools.lp.lpapicache import Launchpad

class File(object):
	def __init__(self, url, checksum, size):
		self.url = url
		self.name = os.path.basename(url)
		self.checksum = checksum
		self.size = size

	def __repr__(self):
		return self.name + " (" + self.checksum + " " + self.size + ") source " + \
				str(bool(self.is_source_file()))

	def __eq__(self, other):
		return self.name == other.name and self.checksum == other.checksum and \
				self.size == other.size

	def get_name(self):
		return self.name

	def is_source_file(self):
		return re.match(".*\.orig.*\.tar\..*", self.name)

	def download(self, verbose=False):
		'''Download file (by URL)  to the current directory.

		If the file is already present, this function does nothing.'''

		file_exists = os.path.exists(self.name)

		if file_exists:
			# Check for correct checksum
			m = hashlib.md5()
			m.update(open(self.name).read())
			file_exists = m.hexdigest() == self.checksum

		if not file_exists:
			if verbose:
				print 'downloading', self.url
			urllib.urlretrieve(self.url, self.name)


def strip_epoch(version):
	'''Removes the epoch from a Debian version string.
	
	strip_epoch(1:1.52-1) will return "1.52-1" and strip_epoch(1.1.3-1) will
	return "1.1.3-1".'''

	parts = version.split(':')
	if len(parts) > 1:
		del parts[0]
	version = ':'.join(parts)
	return version

def remove_signature(dscname, verbose=False):
	'''Removes the signature from a .dsc file if the .dsc file is signed.'''

	f = open(dscname)
	if f.readline().strip() == "-----BEGIN PGP SIGNED MESSAGE-----":
		unsigned_file = []
		# search until begin of body found
		for l in f:
			if l.strip() == "":
				break

		# search for end of body
		for l in f:
			if l.strip() == "":
				break
			unsigned_file.append(l)

		f.close()
		f = open(dscname, "w")
		f.writelines(unsigned_file)
		f.close()

def dsc_getfiles(dsc):
	'''Return list of files in a .dsc file (excluding the .dsc file itself).'''

	basepath = os.path.dirname(dsc)
	f = urllib.urlopen(dsc)
	files = []

	# skip until 'Files:'
	for l in f:
		if l.strip() == 'Files:':
			break

	for l in f:
		if not l.startswith(' '):
			break
		(checksum, size, fname) = l.split()
		url = os.path.join(basepath, fname)
		if not fname.endswith('.dsc'):
			files.append(File(url, checksum, size))

	f.close()
	return files

def add_fixed_bugs(changes, bugs, verbose=False):
	'''Add additional Launchpad bugs to the list of fixed bugs in changes file.'''

	changes = filter(lambda l: l.strip() != "", changes.split("\n"))
	# Remove duplicates
	bugs = set(bugs)

	for i in xrange(len(changes)):
		if changes[i].startswith("Launchpad-Bugs-Fixed:"):
			bugs.update(changes[i][22:].strip().split(" "))
			changes[i] = "Launchpad-Bugs-Fixed: %s" % (" ".join(bugs))
			break
		elif i == len(changes) - 1:
			# Launchpad-Bugs-Fixed entry does not exist in changes file
			line = "Launchpad-Bugs-Fixed: %s" % (" ".join(bugs))
			changes.append(line)

	return "\n".join(changes + [""])

def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None, verbose=False):
	assert dscurl.endswith(".dsc")
	dscname = os.path.basename(dscurl)
	basepath = os.path.dirname(dscurl)
	(srcpkg, new_ver) = dscname.split('_')
	uploader = name + " <" + email + ">"

	if os.path.exists(os.path.join(basepath, dscname)):
		dscfile = dscurl
	else:
		urllib.urlretrieve(dscurl, dscname)
	dscfile = open(dscname).readlines()
	new_ver = filter(lambda l: l.startswith("Version:"), dscfile)[0][8:].strip()

	try:
		ubuntu_source = getUbuntuSrcPkg(srcpkg, release)
		ubuntu_ver = ubuntu_source.getVersion()
		ubuntu_dsc = filter(lambda f: f.endswith(".dsc"), ubuntu_source.sourceFileUrls())
		assert len(ubuntu_dsc) == 1
		ubuntu_dsc = ubuntu_dsc[0]
	except udtexceptions.PackageNotFoundException:
		ubuntu_ver = '~'
		ubuntu_dsc = None

	# No need to continue if version is not greater than current one
	apt_pkg.init()
	if not apt_pkg.check_dep(new_ver, '>', ubuntu_ver):
		raise Exception('%s version %s is not greater than already available %s' % (srcpkg, new_ver, ubuntu_ver))
	if verbose:
		print 'Source %s: current version %s, new version %s' % (srcpkg, ubuntu_ver, new_ver)

	files = dsc_getfiles(dscurl)
	source_files = filter(lambda f: f.is_source_file(), files)
	if verbose:
		print 'Files:', map(lambda x: x.get_name(), files)
		print 'Source files:', map(lambda x: x.get_name(), source_files)
	map(lambda f: f.download(verbose), files)

	if ubuntu_dsc is None:
		ubuntu_files = None
	else:
		ubuntu_files = dsc_getfiles(ubuntu_dsc)

	# do we need the orig.tar.gz?
	need_orig = True
	fakesync_files = []
	if ubuntu_ver.find('-') > 0 and new_ver.find('-') > 0 and \
		ubuntu_ver.split('-')[0] == new_ver.split('-')[0]:
		# We need to check if all .orig*.tar.* tarballs exist in Ubuntu
		need_orig = False
		for source_file in source_files:
			ubuntu_file = filter(lambda f: f.get_name() == source_file.get_name(), ubuntu_files)
			if len(ubuntu_file) == 0:
				# The source file does not exist in Ubuntu
				if verbose:
					print "%s does not exist in Ubuntu." % (source_file.get_name())
				need_orig = True
			elif not ubuntu_file[0] == source_file:
				# The checksum of the files mismatch -> We need a fake sync
				print "WARNING! The checksum of the file %s mismatch. A fake sync is required." % (source_file.get_name())
				fakesync_files.append(ubuntu_file[0])
				if verbose:
					print "Ubuntu version:", ubuntu_file[0]
					print "Debian version:", source_file
	if verbose:
		print 'needs orig.tar.gz', need_orig

	uidx = ubuntu_ver.find('ubuntu')
	if uidx > 0:
		cur_ver = ubuntu_ver[:uidx]
		print 'WARNING! Overwriting modified Ubuntu version %s, setting current version to %s' % (ubuntu_ver, cur_ver)
	else:
		cur_ver = ubuntu_ver

	uidx = cur_ver.find('build')
	if uidx > 0:
		cur_ver = cur_ver[:uidx]

	# extract package
	subprocess.check_call(['dpkg-source', '-x', dscname])

	# Do a fake sync if required
	if len(fakesync_files) > 0:
		# Download Ubuntu files (override Debian source tarballs)
		map(lambda f: f.download(verbose), fakesync_files)

	# change into package directory
	directory = srcpkg + '-' + strip_epoch(new_ver).split('-')[0]
	if verbose:
		print "cd " + directory
	os.chdir(directory)

	# read Debian distribution from debian/changelog if not specified
	if debian_dist is None:
		line = open("debian/changelog").readline()
		debian_dist = line.split(" ")[2].strip(";")

	if len(fakesync_files) == 0:
		# create the changes file
		changes_file = "%s_%s_source.changes" % (srcpkg, strip_epoch(new_ver))
		cmd = ["dpkg-genchanges", "-S", "-v" + cur_ver,
				"-DDistribution=" + release,
				"-DOrigin=debian/" + debian_dist,
				"-e" + uploader]
		if need_orig:
			cmd += ['-sa']
		if not verbose:
			cmd += ["-q"]
		if verbose:
			print " ".join(cmd) + " > ../" + changes_file
		changes = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]

		# Add additional bug numbers
		if len(bugs) > 0:
			changes = add_fixed_bugs(changes, bugs, verbose)

		# remove extracted (temporary) files
		os.chdir('..')
		shutil.rmtree(directory, True)

		# write changes file
		f = open(changes_file, "w")
		f.writelines(changes)
		f.close()

		# remove signature and sign package
		remove_signature(dscname)
		cmd = ["debsign", changes_file]
		if not keyid is None:
			cmd.insert(1, "-k" + keyid)
		if verbose:
			print " ".join(cmd)
		subprocess.check_call(cmd)
	else:
		# Create fakesync changelog entry
		new_ver += "fakesync1"
		changes_file = "%s_%s_source.changes" % (srcpkg, strip_epoch(new_ver))
		if len(bugs) > 0:
			message = "Fake sync due to mismatching orig tarball (LP: %s)." % \
					(", ".join(map(lambda b: "#" + str(b), bugs)))
		else:
			message = "Fake sync due to mismatching orig tarball."
		cmd = ["dch", "-v", new_ver, "-D", release, message]
		env = {"DEBFULLNAME": name, "DEBEMAIL": email}

		# update the Maintainer field
		subprocess.check_call(cmd, env=env)
		subprocess.check_call(["update-maintainer"])
		
		# Build source package
		cmd = ["debuild", "--no-lintian", "-S", "-v" + cur_ver]
		if need_orig:
			cmd += ['-sa']
		if not keyid is None:
			cmd += ["-k" + keyid]
		if verbose:
			print " ".join(cmd)
		subprocess.check_call(cmd)

	print changes_file

def get_debian_dscurl(package, dist, release, version=None, component=None):
	if dist is None:
		dist="unstable"

	if version is None or component is None:
		debian_srcpkg = getDebianSrcPkg(package, dist)
		try:
			ubuntu_version = getUbuntuSrcPkg(package, release).getVersion()
		except udtexceptions.PackageNotFoundException:
			ubuntu_version = '~'
		apt_pkg.init()
		if apt_pkg.check_dep(ubuntu_version, ">=", debian_srcpkg.getVersion()):
			# The LP importer is maybe out of date
			debian_srcpkg = ubuntutools_requestsync_mail_getDebianSrcPkg(package, dist)

		if version is None:
			version = debian_srcpkg.getVersion()
		if component is None:
			component = debian_srcpkg.getComponent()

	assert component in ("main", "contrib", "non-free")

	if package.startswith("lib"):
		group = package[0:4]
	else:
		group = package[0]

	dsc_file = package + "_" + strip_epoch(version) + ".dsc"
	dscurl = os.path.join("http://ftp.debian.org/debian/pool", component, group, package, dsc_file)
	return dscurl

if __name__ == "__main__":
	script_name = os.path.basename(sys.argv[0])
	usage = "%s [options] <.dsc URL/path or package name>" % (script_name)
	epilog = "See %s(1) for more info." % (script_name)
	parser = optparse.OptionParser(usage=usage, epilog=epilog)

	parser.add_option("-d", "--distribution", type = "string",
			dest = "dist", default = None,
			help = "Debian distribution to sync from.")
	parser.add_option("-r", "--release",
			help="Specify target Ubuntu release.", dest="release", default=None)
	parser.add_option("-V", "--debian-version",
			help="Specify the version to sync from.", dest="debversion", default=None)
	parser.add_option("-c", "--component",
			help="Specify the component to sync from.", dest="component", default=None)
	parser.add_option("-v", "--verbose", help="print more information",
			dest="verbose", action="store_true", default=False)
	parser.add_option("-n", "--uploader-name", dest="uploader_name",
			help="Use UPLOADER_NAME as the name of the maintainer "
			     "for this upload instead of evaluating DEBFULLNAME.",
			default = os.environ["DEBFULLNAME"])
	parser.add_option("-e", "--uploader-email", dest="uploader_email",
			help="Use UPLOADER_EMAIL as email address of the maintainer "
			     "for this upload instead of evaluating DEBEMAIL.",
			default = os.environ["DEBEMAIL"])
	parser.add_option("-k", "--key", dest="keyid",
			help="Specify the key ID to be used for signing.", default=None)
	parser.add_option("-b", "--bug", metavar="BUG",
			help="Mark a Launchpad bug as being fixed by this upload",
			dest="bugs", action="append", default=list())

	(options, args) = parser.parse_args()

	if len(args) == 0:
		print >> sys.stderr, "%s: Error: No .dsc URL/path or package name specified." % (script_name)
		sys.exit(1)
	elif len(args) > 1:
		print >> sys.stderr, script_name + ": Error: Multiple .dsc URLs/paths or package names specified: " + ", ".join(args)
		sys.exit(1)

	invalid_bug_numbers = filter(lambda x: not x.isdigit(), options.bugs)
	if len(invalid_bug_numbers) > 0:
		print >> sys.stderr, "%s: Error: Invalid bug number(s) specified: %s" % \
				(script_name, ", ".join(invalid_bug_numbers))
		sys.exit(1)

	Launchpad.login_anonymously()
	if options.release is None:
		options.release = Launchpad.distributions["ubuntu"].current_series.name

	if args[0].endswith(".dsc"):
		dscurl = args[0]
	else:
		dscurl = get_debian_dscurl(args[0], options.dist, options.release,
				options.debversion, options.component)

	if options.verbose:
		print ".dsc url: " + dscurl
	sync_dsc(dscurl, options.dist, options.release, options.uploader_name,
			options.uploader_email, options.bugs, options.keyid, options.verbose)