syncpackage: add mismatching source tarball detection (for fake syncs).

debian/changelog

@@ -15,9 +15,11 @@ ubuntu-dev-tools (0.100) UNRELEASED; urgency=low
     rmadison isn't. (LP: #574398)
 
   [ Benjamin Drung ]
-  * syncpackage: add more options and allow pulling packages from Debian.
+  * syncpackage:
+   - add more options and allow pulling packages from Debian.
+   - add mismatching source tarball detection (for fake syncs).
 
- -- Benjamin Drung <bdrung@ubuntu.com>  Mon, 10 May 2010 00:04:52 +0200
+ -- Benjamin Drung <bdrung@ubuntu.com>  Sat, 15 May 2010 17:55:22 +0200
 
 ubuntu-dev-tools (0.99) lucid; urgency=low
 

syncpackage

@@ -20,8 +20,10 @@
 # ##################################################################
 
 import apt_pkg
+import hashlib
 import optparse
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -33,6 +35,46 @@ from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg
 from ubuntutools.lp import udtexceptions
 from ubuntutools.lp.libsupport import get_launchpad
 
+class File(object):
+	def __init__(self, url, checksum, size):
+		self.url = url
+		self.name = os.path.basename(url)
+		self.checksum = checksum
+		self.size = size
+
+	def __repr__(self):
+		return self.name + " (" + self.checksum + " " + self.size + ") source " + \
+				str(bool(self.is_source_file()))
+
+	def __eq__(self, other):
+		return self.name == other.name and self.checksum == other.checksum and \
+				self.size == other.size
+
+	def get_name(self):
+		return self.name
+
+	def is_source_file(self):
+		return re.match(".*\.orig.*\.tar\..*", self.name)
+
+	def download(self, verbose=False):
+		'''Download file (by URL)  to the current directory.
+
+		If the file is already present, this function does nothing.'''
+
+		file_exists = os.path.exists(self.name)
+
+		if file_exists:
+			# Check for correct checksum
+			m = hashlib.md5()
+			m.update(open(self.name).read())
+			file_exists = m.hexdigest() == self.checksum
+
+		if not file_exists:
+			if verbose:
+				print 'downloading', self.url
+			urllib.urlretrieve(self.url, self.name)
+
+
 def strip_epoch(version):
 	'''Removes the epoch from a Debian version string.
 	
@@ -67,21 +109,11 @@ def remove_signature(dscname, verbose=False):
 		f.writelines(unsigned_file)
 		f.close()
 
-def retrieve_file(url, verbose=False):
-	'''Download file (by URL)  to the current directory.
-
-	If the file is already present, this function does nothing.'''
-
-	fname = os.path.basename(url)
-	if not os.path.exists(fname):
-		if verbose:
-			print 'downloading', url
-		urllib.urlretrieve(url, fname)
-
 def dsc_getfiles(dsc):
 	'''Return list of files in a .dsc file (excluding the .dsc file itself).'''
 
-	f = open(dsc)
+	basepath = os.path.dirname(dsc)
+	f = urllib.urlopen(dsc)
 	files = []
 
 	# skip until 'Files:'
@@ -91,12 +123,11 @@ def dsc_getfiles(dsc):
 
 	for l in f:
 		if not l.startswith(' '):
-			continue
-		if l.strip() == '':
 			break
-		fname = l.split()[2]
+		(checksum, size, fname) = l.split()
+		url = os.path.join(basepath, fname)
 		if not fname.endswith('.dsc'):
-			files.append(fname)
+			files.append(File(url, checksum, size))
 
 	f.close()
 	return files
@@ -126,12 +157,16 @@ def sync_dsc(dscurl, debian_dist, release, uploader, bugs, keyid=None, verbose=F
 	basepath = os.path.dirname(dscurl)
 	(srcpkg, new_ver) = dscname.split('_')
 
-	retrieve_file(dscurl, verbose)
+	urllib.urlretrieve(dscurl, dscname)
 	dscfile = open(dscname).readlines()
 	new_ver = filter(lambda l: l.startswith("Version:"), dscfile)[0][8:].strip()
 
 	try:
-		ubuntu_ver = getUbuntuSrcPkg(srcpkg, release).getVersion()
+		ubuntu_source = getUbuntuSrcPkg(srcpkg, release)
+		ubuntu_ver = ubuntu_source.getVersion()
+		ubuntu_dsc = filter(lambda f: f.endswith(".dsc"), ubuntu_source.sourceFileUrls())
+		assert len(ubuntu_dsc) == 1
+		ubuntu_dsc = ubuntu_dsc[0]
 	except udtexceptions.PackageNotFoundException:
 		ubuntu_ver = '~'
 
@@ -139,25 +174,48 @@ def sync_dsc(dscurl, debian_dist, release, uploader, bugs, keyid=None, verbose=F
 	apt_pkg.init()
 	if not apt_pkg.check_dep(new_ver, '>', ubuntu_ver):
 		raise Exception('%s version %s is not greater than already available %s' % (srcpkg, new_ver, ubuntu_ver))
+	if verbose:
+		print 'Source %s: current version %s, new version %s' % (srcpkg, ubuntu_ver, new_ver)
+
+	files = dsc_getfiles(dscurl)
+	source_files = filter(lambda f: f.is_source_file(), files)
+	if verbose:
+		print 'Files:', map(lambda x: x.get_name(), files)
+		print 'Source files:', map(lambda x: x.get_name(), source_files)
+	map(lambda f: f.download(verbose), files)
+
+	ubuntu_files = dsc_getfiles(ubuntu_dsc)
 
 	# do we need the orig.tar.gz?
 	need_orig = True
+	fakesync_files = []
 	if ubuntu_ver.find('-') > 0 and new_ver.find('-') > 0 and \
 		ubuntu_ver.split('-')[0] == new_ver.split('-')[0]:
+		# We need to check if all .orig*.tar.* tarballs exist in Ubuntu
 		need_orig = False
-
+		for source_file in source_files:
-	files = dsc_getfiles(dscname)
+			ubuntu_file = filter(lambda f: f.get_name() == source_file.get_name(), ubuntu_files)
+			if len(ubuntu_file) == 0:
+				# The source file does not exist in Ubuntu
+				if verbose:
+					print "%s does not exist in Ubuntu." % (source_file.get_name())
+				need_orig = True
+			elif not ubuntu_file[0] == source_file:
+				# The checksum of the files mismatch -> We need a fake sync
+				print "WARNING! The checksum of the file %s mismatch. A fake sync is required." % (source_file.get_name())
+				fakesync_files.append(ubuntu_file[0])
+				if verbose:
+					print "Ubuntu version:", ubuntu_file[0]
+					print "Debian version:", source_file
 	if verbose:
-		print 'Source %s: current version %s, new version %s' % (srcpkg, ubuntu_ver, new_ver)
 		print 'needs orig.tar.gz', need_orig
-		print 'Files:', files
-	for f in files:
-		retrieve_file(os.path.join(basepath, f), verbose)
 
 	uidx = ubuntu_ver.find('ubuntu')
 	if uidx > 0:
 		cur_ver = ubuntu_ver[:uidx]
 		print 'WARNING! Overwriting modified Ubuntu version %s, setting current version to %s' % (ubuntu_ver, cur_ver)
+	else:
+		cur_ver = ubuntu_ver
 
 	uidx = cur_ver.find('build')
 	if uidx > 0:
@@ -166,6 +224,11 @@ def sync_dsc(dscurl, debian_dist, release, uploader, bugs, keyid=None, verbose=F
 	# extract package
 	subprocess.check_call(['dpkg-source', '-x', dscname])
 
+	# Do a fake sync if required
+	if len(fakesync_files) > 0:
+		# Download Ubuntu files (override Debian source tarballs)
+		map(lambda f: f.download(verbose), fakesync_files)
+
 	# change into package directory
 	directory = srcpkg + '-' + strip_epoch(new_ver).split('-')[0]
 	if verbose:
@@ -177,6 +240,7 @@ def sync_dsc(dscurl, debian_dist, release, uploader, bugs, keyid=None, verbose=F
 		line = open("debian/changelog").readline()
 		debian_dist = line.split(" ")[2].strip(";")
 
+	if len(fakesync_files) == 0:
 		# create the changes file
 		changes_file = "%s_%s_source.changes" % (srcpkg, strip_epoch(new_ver))
 		cmd = ["dpkg-genchanges", "-S", "-v" + cur_ver,
@@ -212,6 +276,32 @@ def sync_dsc(dscurl, debian_dist, release, uploader, bugs, keyid=None, verbose=F
 		if verbose:
 			print " ".join(cmd)
 		subprocess.check_call(cmd)
+	else:
+		# Create fakesync changelog entry
+		new_ver += "fakesync1"
+		changes_file = "%s_%s_source.changes" % (srcpkg, strip_epoch(new_ver))
+		if len(bugs) > 0:
+			message = "Fake sync due to mismatching orig tarball (LP: %s)." % \
+					(", ".join(map(lambda b: "#" + str(b), bugs)))
+		else:
+			message = "Fake sync due to mismatching orig tarball."
+		cmd = ["dch", "-v", new_ver, "-D", release, message]
+
+		# update the Maintainer field
+		subprocess.check_call(cmd)
+		subprocess.check_call(["update-maintainer"])
+		
+		# Build source package
+		cmd = ["dpkg-buildpackage", "-S", "-v" + cur_ver, "-e" + uploader]
+		if need_orig:
+			cmd += ['-sa']
+		if not keyid is None:
+			cmd += ["-k" + keyid]
+		if verbose:
+			print " ".join(cmd)
+		subprocess.check_call(cmd)
+
+	print changes_file
 
 def get_debian_dscurl(package, dist, release, version=None, component=None):
 	if dist is None: