From c46bb9ed882a7725b769f8e6e91caeee0ecd4b07 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 27 Dec 2010 18:24:21 +0200 Subject: [PATCH 01/43] Use ubuntutools.logger in syncpackage --- syncpackage | 188 +++++++++++++++++++--------------------------------- 1 file changed, 70 insertions(+), 118 deletions(-) diff --git a/syncpackage b/syncpackage index b5ab2ff..706f6f2 100755 --- a/syncpackage +++ b/syncpackage @@ -1,8 +1,9 @@ #!/usr/bin/python # -*- coding: utf-8 -*- # -# Copyright (C) 2008-2010 Martin Pitt -# 2010 Benjamin Drung +# Copyright (C) 2008-2010 Martin Pitt , +# 2010 Benjamin Drung , +# 2010 Stefano Rivera # # ################################################################## # @@ -30,10 +31,10 @@ import subprocess import sys import urllib -# ubuntu-dev-tools modules from ubuntutools.requestsync.mail import getDebianSrcPkg \ as requestsync_mail_getDebianSrcPkg from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg +from ubuntutools.logger import Logger from ubuntutools.lp import udtexceptions from ubuntutools.lp.lpapicache import Launchpad @@ -58,7 +59,7 @@ class File(object): def is_source_file(self): return re.match(".*\.orig.*\.tar\..*", self.name) - def download(self, script_name=None, verbose=False): + def download(self): '''Download file (by URL) to the current directory. If the file is already present, this function does nothing.''' @@ -72,14 +73,12 @@ class File(object): file_exists = md5.hexdigest() == self.checksum if not file_exists: - if verbose: - print '%s: I: Downloading %s...' % (script_name, self.url) + Logger.info('Downloading %s...', self.url) try: urllib.urlretrieve(self.url, self.name) except IOError as err: - parameters = (script_name, self.name, err.errno, err.strerror) - print >> sys.stderr, ("%s: Error: Failed to download %s " - "[Errno %i]: %s.") % parameters + Logger.error('Failed to download %s [Errno %i]: %s.', + self.name, err.errno, err.strerror) sys.exit(1) @@ -109,17 +108,6 @@ class Version(debian.debian_support.Version): def is_modified_in_ubuntu(self): return self.full_version.find('ubuntu') > 0 - -def quote_parameter(parameter): - if parameter.find(" ") >= 0: - return '"' + parameter + '"' - else: - return parameter - -def print_command(script_name, cmd): - cmd = [quote_parameter(x) for x in cmd] - print "%s: I: %s" % (script_name, " ".join(cmd)) - def remove_signature(dscname): '''Removes the signature from a .dsc file if the .dsc file is signed.''' @@ -142,16 +130,15 @@ def remove_signature(dscname): dsc_file.writelines(unsigned_file) dsc_file.close() -def dsc_getfiles(dscurl, script_name): +def dsc_getfiles(dscurl): '''Return list of files in a .dsc file (excluding the .dsc file itself).''' basepath = os.path.dirname(dscurl) dsc = debian.deb822.Dsc(urllib.urlopen(dscurl)) if 'Files' not in dsc: - parameters = (script_name, os.path.basename(dscurl)) - print >> sys.stderr, ("%s: Error: No Files field found in the dsc " - "file. Please check %s!") % parameters + Logger.error('No Files field found in the dsc file. Please check %s!', + os.path.basename(dscurl)) sys.exit(1) files = [] @@ -181,8 +168,7 @@ def add_fixed_bugs(changes, bugs): return "\n".join(changes + [""]) -def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, - keyid=None, verbose=False): +def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None): assert dscurl.endswith(".dsc") dscname = os.path.basename(dscurl) basepath = os.path.dirname(dscurl) @@ -195,14 +181,13 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, try: urllib.urlretrieve(dscurl, dscname) except IOError as error: - parameters = (script_name, dscname, error.errno, error.strerror) - print >> sys.stderr, ("%s: Error: Failed to download %s " - "[Errno %i]: %s.") % parameters + Logger.error('Failed to download %s [Errno %i]: %s.', + dscname, error.errno, error.strerror) sys.exit(1) dscfile = debian.deb822.Dsc(file(dscname)) if "Version" not in dscfile: - print >> sys.stderr, ("%s: Error: No Version field found in the dsc " - "file. Please check %s!") % (script_name, dscname) + Logger.error('No Version field found in the dsc file. Please check %s!', + dscname) sys.exit(1) new_ver = Version(dscfile["Version"]) @@ -219,27 +204,22 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, # No need to continue if version is not greater than current one if new_ver <= ubuntu_ver: - parameters = (script_name, srcpkg, new_ver, ubuntu_ver) - print >> sys.stderr, ("%s: Error: %s version %s is not greater than " - "already available %s") % parameters + Logger.error('%s version %s is not greater than already available %s', + srcpkg, new_ver, ubuntu_ver) sys.exit(1) - if verbose: - print '%s: D: Source %s: current version %s, new version %s' % \ - (script_name, srcpkg, ubuntu_ver, new_ver) + Logger.debug('Source %s: current version %s, new version %s', + srcpkg, ubuntu_ver, new_ver) - files = dsc_getfiles(dscurl, script_name) + files = dsc_getfiles(dscurl) source_files = [f for f in files if f.is_source_file()] - if verbose: - print '%s: D: Files: %s' % (script_name, - str([x.get_name() for x in files])) - print '%s: D: Source files: %s' % \ - (script_name, str([x.get_name() for x in source_files])) - [f.download(script_name, verbose) for f in files] + Logger.debug('Files: %s', str([x.get_name() for x in files])) + Logger.debug('Source files: %s', str([x.get_name() for x in source_files])) + [f.download() for f in files] if ubuntu_dsc is None: ubuntu_files = None else: - ubuntu_files = dsc_getfiles(ubuntu_dsc, script_name) + ubuntu_files = dsc_getfiles(ubuntu_dsc) # do we need the orig.tar.gz? need_orig = True @@ -252,49 +232,41 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, if f.get_name() == source_file.get_name()] if len(ubuntu_file) == 0: # The source file does not exist in Ubuntu - if verbose: - parameters = (script_name, source_file.get_name()) - print "%s: I: %s does not exist in Ubuntu." % parameters + Logger.info('%s does not exist in Ubuntu.', + source_file.get_name()) need_orig = True elif not ubuntu_file[0] == source_file: # The checksum of the files mismatch -> We need a fake sync - parameters = (script_name, source_file.get_name()) - print ("%s: Warning: The checksum of the file %s mismatch. " - "A fake sync is required.") % parameters + Logger.warn('The checksum of the file %s mismatch. ' + 'A fake sync is required.', source_file.get_name()) fakesync_files.append(ubuntu_file[0]) - if verbose: - print "%s: D: Ubuntu version: %s" % (script_name, - ubuntu_file[0]) - print "%s: D: Debian version: %s" % (script_name, - source_file) - if verbose: - print '%s: D: needs source tarball: %s' % (script_name, str(need_orig)) + Logger.debug('Ubuntu version: %s', ubuntu_file[0]) + Logger.debug('Debian version: %s', source_file) + Logger.debug('Needs source tarball: %s', str(need_orig)) cur_ver = ubuntu_ver.get_related_debian_version() if ubuntu_ver.is_modified_in_ubuntu(): - params = (script_name, ubuntu_ver.full_version, cur_ver.full_version) - print ('%s: Warning: Overwriting modified Ubuntu version %s, ' - 'setting current version to %s') % params + Logger.warn('Overwriting modified Ubuntu version %s, ' + 'setting current version to %s', + ubuntu_ver.full_version, cur_ver.full_version) # extract package cmd = ['dpkg-source', '-x', dscname] env = os.environ env['DEB_VENDOR'] = 'Ubuntu' - if not verbose: + if not Logger.verbose: cmd.insert(1, "-q") - if verbose: - print_command(script_name, cmd) + Logger.command(cmd) subprocess.check_call(cmd, env=env) # Do a fake sync if required if len(fakesync_files) > 0: # Download Ubuntu files (override Debian source tarballs) - [f.download(script_name, verbose) for f in fakesync_files] + [f.download() for f in fakesync_files] # change into package directory directory = srcpkg + '-' + new_ver.upstream_version - if verbose: - print_command(script_name, ["cd", directory]) + Logger.command(('cd', directory)) os.chdir(directory) # read Debian distribution from debian/changelog if not specified @@ -314,10 +286,9 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, cmd.append("-sa") else: cmd.append("-sd") - if not verbose: + if not Logger.verbose: cmd += ["-q"] - if verbose: - print_command(script_name, cmd + [">", "../" + changes_filename]) + Logger.command(cmd + ['>', '../' + changes_filename]) changes = subprocess.Popen(cmd, stdout=subprocess.PIPE, env={"DEB_VENDOR": "Ubuntu"}).communicate()[0] @@ -326,8 +297,7 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, changes = add_fixed_bugs(changes, bugs) # remove extracted (temporary) files - if verbose: - print_command(script_name, ["cd", ".."]) + Logger.command(('cd', '..')) os.chdir('..') shutil.rmtree(directory, True) @@ -342,8 +312,7 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, cmd = ["debsign", changes_filename] if not keyid is None: cmd.insert(1, "-k" + keyid) - if verbose: - print_command(script_name, cmd) + Logger.command(cmd) subprocess.check_call(cmd) else: # Create fakesync changelog entry @@ -357,16 +326,14 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, message = "Fake sync due to mismatching orig tarball." cmd = ["dch", "-v", new_ver.full_version, "-D", release, message] env = {"DEBFULLNAME": name, "DEBEMAIL": email} - if verbose: - print_command(script_name, cmd) + Logger.command(cmd) subprocess.check_call(cmd, env=env) # update the Maintainer field cmd = ["update-maintainer"] - if not verbose: + if not Logger.verbose: cmd.append("-q") - if verbose: - print_command(script_name, cmd) + Logger.command(cmd) subprocess.check_call(cmd) # Build source package @@ -377,13 +344,11 @@ def sync_dsc(script_name, dscurl, debian_dist, release, name, email, bugs, cmd += ['-sa'] if keyid: cmd += ["-k" + keyid] - if verbose: - print_command(script_name, cmd) + Logger.command(cmd) returncode = subprocess.call(cmd, env=env) if returncode != 0: - print >> sys.stderr, ("%s: Error: Source-only build with debuild " - "failed. Please check build log above.") % \ - (script_name) + Logger.error('Source-only build with debuild failed. ' + 'Please check build log above.') sys.exit(1) def get_debian_dscurl(package, dist, release, version=None, component=None): @@ -421,9 +386,8 @@ def get_debian_dscurl(package, dist, release, version=None, component=None): return dscurl def main(): - script_name = os.path.basename(sys.argv[0]) - usage = "%s [options] <.dsc URL/path or package name>" % (script_name) - epilog = "See %s(1) for more info." % (script_name) + usage = "%prog [options] <.dsc URL/path or package name>" + epilog = "See %s(1) for more info." % os.path.basename(sys.argv[0]) parser = optparse.OptionParser(usage=usage, epilog=epilog) parser.add_option("-d", "--distribution", type="string", @@ -456,40 +420,31 @@ def main(): (options, args) = parser.parse_args() if len(args) == 0: - print >> sys.stderr, ("%s: Error: No .dsc URL/path or package name " - "specified.") % (script_name) - sys.exit(1) - elif len(args) > 1: - parameters = (script_name, ", ".join(args)) - print >> sys.stderr, ("%s: Error: Multiple .dsc URLs/paths or " - "package names specified: %s") % parameters - sys.exit(1) + parser.error('No .dsc URL/path or package name specified.') + if len(args) > 1: + parser.error('Multiple .dsc URLs/paths or package names specified: ' + + ', '.join(args)) invalid_bug_numbers = [bug for bug in options.bugs if not bug.isdigit()] if len(invalid_bug_numbers) > 0: - print >> sys.stderr, "%s: Error: Invalid bug number(s) specified: %s" \ - % (script_name, ", ".join(invalid_bug_numbers)) - sys.exit(1) + parser.error('Invalid bug number(s) specified: ' + + ', '.join(invalid_bug_numbers)) if options.uploader_name is None: if "DEBFULLNAME" in os.environ: options.uploader_name = os.environ["DEBFULLNAME"] else: - print >> sys.stderr, ("%s: Error: No uploader name specified. You " - "must pass the --uploader-name option or set " - "the DEBFULLNAME environment variable.") % \ - (script_name) - sys.exit(1) + parser.error('No uploader name specified. You must pass the ' + '--uploader-name option or set the DEBFULLNAME ' + 'environment variable.') if options.uploader_email is None: if "DEBEMAIL" in os.environ: options.uploader_email = os.environ["DEBEMAIL"] else: - print >> sys.stderr, ("%s: Error: No uploader email address " - "specified. You must pass the " - "--uploader-email option or set the DEBEMAIL" - " environment variable.") % (script_name) - sys.exit(1) + parser.error('No uploader email address specified. You must pass ' + 'the --uploader-email option or set the DEBEMAIL ' + 'environment variable.') Launchpad.login_anonymously() if options.release is None: @@ -499,19 +454,16 @@ def main(): dscurl = args[0] else: if options.component not in (None, "main", "contrib", "non-free"): - parameters = (script_name, options.component) - print >> sys.stderr, ("%s: Error: %s is not a valid Debian " - "component. It should be one of main, " - "contrib, or non-free.") % parameters - sys.exit(1) + parser.error('%s is not a valid Debian component. ' + 'It should be one of main, contrib, or non-free.' + % options.component) dscurl = get_debian_dscurl(args[0], options.dist, options.release, options.debversion, options.component) - if options.verbose: - print "%s: D: .dsc url: %s" % (script_name, dscurl) - sync_dsc(script_name, dscurl, options.dist, options.release, - options.uploader_name, options.uploader_email, options.bugs, - options.keyid, options.verbose) + Logger.verbose = options.verbose + Logger.debug('.dsc url: %s', dscurl) + sync_dsc(dscurl, options.dist, options.release, options.uploader_name, + options.uploader_email, options.bugs, options.keyid) if __name__ == "__main__": main() From f7525d6dd5b409b2641d5e1aa07d29e0a66caabe Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 27 Dec 2010 21:09:24 +0200 Subject: [PATCH 02/43] Add UDTConfig and tidy OptParser --- syncpackage | 61 ++++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/syncpackage b/syncpackage index 706f6f2..7e4bfcd 100755 --- a/syncpackage +++ b/syncpackage @@ -31,6 +31,7 @@ import subprocess import sys import urllib +from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.requestsync.mail import getDebianSrcPkg \ as requestsync_mail_getDebianSrcPkg from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg @@ -390,32 +391,42 @@ def main(): epilog = "See %s(1) for more info." % os.path.basename(sys.argv[0]) parser = optparse.OptionParser(usage=usage, epilog=epilog) - parser.add_option("-d", "--distribution", type="string", + parser.add_option("-d", "--distribution", dest="dist", default=None, help="Debian distribution to sync from.") - parser.add_option("-r", "--release", dest="release", default=None, + parser.add_option("-r", "--release", + dest="release", default=None, help="Specify target Ubuntu release.") - parser.add_option("-V", "--debian-version", dest="debversion", default=None, + parser.add_option("-V", "--debian-version", + dest="debversion", default=None, help="Specify the version to sync from.") - parser.add_option("-c", "--component", dest="component", default=None, + parser.add_option("-c", "--component", + dest="component", default=None, help="Specify the Debian component to sync from.") - parser.add_option("-v", "--verbose", help="print more information", - dest="verbose", action="store_true", default=False) - parser.add_option("-n", "--uploader-name", dest="uploader_name", + parser.add_option("-v", "--verbose", + dest="verbose", action="store_true", default=False, + help="Display more progress information.") + parser.add_option("-n", "--uploader-name", + dest="uploader_name", default=None, help="Use UPLOADER_NAME as the name of the maintainer " - "for this upload instead of evaluating DEBFULLNAME.", - default=None) - parser.add_option("-e", "--uploader-email", dest="uploader_email", + "for this upload.") + parser.add_option("-e", "--uploader-email", + dest="uploader_email", default=None, help="Use UPLOADER_EMAIL as email address of the " - "maintainer for this upload instead of evaluating " - "DEBEMAIL.", default=None) - parser.add_option("-k", "--key", dest="keyid", default=None, + "maintainer for this upload.") + parser.add_option("-k", "--key", + dest="keyid", default=None, help="Specify the key ID to be used for signing.") - parser.add_option('--dont-sign', dest='keyid', action='store_false', - help='Do not sign the upload') + parser.add_option('--dont-sign', + dest='keyid', action='store_false', + help='Do not sign the upload.') parser.add_option("-b", "--bug", metavar="BUG", - help="Mark a Launchpad bug as being fixed by this upload", - dest="bugs", action="append", default=list()) + dest="bugs", action="append", default=list(), + help="Mark Launchpad bug BUG as being fixed by this " + "upload.") + parser.add_option('--no-conf', + dest='no_conf', default=False, action='store_true', + help="Don't read config files or environment variables.") (options, args) = parser.parse_args() @@ -430,21 +441,13 @@ def main(): parser.error('Invalid bug number(s) specified: ' + ', '.join(invalid_bug_numbers)) + config = UDTConfig(options.no_conf) + if options.uploader_name is None: - if "DEBFULLNAME" in os.environ: - options.uploader_name = os.environ["DEBFULLNAME"] - else: - parser.error('No uploader name specified. You must pass the ' - '--uploader-name option or set the DEBFULLNAME ' - 'environment variable.') + options.uploader_name = ubu_email(export=False)[0] if options.uploader_email is None: - if "DEBEMAIL" in os.environ: - options.uploader_email = os.environ["DEBEMAIL"] - else: - parser.error('No uploader email address specified. You must pass ' - 'the --uploader-email option or set the DEBEMAIL ' - 'environment variable.') + options.uploader_email = ubu_email(export=False)[1] Launchpad.login_anonymously() if options.release is None: From 2c27cf68a61a382e13041eb5542a33fa0cdc0187 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 28 Dec 2010 00:23:53 +0200 Subject: [PATCH 03/43] Move common downloading code into new ubuntutools.mirrors. Use in backportpackage, pull-debian-debdiff, pull-lp-source --- backportpackage | 53 ++++--------- debian/copyright | 1 + pull-debian-debdiff | 116 +++------------------------- pull-lp-source | 32 ++------ ubuntutools/config.py | 8 +- ubuntutools/mirrors.py | 170 +++++++++++++++++++++++++++++++++++++++++ ubuntutools/misc.py | 12 --- 7 files changed, 210 insertions(+), 182 deletions(-) create mode 100644 ubuntutools/mirrors.py diff --git a/backportpackage b/backportpackage index 716883c..39a4792 100755 --- a/backportpackage +++ b/backportpackage @@ -24,7 +24,6 @@ import shutil import subprocess import sys import tempfile -import urllib from debian.deb822 import Dsc from launchpadlib.launchpad import Launchpad @@ -33,8 +32,8 @@ import lsb_release from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.builder import get_builder from ubuntutools.logger import Logger +from ubuntutools.mirrors import dsc_name, pull_source_pkg from ubuntutools.question import YesNoQuestion -from ubuntutools.misc import dsc_url def error(msg): Logger.error(msg) @@ -179,7 +178,17 @@ def find_version_package(launchpad, package, version): error('Version %s of package %s was never published in Ubuntu.' % (version, package)) -def dscurls_from_package(launchpad, mirror, package, version, source_release): +def fetch_package(launchpad, mirror, workdir, package, version, source_release): + "Returns the path to the .dsc file that was fetched" + + if package.endswith('.dsc'): + cmd = ('dget', '--download-only', '--allow-unauthenticated', package) + Logger.command(cmd) + ret = subprocess.call(cmd, cwd=workdir) + if ret == 0: + return os.path.join(workdir, os.path.basename(package)) + sys.exit(1) + if not source_release and not version: source_release = launchpad.distributions['ubuntu'].current_series.name @@ -190,40 +199,10 @@ def dscurls_from_package(launchpad, mirror, package, version, source_release): else: srcpkg = find_version_package(launchpad, package, version) - urls = [] - if mirror: - urls.append(dsc_url(mirror, srcpkg.component_name, package, - srcpkg.source_package_version)) - - for source_file in srcpkg.sourceFileUrls(): - if source_file.endswith('.dsc'): - urls.append(urllib.unquote(source_file)) - return urls - else: - error('Package %s contains no .dsc file.' % package) - -def dscurl_from_dsc(package): - path = os.path.abspath(os.path.expanduser(package)) - if os.path.exists(path): - return 'file://%s' % path - else: - # Can't resolve it as a local path? Let's just hope it's good as-is - return package - -def fetch_package(launchpad, mirror, workdir, package, version, source_release): - # Returns the path to the .dsc file that was fetched - if package.endswith('.dsc'): - dscs = [dscurl_from_dsc(package)] - else: - dscs = dscurls_from_package(launchpad, mirror, package, version, - source_release) - - for dsc in dscs: - cmd = ('dget', '--download-only', '--allow-unauthenticated', dsc) - Logger.command(cmd) - ret = subprocess.call(cmd, cwd=workdir) - if ret == 0: - return os.path.join(workdir, os.path.basename(dsc)) + version = srcpkg.source_package_version + pull_source_pkg('UBUNTU', mirror, srcpkg.component_name, package, version, + workdir=workdir, unpack=False) + return dsc_name(package, version) def get_backport_version(version, suffix, upload, release): backport_version = version + ('~%s1' % release) diff --git a/debian/copyright b/debian/copyright index e6bd445..8e9f677 100644 --- a/debian/copyright +++ b/debian/copyright @@ -187,6 +187,7 @@ Files: doc/pull-debian-debdiff.1, ubuntutools/config.py, ubuntutools/control.py, ubuntutools/logger.py, + ubuntutools/mirrors.py, ubuntutools/question.py, ubuntutools/sponsor_patch/*, ubuntutools/test/*, diff --git a/pull-debian-debdiff b/pull-debian-debdiff index aa4ac5c..bcccd58 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -17,112 +17,15 @@ # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. -import hashlib import optparse -import os.path import subprocess import sys -import urllib2 import debian.changelog from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger -from ubuntutools.misc import dsc_name, dsc_url - -DEFAULT_DEBIAN_MIRROR = 'http://ftp.debian.org/debian' -DEFAULT_DEBSEC_MIRROR = 'http://security.debian.org' - -def pull(package, version, opts, unpack=False): - "Download Debian source package version version" - urls = [] - # TODO: Not all packages are main :) - # Practically this is fine, as it'll be found on snapshot, but still ugly. - if opts.debsec_mirror and opts.debsec_mirror != DEFAULT_DEBSEC_MIRROR: - urls.append(dsc_url(opts.debsec_mirror, 'main', package, version)) - urls.append(dsc_url(DEFAULT_DEBSEC_MIRROR, 'main', package, version)) - if opts.debian_mirror and opts.debian_mirror != DEFAULT_DEBIAN_MIRROR: - urls.append(dsc_url(opts.debian_mirror, 'main', package, version)) - urls.append(dsc_url(DEFAULT_DEBIAN_MIRROR, 'main', package, version)) - - for url in urls: - cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd) - if return_code == 0: - return True - - Logger.normal('Trying snapshot.debian.org') - return pull_from_snapshot(package, version, unpack) - -def pull_from_snapshot(package, version, unpack=False): - "Download Debian source package version version from snapshot.debian.org" - try: - import json - except ImportError: - import simplejson as json - except ImportError: - Logger.error("Please install python-simplejson.") - sys.exit(1) - - try: - srcfiles = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' - % (package, version))) - except urllib2.HTTPError: - Logger.error('Version %s of %s not found on snapshot.debian.org', - version, package) - return False - - for hash_ in srcfiles['result']: - hash_ = hash_['hash'] - - try: - info = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) - except urllib2.URLError: - Logger.error('Unable to dowload info for hash.') - return False - - filename = info['result'][0]['name'] - if '/' in filename: - Logger.error('Unacceptable file name: %s', filename) - return False - - if os.path.exists(filename): - source_file = open(filename, 'r') - sha1 = hashlib.sha1() - sha1.update(source_file.read()) - source_file.close() - if sha1.hexdigest() == hash_: - Logger.normal('Using existing %s', filename) - continue - - Logger.normal('Downloading: %s (%0.3f MiB)', filename, - info['result'][0]['size'] / 1024.0 / 1024) - try: - in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) - out = open(filename, 'w') - while True: - block = in_.read(10240) - if block == '': - break - out.write(block) - sys.stdout.write('.') - sys.stdout.flush() - sys.stdout.write('\n') - sys.stdout.flush() - out.close() - except urllib2.URLError: - Logger.error('Error downloading %s', filename) - return False - - if unpack: - cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version)) - Logger.command(cmd) - subprocess.check_call(cmd) - - return True +from ubuntutools.mirrors import dsc_name, pull_source_pkg def previous_version(package, version, distance): "Given an (extracted) package, determine the version distance versions ago" @@ -177,9 +80,13 @@ def main(): opts.debsec_mirror = config.get_value('DEBSEC_MIRROR') Logger.normal('Downloading %s %s', package, version) - if not pull(package, version, opts, unpack=not opts.fetch_only): - Logger.error("Couldn't locate version %s of %s.", version, package) - sys.exit(1) + + # TODO: Not all packages are main, but snapshot.debian.org should save + # the day, as it doesn't care about component. + pull_source_pkg(('DEBSEC', 'DEBIAN'), + {'DEBSEC': opts.debsec_mirror, + 'DEBIAN': opts.debian_mirror}, + 'main', package, version, unpack=True) if opts.fetch_only: sys.exit(0) @@ -189,9 +96,10 @@ def main(): Logger.error('No previous version could be found') sys.exit(1) Logger.normal('Downloading %s %s', package, oldversion) - if not pull(package, oldversion, opts, unpack=True): - Logger.error("Couldn't locate version %s of %s.", oldversion, package) - sys.exit(1) + pull_source_pkg(('DEBSEC', 'DEBIAN'), + {'DEBSEC': opts.debsec_mirror, + 'DEBIAN': opts.debian_mirror}, + 'main', package, oldversion, unpack=True) cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version)) Logger.command(cmd) diff --git a/pull-lp-source b/pull-lp-source index f11fb92..feb060c 100755 --- a/pull-lp-source +++ b/pull-lp-source @@ -25,16 +25,16 @@ import os import sys -import subprocess -import urllib from optparse import OptionParser from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.lp.lpapicache import Distribution, Launchpad from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, - PackageNotFoundException, PocketDoesNotExistError) -from ubuntutools.misc import split_release_pocket, dsc_url + PackageNotFoundException, + PocketDoesNotExistError) +from ubuntutools.mirrors import pull_source_pkg +from ubuntutools.misc import split_release_pocket def main(): usage = "Usage: %prog [release]" @@ -83,27 +83,9 @@ def main(): Logger.error(error) sys.exit(1) - urls = [] - if options.ubuntu_mirror: - urls.append(dsc_url(options.ubuntu_mirror, spph.getComponent(), - package, spph.getVersion())) - dsc_url = [url for url in spph.sourceFileUrls() if url.endswith('.dsc')] - assert dsc_url, 'No .dsc file found' - urls.append(urllib.unquote(dsc_url[0])) - - Logger.normal('Fetching the source for %s from %s (%s)...', - package, release.capitalize(), pocket) - for url in urls: - cmd = ('dget', '-u' + ('d' if options.download_only else 'x'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd) - if return_code == 0: - Logger.normal("Success!") - sys.exit(0) - - Logger.error('Failed to fetch and extrace the source. ' - 'Please check the output for the error.') - sys.exit(1) + pull_source_pkg('UBUNTU', options.ubuntu_mirror, spph.getComponent(), + package, spph.getVersion(), + unpack=not options.download_only) if __name__ == '__main__': main() diff --git a/ubuntutools/config.py b/ubuntutools/config.py index 1445b3a..8793aa6 100644 --- a/ubuntutools/config.py +++ b/ubuntutools/config.py @@ -34,11 +34,11 @@ class UDTConfig(object): # These are reqired to be used by at least two scripts. defaults = { 'BUILDER': 'pbuilder', - 'DEBIAN_MIRROR': None, - 'DEBSEC_MIRROR': None, + 'DEBIAN_MIRROR': 'http://ftp.debian.org/debian', + 'DEBSEC_MIRROR': 'http://security.debian.org', 'LPINSTANCE': 'production', 'MIRROR_FALLBACK': True, - 'UBUNTU_MIRROR': None, + 'UBUNTU_MIRROR': 'http://archive.ubuntu.com/ubuntu', 'UPDATE_BUILDER': False, 'WORKDIR': None, } @@ -74,7 +74,7 @@ class UDTConfig(object): f.close() return config - def get_value(self, key, default=None, boolean=False, compat_keys=[]): + def get_value(self, key, default=None, boolean=False, compat_keys=()): """Retrieve a value from the environment or configuration files. keys are prefixed with the script name, falling back to UBUNTUTOOLS for package-wide keys. diff --git a/ubuntutools/mirrors.py b/ubuntutools/mirrors.py new file mode 100644 index 0000000..4cdc33e --- /dev/null +++ b/ubuntutools/mirrors.py @@ -0,0 +1,170 @@ +# mirrors.py - Functions for dealing with Debian source packages and mirrors. +# +# Copyright (C) 2010, Stefano Rivera +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +import hashlib +import os.path +import subprocess +import urllib2 +import sys + +from ubuntutools.config import UDTConfig +from ubuntutools.logger import Logger + +def dsc_name(package, version): + "Return the source package dsc filename for the given package" + if ':' in version: + version = version.split(':', 1)[1] + return '%s_%s.dsc' % (package, version) + +def dsc_url(mirror, component, package, version): + "Build a source package URL" + group = package[:4] if package.startswith('lib') else package[0] + filename = dsc_name(package, version) + return os.path.join(mirror, 'pool', component, group, package, filename) + +def pull_source_pkg(archives, mirrors, component, package, version, workdir='.', + unpack=False): + """Download a source package or die. + archives may be a list or single item (in which case mirrors can be too) + mirrors should be a dict (keyed on archive) unless archives is single""" + + if not isinstance(archives, (tuple, list)): + if not isinstance(mirrors, dict): + mirrors = {archives: mirrors} + archives = [archives] + assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives) + + for archive in archives: + if try_pull_from_archive(archive, mirrors.get(archive), component, + package, version, workdir, unpack): + return + + if 'DEBIAN' in archives or 'DEBSEC' in archives: + Logger.info('Trying snapshot.debian.org') + if try_pull_from_snapshot(package, version, workdir, unpack): + return + + if 'UBUNTU' in archives: + Logger.info('Trying Launchpad') + if try_pull_from_lp(package, 'ubuntu', version, workdir, unpack): + return + + raise Exception('Unable to locate %s/%s %s' % (package, component, version)) + +def try_pull_from_archive(archive, mirror, component, package, version, + workdir='.', unpack=False): + """Download a source package from the specified source or return False. + Try mirror first, then master. + """ + assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU') + urls = [] + if mirror and mirror != UDTConfig.defaults[archive + '_MIRROR']: + urls.append(dsc_url(mirror, component, package, version)) + urls.append(dsc_url(UDTConfig.defaults[archive + '_MIRROR'], component, + package, version)) + + for url in urls: + cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) + Logger.command(cmd) + return_code = subprocess.call(cmd, cwd=workdir) + if return_code == 0: + return True + + return False + +def try_pull_from_snapshot(package, version, workdir='.', unpack=False): + """Download Debian source package version version from snapshot.debian.org + or return False + """ + try: + import json + except ImportError: + import simplejson as json + except ImportError: + Logger.error("Please install python-simplejson.") + sys.exit(1) + + try: + srcfiles = json.load(urllib2.urlopen( + 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' + % (package, version))) + except urllib2.HTTPError: + Logger.error('Version %s of %s not found on snapshot.debian.org', + version, package) + return False + + for hash_ in srcfiles['result']: + hash_ = hash_['hash'] + + try: + info = json.load(urllib2.urlopen( + 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) + except urllib2.URLError: + Logger.error('Unable to dowload info for hash.') + return False + + filename = info['result'][0]['name'] + if '/' in filename: + Logger.error('Unacceptable file name: %s', filename) + return False + pathname = os.path.join(workdir, filename) + + if os.path.exists(pathname): + source_file = open(pathname, 'r') + sha1 = hashlib.sha1() + sha1.update(source_file.read()) + source_file.close() + if sha1.hexdigest() == hash_: + Logger.normal('Using existing %s', filename) + continue + + Logger.normal('Downloading: %s (%0.3f MiB)', filename, + info['result'][0]['size'] / 1024.0 / 1024) + try: + in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) + out = open(pathname, 'w') + while True: + block = in_.read(10240) + if block == '': + break + out.write(block) + sys.stdout.write('.') + sys.stdout.flush() + sys.stdout.write('\n') + sys.stdout.flush() + out.close() + except urllib2.URLError: + Logger.error('Error downloading %s', filename) + return False + + if unpack: + cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version)) + Logger.command(cmd) + subprocess.check_call(cmd) + return True + +def try_pull_from_lp(package, distro, version, workdir='.', unpack=False): + """Try to download the specified version of a source package from Launchpad + or return False + """ + url = ('https://launchpad.net/%s/+archive/primary/+files/%s' + % (distro, dsc_name(package, version))) + cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) + Logger.command(cmd) + return_code = subprocess.call(cmd, cwd=workdir) + if return_code == 0: + return True + return False diff --git a/ubuntutools/misc.py b/ubuntutools/misc.py index 84ad93e..dd581d3 100644 --- a/ubuntutools/misc.py +++ b/ubuntutools/misc.py @@ -119,15 +119,3 @@ def split_release_pocket(release): pocket) return (release, pocket) - -def dsc_name(package, version): - "Return the source package dsc filename for the given package" - if ':' in version: - version = version.split(':', 1)[1] - return '%s_%s.dsc' % (package, version) - -def dsc_url(mirror, component, package, version): - "Build a source package URL" - group = package[:4] if package.startswith('lib') else package[0] - filename = dsc_name(package, version) - return os.path.join(mirror, 'pool', component, group, package, filename) From be79f00dff1d9cd4d32d3f713f51658cef61c294 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 28 Dec 2010 00:42:35 +0200 Subject: [PATCH 04/43] Lets make ubuntutools.mirrors return filenames --- backportpackage | 7 +++--- pull-debian-debdiff | 22 +++++++++--------- ubuntutools/mirrors.py | 51 ++++++++++++++++++++++-------------------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/backportpackage b/backportpackage index 39a4792..cb3309c 100755 --- a/backportpackage +++ b/backportpackage @@ -32,7 +32,7 @@ import lsb_release from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.builder import get_builder from ubuntutools.logger import Logger -from ubuntutools.mirrors import dsc_name, pull_source_pkg +from ubuntutools.mirrors import pull_source_pkg from ubuntutools.question import YesNoQuestion def error(msg): @@ -200,9 +200,8 @@ def fetch_package(launchpad, mirror, workdir, package, version, source_release): srcpkg = find_version_package(launchpad, package, version) version = srcpkg.source_package_version - pull_source_pkg('UBUNTU', mirror, srcpkg.component_name, package, version, - workdir=workdir, unpack=False) - return dsc_name(package, version) + return pull_source_pkg('UBUNTU', mirror, srcpkg.component_name, package, + version, workdir=workdir, unpack=False) def get_backport_version(version, suffix, upload, release): backport_version = version + ('~%s1' % release) diff --git a/pull-debian-debdiff b/pull-debian-debdiff index bcccd58..2caff87 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -25,7 +25,7 @@ import debian.changelog from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger -from ubuntutools.mirrors import dsc_name, pull_source_pkg +from ubuntutools.mirrors import pull_source_pkg def previous_version(package, version, distance): "Given an (extracted) package, determine the version distance versions ago" @@ -83,10 +83,10 @@ def main(): # TODO: Not all packages are main, but snapshot.debian.org should save # the day, as it doesn't care about component. - pull_source_pkg(('DEBSEC', 'DEBIAN'), - {'DEBSEC': opts.debsec_mirror, - 'DEBIAN': opts.debian_mirror}, - 'main', package, version, unpack=True) + newdsc = pull_source_pkg(('DEBSEC', 'DEBIAN'), + {'DEBSEC': opts.debsec_mirror, + 'DEBIAN': opts.debian_mirror}, + 'main', package, version, unpack=True) if opts.fetch_only: sys.exit(0) @@ -96,14 +96,14 @@ def main(): Logger.error('No previous version could be found') sys.exit(1) Logger.normal('Downloading %s %s', package, oldversion) - pull_source_pkg(('DEBSEC', 'DEBIAN'), - {'DEBSEC': opts.debsec_mirror, - 'DEBIAN': opts.debian_mirror}, - 'main', package, oldversion, unpack=True) + olddsc = pull_source_pkg(('DEBSEC', 'DEBIAN'), + {'DEBSEC': opts.debsec_mirror, + 'DEBIAN': opts.debian_mirror}, + 'main', package, oldversion, unpack=True) - cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version)) + cmd = ('debdiff', olddsc, newdsc) Logger.command(cmd) - difffn = dsc_name(package, version)[:-3] + 'debdiff' + difffn = newdsc[:-3] + 'debdiff' debdiff_file = open(difffn, 'w') if subprocess.call(cmd, stdout=debdiff_file) > 2: Logger.error('Debdiff failed.') diff --git a/ubuntutools/mirrors.py b/ubuntutools/mirrors.py index 4cdc33e..db100a5 100644 --- a/ubuntutools/mirrors.py +++ b/ubuntutools/mirrors.py @@ -48,25 +48,29 @@ def pull_source_pkg(archives, mirrors, component, package, version, workdir='.', assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives) for archive in archives: - if try_pull_from_archive(archive, mirrors.get(archive), component, - package, version, workdir, unpack): - return + filename = try_pull_from_archive(archive, mirrors.get(archive), + component, package, version, + workdir, unpack) + if filename: + return filename if 'DEBIAN' in archives or 'DEBSEC' in archives: Logger.info('Trying snapshot.debian.org') - if try_pull_from_snapshot(package, version, workdir, unpack): - return + filename = try_pull_from_snapshot(package, version, workdir, unpack) + if filename: + return filename if 'UBUNTU' in archives: Logger.info('Trying Launchpad') - if try_pull_from_lp(package, 'ubuntu', version, workdir, unpack): - return + filename = try_pull_from_lp(package, 'ubuntu', version, workdir, unpack) + if filename: + return filename raise Exception('Unable to locate %s/%s %s' % (package, component, version)) def try_pull_from_archive(archive, mirror, component, package, version, workdir='.', unpack=False): - """Download a source package from the specified source or return False. + """Download a source package from the specified source, return filename. Try mirror first, then master. """ assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU') @@ -81,13 +85,11 @@ def try_pull_from_archive(archive, mirror, component, package, version, Logger.command(cmd) return_code = subprocess.call(cmd, cwd=workdir) if return_code == 0: - return True - - return False + return os.path.basename(url) def try_pull_from_snapshot(package, version, workdir='.', unpack=False): - """Download Debian source package version version from snapshot.debian.org - or return False + """Download Debian source package version version from snapshot.debian.org. + Return filename. """ try: import json @@ -104,7 +106,7 @@ def try_pull_from_snapshot(package, version, workdir='.', unpack=False): except urllib2.HTTPError: Logger.error('Version %s of %s not found on snapshot.debian.org', version, package) - return False + return for hash_ in srcfiles['result']: hash_ = hash_['hash'] @@ -114,12 +116,12 @@ def try_pull_from_snapshot(package, version, workdir='.', unpack=False): 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) except urllib2.URLError: Logger.error('Unable to dowload info for hash.') - return False + return filename = info['result'][0]['name'] if '/' in filename: Logger.error('Unacceptable file name: %s', filename) - return False + return pathname = os.path.join(workdir, filename) if os.path.exists(pathname): @@ -148,23 +150,24 @@ def try_pull_from_snapshot(package, version, workdir='.', unpack=False): out.close() except urllib2.URLError: Logger.error('Error downloading %s', filename) - return False + return + filename = dsc_name(package, version) if unpack: - cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version)) + cmd = ('dpkg-source', '--no-check', '-x', filename) Logger.command(cmd) subprocess.check_call(cmd) - return True + return filename def try_pull_from_lp(package, distro, version, workdir='.', unpack=False): - """Try to download the specified version of a source package from Launchpad - or return False + """Try to download the specified version of a source package from Launchpad. + Return filename. """ + filename = dsc_name(package, version) url = ('https://launchpad.net/%s/+archive/primary/+files/%s' - % (distro, dsc_name(package, version))) + % (distro, filename)) cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) Logger.command(cmd) return_code = subprocess.call(cmd, cwd=workdir) if return_code == 0: - return True - return False + return filename From 1b93ed57ed80739f55fd3d9e8715163b2aa6089b Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 28 Dec 2010 15:34:23 +0200 Subject: [PATCH 05/43] Don't modify commands when logging them --- ubuntutools/logger.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ubuntutools/logger.py b/ubuntutools/logger.py index 1b0ecf7..f871e30 100644 --- a/ubuntutools/logger.py +++ b/ubuntutools/logger.py @@ -20,6 +20,12 @@ import os import sys +def escape_arg(arg): + "Shell-escpae arg, if necessary" + if ' ' not in arg: + return arg + return '"%s"' % arg.replace('\\', r'\\').replace('"', r'\"') + class Logger(object): script_name = os.path.basename(sys.argv[0]) verbose = False @@ -30,10 +36,9 @@ class Logger(object): @classmethod def command(cls, cmd): if cls.verbose: - for i in xrange(len(cmd)): - if cmd[i].find(" ") >= 0: - cmd[i] = '"' + cmd[i] + '"' - print >> cls.stdout, "%s: I: %s" % (cls.script_name, " ".join(cmd)) + print >> cls.stdout, "%s: I: %s" % (cls.script_name, + " ".join(escape_arg(arg) + for arg in cmd)) @classmethod def debug(cls, message, *args): From 62073c1085702976ab1b2f59ab44b48ee57096bd Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 28 Dec 2010 15:35:15 +0200 Subject: [PATCH 06/43] Mirror support in syncpackage --- syncpackage | 149 +++++++++++++++++------------------ ubuntutools/test/pylint.conf | 3 +- 2 files changed, 74 insertions(+), 78 deletions(-) diff --git a/syncpackage b/syncpackage index 7e4bfcd..807bb6c 100755 --- a/syncpackage +++ b/syncpackage @@ -22,7 +22,6 @@ import debian.deb822 import debian.debian_support -import hashlib import optparse import os import re @@ -30,6 +29,7 @@ import shutil import subprocess import sys import urllib +import urlparse from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.requestsync.mail import getDebianSrcPkg \ @@ -38,6 +38,7 @@ from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg from ubuntutools.logger import Logger from ubuntutools.lp import udtexceptions from ubuntutools.lp.lpapicache import Launchpad +from ubuntutools.mirrors import pull_source_pkg class File(object): def __init__(self, url, checksum, size): @@ -60,28 +61,6 @@ class File(object): def is_source_file(self): return re.match(".*\.orig.*\.tar\..*", self.name) - def download(self): - '''Download file (by URL) to the current directory. - - If the file is already present, this function does nothing.''' - - file_exists = os.path.exists(self.name) - - if file_exists: - # Check for correct checksum - md5 = hashlib.md5() - md5.update(open(self.name).read()) - file_exists = md5.hexdigest() == self.checksum - - if not file_exists: - Logger.info('Downloading %s...', self.url) - try: - urllib.urlretrieve(self.url, self.name) - except IOError as err: - Logger.error('Failed to download %s [Errno %i]: %s.', - self.name, err.errno, err.strerror) - sys.exit(1) - class Version(debian.debian_support.Version): def strip_epoch(self): @@ -107,7 +86,7 @@ class Version(debian.debian_support.Version): return Version(related_debian_version) def is_modified_in_ubuntu(self): - return self.full_version.find('ubuntu') > 0 + return 'ubuntu' in self.full_version def remove_signature(dscname): '''Removes the signature from a .dsc file if the .dsc file is signed.''' @@ -169,22 +148,14 @@ def add_fixed_bugs(changes, bugs): return "\n".join(changes + [""]) -def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None): - assert dscurl.endswith(".dsc") - dscname = os.path.basename(dscurl) - basepath = os.path.dirname(dscurl) +def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, + keyid=None): + assert dscname.endswith(".dsc") + assert os.path.exists(dscname) + assert '/' not in dscname (srcpkg, new_ver) = dscname.split('_') uploader = name + " <" + email + ">" - if os.path.exists(os.path.join(basepath, dscname)): - dscfile = dscurl - else: - try: - urllib.urlretrieve(dscurl, dscname) - except IOError as error: - Logger.error('Failed to download %s [Errno %i]: %s.', - dscname, error.errno, error.strerror) - sys.exit(1) dscfile = debian.deb822.Dsc(file(dscname)) if "Version" not in dscfile: Logger.error('No Version field found in the dsc file. Please check %s!', @@ -203,19 +174,13 @@ def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None): ubuntu_ver = Version('~') ubuntu_dsc = None - # No need to continue if version is not greater than current one - if new_ver <= ubuntu_ver: - Logger.error('%s version %s is not greater than already available %s', - srcpkg, new_ver, ubuntu_ver) - sys.exit(1) Logger.debug('Source %s: current version %s, new version %s', srcpkg, ubuntu_ver, new_ver) - files = dsc_getfiles(dscurl) + files = dsc_getfiles(dscname) source_files = [f for f in files if f.is_source_file()] Logger.debug('Files: %s', str([x.get_name() for x in files])) Logger.debug('Source files: %s', str([x.get_name() for x in source_files])) - [f.download() for f in files] if ubuntu_dsc is None: ubuntu_files = None @@ -263,7 +228,8 @@ def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None): # Do a fake sync if required if len(fakesync_files) > 0: # Download Ubuntu files (override Debian source tarballs) - [f.download() for f in fakesync_files] + pull_source_pkg('UBUNTU', ubuntu_mirror, ubuntu_source.getComponent(), + srcpkg, ubuntu_ver.full_version) # change into package directory directory = srcpkg + '-' + new_ver.upstream_version @@ -352,39 +318,52 @@ def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None): 'Please check build log above.') sys.exit(1) -def get_debian_dscurl(package, dist, release, version=None, component=None): +def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): + """Download the specified source package. + dist, version, component, mirror can all be None. + """ if dist is None: dist = "unstable" + requested_version = version if type(version) == str: version = Version(version) if version is None or component is None: debian_srcpkg = getDebianSrcPkg(package, dist) - try: - src_pkg = getUbuntuSrcPkg(package, release) - ubuntu_version = Version(src_pkg.getVersion()) - except udtexceptions.PackageNotFoundException: - ubuntu_version = Version('~') - if ubuntu_version >= Version(debian_srcpkg.getVersion()): - # The LP importer is maybe out of date - debian_srcpkg = requestsync_mail_getDebianSrcPkg(package, dist) - if version is None: version = Version(debian_srcpkg.getVersion()) + try: + ubuntu_srcpkg = getUbuntuSrcPkg(package, ubuntu_release) + ubuntu_version = Version(ubuntu_srcpkg.getVersion()) + except udtexceptions.PackageNotFoundException: + ubuntu_version = Version('~') + if ubuntu_version >= version: + # The LP importer is maybe out of date + debian_srcpkg = requestsync_mail_getDebianSrcPkg(package, dist) + if requested_version is None: + version = Version(debian_srcpkg.getVersion()) + if ubuntu_version >= version: + Logger.error("Version in Debian %s (%s) isn't newer than " + "Ubuntu %s (%s)", + version, dist, ubuntu_version, ubuntu_release) + sys.exit(1) if component is None: component = debian_srcpkg.getComponent() - assert component in ("main", "contrib", "non-free") + assert component in ('main', 'contrib', 'non-free') - if package.startswith("lib"): - group = package[0:4] - else: - group = package[0] + return pull_source_pkg('DEBIAN', mirror, component, package, + version.full_version) - dsc_file = package + "_" + version.strip_epoch() + ".dsc" - dscurl = os.path.join("http://ftp.debian.org/debian/pool", component, group, - package, dsc_file) - return dscurl +def fetch_dsc(dscfile): + "Fetch a dsc" + url = urlparse.urlparse(dscfile) + if not url.scheme: + dscfile = 'file://' + os.path.abspath(dscfile) + cmd = ('dget', '--allow-unauthenticated', '-d', dscfile) + Logger.command(cmd) + subprocess.check_call(cmd) + return os.path.basename(url.path) def main(): usage = "%prog [options] <.dsc URL/path or package name>" @@ -424,6 +403,16 @@ def main(): dest="bugs", action="append", default=list(), help="Mark Launchpad bug BUG as being fixed by this " "upload.") + parser.add_option('-D', '--debian-mirror', metavar='DEBIAN_MIRROR', + dest='debian_mirror', + help='Preferred Debian mirror ' + '(default: %s)' + % UDTConfig.defaults['DEBIAN_MIRROR']) + parser.add_option('-U', '--ubuntu-mirror', metavar='UBUNTU_MIRROR', + dest='ubuntu_mirror', + help='Prefeed Ubuntu mirror ' + '(default: %s)' + % UDTConfig.defaults['UBUNTU_MIRROR']) parser.add_option('--no-conf', dest='no_conf', default=False, action='store_true', help="Don't read config files or environment variables.") @@ -441,11 +430,19 @@ def main(): parser.error('Invalid bug number(s) specified: ' + ', '.join(invalid_bug_numbers)) - config = UDTConfig(options.no_conf) + if options.component not in (None, "main", "contrib", "non-free"): + parser.error('%s is not a valid Debian component. ' + 'It should be one of main, contrib, or non-free.' + % options.component) + Logger.verbose = options.verbose + config = UDTConfig(options.no_conf) + if options.debian_mirror is None: + options.debian_mirror = config.get_value('DEBIAN_MIRROR') + if options.ubuntu_mirror is None: + options.ubuntu_mirror = config.get_value('UBUNTU_MIRROR') if options.uploader_name is None: options.uploader_name = ubu_email(export=False)[0] - if options.uploader_email is None: options.uploader_email = ubu_email(export=False)[1] @@ -454,19 +451,17 @@ def main(): options.release = Launchpad.distributions["ubuntu"].current_series.name if args[0].endswith(".dsc"): - dscurl = args[0] + dscfile = args[0] + if '/' in dscfile: + dscfile = fetch_dsc(dscfile) else: - if options.component not in (None, "main", "contrib", "non-free"): - parser.error('%s is not a valid Debian component. ' - 'It should be one of main, contrib, or non-free.' - % options.component) - dscurl = get_debian_dscurl(args[0], options.dist, options.release, - options.debversion, options.component) + dscfile = fetch_source_pkg(args[0], options.dist, options.debversion, + options.component, options.release, + options.debian_mirror) - Logger.verbose = options.verbose - Logger.debug('.dsc url: %s', dscurl) - sync_dsc(dscurl, options.dist, options.release, options.uploader_name, - options.uploader_email, options.bugs, options.keyid) + sync_dsc(dscfile, options.dist, options.release, options.uploader_name, + options.uploader_email, options.bugs, options.ubuntu_mirror, + options.keyid) if __name__ == "__main__": main() diff --git a/ubuntutools/test/pylint.conf b/ubuntutools/test/pylint.conf index 890de6e..cf55cbb 100644 --- a/ubuntutools/test/pylint.conf +++ b/ubuntutools/test/pylint.conf @@ -2,7 +2,8 @@ # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). -ignored-classes=Launchpad,BaseWrapper,PersonTeam,Distribution,Consumer,Credentials +# lpapicache clasess, urlparse +ignored-classes=Launchpad,BaseWrapper,PersonTeam,Distribution,Consumer,Credentials,ParseResult [FORMAT] From adc2b7869d0aa2f1e835567857a36f4772c539a7 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Wed, 29 Dec 2010 22:50:00 +0200 Subject: [PATCH 07/43] Allow reusing existing Launchpads with lpapicache --- ubuntutools/lp/lpapicache.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 0e44e92..ea5ef95 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -56,7 +56,7 @@ class _Launchpad(object): def login(self, service=service): '''Enforce a non-anonymous login.''' - if '_Launchpad__lp' not in self.__dict__: + if not self.logged_in: try: self.__lp = libsupport.get_launchpad('ubuntu-dev-tools', server=service) @@ -68,14 +68,26 @@ class _Launchpad(object): def login_anonymously(self, service=service, api_version=api_version): '''Enforce an anonymous login.''' - if '_Launchpad__lp' not in self.__dict__: + if not self.logged_in: self.__lp = launchpad.Launchpad.login_anonymously( 'ubuntu-dev-tools', service_root=service, version=api_version) else: raise AlreadyLoggedInError('Already logged in to Launchpad.') + def login_existing(self, lp): + '''Use an already logged in Launchpad object''' + if not self.logged_in: + self.__lp = lp + else: + raise AlreadyLoggedInError('Already logged in to Launchpad.') + + @property + def logged_in(self): + '''Are we logged in?''' + return '_Launchpad__lp' in self.__dict__ + def __getattr__(self, attr): - if '_Launchpad__lp' not in self.__dict__: + if not self.logged_in: self.login() return getattr(self.__lp, attr) From 5553c11dddec3f072c45ec22d3a018f2bd1ad403 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Wed, 29 Dec 2010 23:13:48 +0200 Subject: [PATCH 08/43] Beginnings of an OO ubuntutools.archive.SourcePackage --- debian/copyright | 2 +- pull-debian-debdiff | 34 ++-- pull-lp-source | 11 +- ubuntutools/archive.py | 362 +++++++++++++++++++++++++++++++++++++++++ ubuntutools/mirrors.py | 173 -------------------- 5 files changed, 386 insertions(+), 196 deletions(-) create mode 100644 ubuntutools/archive.py delete mode 100644 ubuntutools/mirrors.py diff --git a/debian/copyright b/debian/copyright index 8e9f677..f1c5b02 100644 --- a/debian/copyright +++ b/debian/copyright @@ -183,11 +183,11 @@ Files: doc/pull-debian-debdiff.1, pull-debian-debdiff, sponsor-patch, suspicious-source, + ubuntutools/archive.py, ubuntutools/builder.py, ubuntutools/config.py, ubuntutools/control.py, ubuntutools/logger.py, - ubuntutools/mirrors.py, ubuntutools/question.py, ubuntutools/sponsor_patch/*, ubuntutools/test/*, diff --git a/pull-debian-debdiff b/pull-debian-debdiff index 2caff87..3d7b292 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -18,21 +18,20 @@ # PERFORMANCE OF THIS SOFTWARE. import optparse +import os.path import subprocess import sys +import debian.debian_support import debian.changelog +from ubuntutools.archive import DebianSourcePackage from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger -from ubuntutools.mirrors import pull_source_pkg def previous_version(package, version, distance): "Given an (extracted) package, determine the version distance versions ago" - upver = version - if ':' in upver: - upver = upver.split(':', 1)[1] - upver = upver.split('-')[0] + upver = debian.debian_support.Version(version).upstream_version filename = '%s-%s/debian/changelog' % (package, upver) changelog_file = open(filename, 'r') changelog = debian.changelog.Changelog(changelog_file.read()) @@ -78,15 +77,14 @@ def main(): opts.debian_mirror = config.get_value('DEBIAN_MIRROR') if opts.debsec_mirror is None: opts.debsec_mirror = config.get_value('DEBSEC_MIRROR') + mirrors = [opts.debsec_mirror, opts.debian_mirror] Logger.normal('Downloading %s %s', package, version) - # TODO: Not all packages are main, but snapshot.debian.org should save - # the day, as it doesn't care about component. - newdsc = pull_source_pkg(('DEBSEC', 'DEBIAN'), - {'DEBSEC': opts.debsec_mirror, - 'DEBIAN': opts.debian_mirror}, - 'main', package, version, unpack=True) + #TODO: Proper snapshot and security support + newpkg = DebianSourcePackage(package, version, mirrors=mirrors) + newpkg.pull() + newpkg.unpack() if opts.fetch_only: sys.exit(0) @@ -96,14 +94,14 @@ def main(): Logger.error('No previous version could be found') sys.exit(1) Logger.normal('Downloading %s %s', package, oldversion) - olddsc = pull_source_pkg(('DEBSEC', 'DEBIAN'), - {'DEBSEC': opts.debsec_mirror, - 'DEBIAN': opts.debian_mirror}, - 'main', package, oldversion, unpack=True) - cmd = ('debdiff', olddsc, newdsc) - Logger.command(cmd) + oldpkg = DebianSourcePackage(package, oldversion, mirrors=mirrors) + oldpkg.pull() + oldpkg.unpack() + + cmd = ['debdiff', oldpkg.dsc_name, newpkg.dsc_name] difffn = newdsc[:-3] + 'debdiff' + Logger.command(cmd + ['> %s' % difffn]) debdiff_file = open(difffn, 'w') if subprocess.call(cmd, stdout=debdiff_file) > 2: Logger.error('Debdiff failed.') @@ -112,7 +110,7 @@ def main(): cmd = ('diffstat', '-p0', difffn) Logger.command(cmd) subprocess.check_call(cmd) - print difffn + print 'file://' + os.path.abspath(difffn) if __name__ == '__main__': main() diff --git a/pull-lp-source b/pull-lp-source index feb060c..af351ae 100755 --- a/pull-lp-source +++ b/pull-lp-source @@ -27,13 +27,13 @@ import os import sys from optparse import OptionParser +from ubuntutools.archive import UbuntuSourcePackage from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.lp.lpapicache import Distribution, Launchpad from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, PackageNotFoundException, PocketDoesNotExistError) -from ubuntutools.mirrors import pull_source_pkg from ubuntutools.misc import split_release_pocket def main(): @@ -83,9 +83,12 @@ def main(): Logger.error(error) sys.exit(1) - pull_source_pkg('UBUNTU', options.ubuntu_mirror, spph.getComponent(), - package, spph.getVersion(), - unpack=not options.download_only) + srcpkg = UbuntuSourcePackage(package, spph.getVersion(), + component=spph.getComponent(), + mirrors=[options.ubuntu_mirror]) + srcpkg.pull() + if not options.download_only: + srcpkg.unpack() if __name__ == '__main__': main() diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py new file mode 100644 index 0000000..4a90ed1 --- /dev/null +++ b/ubuntutools/archive.py @@ -0,0 +1,362 @@ +# archive.py - Functions for dealing with Debian source packages, archives, +# and mirrors. +# +# Copyright (C) 2010, Stefano Rivera +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +"""Pull source packages from archives. + +Approach: +1. Pull dsc from Launchpad (this is over https and can authenticate the + rest of the source package) +2. Attempt to pull the remaining files from: + 1. existing files + 2. mirrors + 3. Launchpad +3. Verify checksums. +""" + +import hashlib +import os.path +import subprocess +import urllib2 +import sys + +import debian.deb822 +import debian.debian_support + +from ubuntutools.config import UDTConfig +from ubuntutools.logger import Logger +from ubuntutools.lp.lpapicache import Launchpad, Distribution + +class DownloadError(Exception): + "Unable to pull a source package" + pass + + +class Dsc(debian.deb822.Dsc): + def get_strongest_checksum(self): + "Return alg, dict by filename of size, hash_ pairs" + if 'Checksums-Sha256' in self: + return ('sha256', + dict((entry['name'], (int(entry['size']), entry['sha256'])) + for entry in self['Checksums-Sha256'])) + if 'Checksums-Sha1' in self: + return ('sha1', + dict((entry['name'], (int(entry['size']), entry['sha1'])) + for entry in self['Checksums-Sha1'])) + return ('md5', + dict((entry['name'], (int(entry['size']), entry['md5sum'])) + for entry in self['Files'])) + + def verify_file(self, pathname): + "Verify that pathname matches the checksums in the dsc" + if os.path.isfile(pathname): + alg, checksums = self.get_strongest_checksum() + size, digest = checksums[os.path.basename(pathname)] + if os.path.getsize(pathname) != size: + return False + hash_func = getattr(hashlib, alg)() + f = open(pathname, 'rb') + while True: + buf = f.read(hash_func.block_size) + if buf == '': + break + hash_func.update(buf) + return hash_func.hexdigest() == digest + return False + + +class SourcePackage(object): + distribution = '' + + def __init__(self, package, version, component=None, lp=None, mirrors=()): + self.source = package + self.version = debian.debian_support.Version(version) + self._component = component + self._lp = lp + self._spph = None + self.mirrors = list(mirrors) + self.masters = [] + self.workdir = '.' + + @property + def lp_spph(self): + "Return the LP Source Package Publishing History entry" + if not self._spph: + if not Launchpad.logged_in: + if self._lp: + Launchpad.login_existing(self._lp) + else: + Launchpad.login_anonymously() + spph = (Distribution(self.distribution).getArchive() + .getPublishedSources( + source_name=self.source, + version=self.version.full_version, + exact_match=True, + )) + self._spph = spph[0] + return self._spph + + @property + def component(self): + "Cached archive component, in available" + if not self._component: + Logger.debug('Determining component from Launchpad') + self._component = self.lp_spph.component_name + return self._component + + @property + def dsc_name(self): + "Return the source package dsc filename for the given package" + version = self.version.upstream_version + if self.version.debian_version: + version += '-' + self.version.debian_version + return '%s_%s.dsc' % (self.source, version) + + @property + def dsc_pathname(self): + "Return the dsc_name, with the workdir path" + return os.path.join(self.workdir, self.dsc_name) + + def _mirror_url(self, mirror, filename): + "Build a source package URL on a mirror" + if self.source.startswith('lib'): + group = self.source[:4] + else: + group = self.source[0] + return os.path.join(mirror, 'pool', self.component, group, + self.source, filename) + + def _lp_url(self, filename): + "Build a source package URL on Launchpad" + return os.path.join('https://launchpad.net', self.distribution, + '+archive', 'primary', '+files', filename) + + def download_file(self, url, dsc=None): + "Download url to pathname" + filename = os.path.basename(url) + pathname = os.path.join(self.workdir, filename) + if dsc: + if dsc.verify_file(pathname): + Logger.debug('Using existing %s', filename) + return True + size = [entry['size'] for entry in dsc['Files'] + if entry['name'] == filename] + assert len(size) == 1 + size = int(size[0]) + Logger.normal('Downloading %s (%0.3f MiB)', url, + size / 1024.0 / 1024) + else: + Logger.normal('Downloading %s', url) + + in_ = urllib2.urlopen(url) + out = open(pathname, 'wb') + while True: + block = in_.read(10240) + if block == '': + break + out.write(block) + sys.stdout.write('.') + sys.stdout.flush() + in_.close() + out.close() + sys.stdout.write(' done\n') + sys.stdout.flush() + if dsc: + if not dsc.verify_file(pathname): + Logger.error('Checksum does not match.') + return False + return True + + def pull(self): + "Pull into workdir" + self.download_file(self._lp_url(self.dsc_name)) + dsc = Dsc(file(self.dsc_pathname, 'rb').read()) + for entry in dsc['Files']: + name = entry['name'] + for mirror in self.mirrors: + try: + if self.download_file(self._mirror_url(mirror, name), dsc): + break + except urllib2.HTTPError, e: + Logger.normal('HTTP Error %i: %s', e.code, str(e)) + except urllib2.URLError, e: + Logger.normal('URL Error: %s', e.reason) + else: + try: + if not self.download_file(self._lp_url(name), dsc): + raise DownloadError('Could not find %s anywhere.' + % name) + except urllib2.HTTPError, e: + Logger.normal('HTTP Error %i: %s', e.code, str(e)) + except urllib2.URLError, e: + Logger.normal('URL Error: %s', e.reason) + return True + + def unpack(self): + "Unpack in workdir" + cmd = ('dpkg-source', '-x', '--require-valid-signature', + self.dsc_name) + Logger.command(cmd) + subprocess.check_call(cmd, cwd=self.workdir) + + +class DebianSourcePackage(SourcePackage): + distribution = 'debian' + # TODO: Security support + # TODO: snapshot support + # TODO: Madison component fallback + # TODO: GPG verification fallback + +class UbuntuSourcePackage(SourcePackage): + distribution = 'ubuntu' + +# TODO: Delete everything after this point. +def pull_source_pkg(archives, mirrors, component, package, version, workdir='.', + unpack=False): + """Download a source package or die. + archives may be a list or single item (in which case mirrors can be too) + mirrors should be a dict (keyed on archive) unless archives is single""" + + if not isinstance(archives, (tuple, list)): + if not isinstance(mirrors, dict): + mirrors = {archives: mirrors} + archives = [archives] + assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives) + + for archive in archives: + filename = try_pull_from_archive(archive, mirrors.get(archive), + component, package, version, + workdir, unpack) + if filename: + return filename + + if 'DEBIAN' in archives or 'DEBSEC' in archives: + Logger.info('Trying snapshot.debian.org') + filename = try_pull_from_snapshot(package, version, workdir, unpack) + if filename: + return filename + + if 'UBUNTU' in archives: + Logger.info('Trying Launchpad') + filename = try_pull_from_lp(package, 'ubuntu', version, workdir, unpack) + if filename: + return filename + + raise Exception('Unable to locate %s/%s %s' % (package, component, version)) + +def try_pull_from_archive(archive, mirror, component, package, version, + workdir='.', unpack=False): + """Download a source package from the specified source, return filename. + Try mirror first, then master. + """ + assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU') + urls = [] + if mirror and mirror != UDTConfig.defaults[archive + '_MIRROR']: + urls.append(dsc_url(mirror, component, package, version)) + urls.append(dsc_url(UDTConfig.defaults[archive + '_MIRROR'], component, + package, version)) + + for url in urls: + cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) + Logger.command(cmd) + return_code = subprocess.call(cmd, cwd=workdir) + if return_code == 0: + return os.path.basename(url) + +def try_pull_from_snapshot(package, version, workdir='.', unpack=False): + """Download Debian source package version version from snapshot.debian.org. + Return filename. + """ + try: + import json + except ImportError: + import simplejson as json + except ImportError: + Logger.error("Please install python-simplejson.") + sys.exit(1) + + try: + srcfiles = json.load(urllib2.urlopen( + 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' + % (package, version))) + except urllib2.HTTPError: + Logger.error('Version %s of %s not found on snapshot.debian.org', + version, package) + return + + for hash_ in srcfiles['result']: + hash_ = hash_['hash'] + + try: + info = json.load(urllib2.urlopen( + 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) + except urllib2.URLError: + Logger.error('Unable to dowload info for hash.') + return + + filename = info['result'][0]['name'] + if '/' in filename: + Logger.error('Unacceptable file name: %s', filename) + return + pathname = os.path.join(workdir, filename) + + if os.path.exists(pathname): + source_file = open(pathname, 'r') + sha1 = hashlib.sha1() + sha1.update(source_file.read()) + source_file.close() + if sha1.hexdigest() == hash_: + Logger.normal('Using existing %s', filename) + continue + + Logger.normal('Downloading: %s (%0.3f MiB)', filename, + info['result'][0]['size'] / 1024.0 / 1024) + try: + in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) + out = open(pathname, 'w') + while True: + block = in_.read(10240) + if block == '': + break + out.write(block) + sys.stdout.write('.') + sys.stdout.flush() + sys.stdout.write('\n') + sys.stdout.flush() + out.close() + except urllib2.URLError: + Logger.error('Error downloading %s', filename) + return + + filename = dsc_name(package, version) + if unpack: + cmd = ('dpkg-source', '--no-check', '-x', filename) + Logger.command(cmd) + subprocess.check_call(cmd) + return filename + +def try_pull_from_lp(package, distro, version, workdir='.', unpack=False): + """Try to download the specified version of a source package from Launchpad. + Return filename. + """ + filename = dsc_name(package, version) + url = ('https://launchpad.net/%s/+archive/primary/+files/%s' + % (distro, filename)) + cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) + Logger.command(cmd) + return_code = subprocess.call(cmd, cwd=workdir) + if return_code == 0: + return filename diff --git a/ubuntutools/mirrors.py b/ubuntutools/mirrors.py deleted file mode 100644 index db100a5..0000000 --- a/ubuntutools/mirrors.py +++ /dev/null @@ -1,173 +0,0 @@ -# mirrors.py - Functions for dealing with Debian source packages and mirrors. -# -# Copyright (C) 2010, Stefano Rivera -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -import hashlib -import os.path -import subprocess -import urllib2 -import sys - -from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger - -def dsc_name(package, version): - "Return the source package dsc filename for the given package" - if ':' in version: - version = version.split(':', 1)[1] - return '%s_%s.dsc' % (package, version) - -def dsc_url(mirror, component, package, version): - "Build a source package URL" - group = package[:4] if package.startswith('lib') else package[0] - filename = dsc_name(package, version) - return os.path.join(mirror, 'pool', component, group, package, filename) - -def pull_source_pkg(archives, mirrors, component, package, version, workdir='.', - unpack=False): - """Download a source package or die. - archives may be a list or single item (in which case mirrors can be too) - mirrors should be a dict (keyed on archive) unless archives is single""" - - if not isinstance(archives, (tuple, list)): - if not isinstance(mirrors, dict): - mirrors = {archives: mirrors} - archives = [archives] - assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives) - - for archive in archives: - filename = try_pull_from_archive(archive, mirrors.get(archive), - component, package, version, - workdir, unpack) - if filename: - return filename - - if 'DEBIAN' in archives or 'DEBSEC' in archives: - Logger.info('Trying snapshot.debian.org') - filename = try_pull_from_snapshot(package, version, workdir, unpack) - if filename: - return filename - - if 'UBUNTU' in archives: - Logger.info('Trying Launchpad') - filename = try_pull_from_lp(package, 'ubuntu', version, workdir, unpack) - if filename: - return filename - - raise Exception('Unable to locate %s/%s %s' % (package, component, version)) - -def try_pull_from_archive(archive, mirror, component, package, version, - workdir='.', unpack=False): - """Download a source package from the specified source, return filename. - Try mirror first, then master. - """ - assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU') - urls = [] - if mirror and mirror != UDTConfig.defaults[archive + '_MIRROR']: - urls.append(dsc_url(mirror, component, package, version)) - urls.append(dsc_url(UDTConfig.defaults[archive + '_MIRROR'], component, - package, version)) - - for url in urls: - cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd, cwd=workdir) - if return_code == 0: - return os.path.basename(url) - -def try_pull_from_snapshot(package, version, workdir='.', unpack=False): - """Download Debian source package version version from snapshot.debian.org. - Return filename. - """ - try: - import json - except ImportError: - import simplejson as json - except ImportError: - Logger.error("Please install python-simplejson.") - sys.exit(1) - - try: - srcfiles = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' - % (package, version))) - except urllib2.HTTPError: - Logger.error('Version %s of %s not found on snapshot.debian.org', - version, package) - return - - for hash_ in srcfiles['result']: - hash_ = hash_['hash'] - - try: - info = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) - except urllib2.URLError: - Logger.error('Unable to dowload info for hash.') - return - - filename = info['result'][0]['name'] - if '/' in filename: - Logger.error('Unacceptable file name: %s', filename) - return - pathname = os.path.join(workdir, filename) - - if os.path.exists(pathname): - source_file = open(pathname, 'r') - sha1 = hashlib.sha1() - sha1.update(source_file.read()) - source_file.close() - if sha1.hexdigest() == hash_: - Logger.normal('Using existing %s', filename) - continue - - Logger.normal('Downloading: %s (%0.3f MiB)', filename, - info['result'][0]['size'] / 1024.0 / 1024) - try: - in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) - out = open(pathname, 'w') - while True: - block = in_.read(10240) - if block == '': - break - out.write(block) - sys.stdout.write('.') - sys.stdout.flush() - sys.stdout.write('\n') - sys.stdout.flush() - out.close() - except urllib2.URLError: - Logger.error('Error downloading %s', filename) - return - - filename = dsc_name(package, version) - if unpack: - cmd = ('dpkg-source', '--no-check', '-x', filename) - Logger.command(cmd) - subprocess.check_call(cmd) - return filename - -def try_pull_from_lp(package, distro, version, workdir='.', unpack=False): - """Try to download the specified version of a source package from Launchpad. - Return filename. - """ - filename = dsc_name(package, version) - url = ('https://launchpad.net/%s/+archive/primary/+files/%s' - % (distro, filename)) - cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd, cwd=workdir) - if return_code == 0: - return filename From 51fe0a4db809b022d17836ac5fa5008ef6dbd813 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 17:13:58 +0200 Subject: [PATCH 09/43] * pull-lp-source: - Support -d (LP: #681699) - str() exceptions before passing to Logger (LP: #695523) --- debian/changelog | 6 ++++-- pull-lp-source | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 6fcdc87..8ed754c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -27,7 +27,9 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low * edit-patch: Don't let cat error through if debian/source/format doesn't exist. * pull-debian-debdiff: Rewrite in Python, and use snapshot.debian.org. - * pull-lp-source: Support -d (LP: #681699) + * pull-lp-source: + - Support -d (LP: #681699) + - str() exceptions before passing to Logger (LP: #695523) * suspicious-source: Whitelist Python source code. [ Michael Bienia ] @@ -52,7 +54,7 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low * add "add-patch" that provides the non-interactive version of edit-patch - -- Benjamin Drung Mon, 27 Dec 2010 22:38:34 +0100 + -- Stefano Rivera Thu, 30 Dec 2010 17:13:05 +0200 ubuntu-dev-tools (0.108) experimental; urgency=low diff --git a/pull-lp-source b/pull-lp-source index af351ae..3645472 100755 --- a/pull-lp-source +++ b/pull-lp-source @@ -71,16 +71,16 @@ def main(): try: (release, pocket) = split_release_pocket(release) - except PocketDoesNotExistError, error: - Logger.error(error) + except PocketDoesNotExistError, e: + Logger.error(str(e)) sys.exit(1) try: spph = Distribution('ubuntu').getArchive().getSourcePackage(package, release, pocket) - except (SeriesNotFoundException, PackageNotFoundException), error: - Logger.error(error) + except (SeriesNotFoundException, PackageNotFoundException), e: + Logger.error(str(e)) sys.exit(1) srcpkg = UbuntuSourcePackage(package, spph.getVersion(), From 6c118ef1fe4be0cf5ef25476be4f6bb53d5df729 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 17:16:21 +0200 Subject: [PATCH 10/43] dsc filename support --- debian/changelog | 4 +- ubuntutools/archive.py | 121 +++++++++++++++++++++++++++++++---------- 2 files changed, 95 insertions(+), 30 deletions(-) diff --git a/debian/changelog b/debian/changelog index 8ed754c..d71fb5a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -13,6 +13,8 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low - Added ubuntu-dev-tools.5 - Support this in many u-d-t scripts, and update manpages. - Deprecate old configuration environment variables. + * New source package downloading framework in ubuntutools.archive. Use in + many scripts. * Support the combined "Name " format in UBUMAIL, DEBFULLNAME, and DEBEMAIL. (LP: #665202) * Add the beginnings of a test suite. (LP: #690386) @@ -54,7 +56,7 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low * add "add-patch" that provides the non-interactive version of edit-patch - -- Stefano Rivera Thu, 30 Dec 2010 17:13:05 +0200 + -- Stefano Rivera Thu, 30 Dec 2010 17:14:09 +0200 ubuntu-dev-tools (0.108) experimental; urgency=low diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 4a90ed1..9b91003 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -31,6 +31,7 @@ import hashlib import os.path import subprocess import urllib2 +import urlparse import sys import debian.deb822 @@ -46,6 +47,8 @@ class DownloadError(Exception): class Dsc(debian.deb822.Dsc): + "Extend deb822's Dsc with checksum verification abilities" + def get_strongest_checksum(self): "Return alg, dict by filename of size, hash_ pairs" if 'Checksums-Sha256' in self: @@ -74,22 +77,50 @@ class Dsc(debian.deb822.Dsc): if buf == '': break hash_func.update(buf) + f.close() return hash_func.hexdigest() == digest return False class SourcePackage(object): - distribution = '' + """Base class for source package downloading. + Use DebianSourcePackage or UbuntuSourcePackage instead of using this + directly. + """ + distribution = 'unknown' + + def __init__(self, package=None, version=None, component=None, + dscfile=None, lp=None, mirrors=(), workdir='.'): + "Can be initialised either using package, version or dscfile" + assert ((package is not None and version is not None) + or dscfile is not None) - def __init__(self, package, version, component=None, lp=None, mirrors=()): self.source = package - self.version = debian.debian_support.Version(version) - self._component = component + self.version = version self._lp = lp + self.workdir = workdir + + # Cached values: + self._component = component + self._dsc = None self._spph = None + + # State: + self._dsc_fetched = False + + # Mirrors + self._dsc_source = dscfile self.mirrors = list(mirrors) - self.masters = [] - self.workdir = '.' + self.masters = [UDTConfig.defaults['%s_MIRROR' + % self.distribution.upper()]] + if dscfile is not None: + d_source, d_version = os.path.basename(dscfile)[:-4].split('_') + if self.source is None: + self.source = d_source + if self.version is None: + self.version = d_version + + self.version = debian.debian_support.Version(self.version) @property def lp_spph(self): @@ -130,6 +161,14 @@ class SourcePackage(object): "Return the dsc_name, with the workdir path" return os.path.join(self.workdir, self.dsc_name) + @property + def dsc(self): + "Return a the Dsc" + if not self._dsc: + if self._dsc_fetched: + self._dsc = Dsc(file(self.dsc_pathname, 'rb').read()) + return self._dsc + def _mirror_url(self, mirror, filename): "Build a source package URL on a mirror" if self.source.startswith('lib'): @@ -144,15 +183,46 @@ class SourcePackage(object): return os.path.join('https://launchpad.net', self.distribution, '+archive', 'primary', '+files', filename) - def download_file(self, url, dsc=None): - "Download url to pathname" + def _source_urls(self, name): + "Generator of sources for name" + if self._dsc_source: + yield os.path.join(os.path.dirname(self._dsc_source), name) + for mirror in self.mirrors: + yield self._mirror_url(mirror, name) + yield self._lp_url(name) + + def pull_dsc(self): + "Retrieve dscfile and parse" + if self._dsc_source: + parsed = urlparse.urlparse(self._dsc_source) + if parsed.scheme == '': + self._dsc_source = 'file://' + os.path.abspath(self._dsc_source) + parsed = urlparse.urlparse(self._dsc_source) + + if (parsed.scheme != 'file' + or os.path.realpath(os.path.dirname(parsed.path)) + != os.path.realpath(self.workdir)): + self._download_file(self._dsc_source) + else: + self._download_file(self._lp_url(self.dsc_name)) + + self._dsc_fetched = True + + assert self.source == self.dsc['Source'] + version = debian.debian_support.Version(self.dsc['Version']) + assert self.version.upstream_version == version.upstream_version + assert self.version.debian_revision == version.debian_revision + self.version = version + + def _download_file(self, url): + "Download url to workdir" filename = os.path.basename(url) pathname = os.path.join(self.workdir, filename) - if dsc: - if dsc.verify_file(pathname): + if self.dsc and not url.endswith('.dsc'): + if self.dsc.verify_file(pathname): Logger.debug('Using existing %s', filename) return True - size = [entry['size'] for entry in dsc['Files'] + size = [entry['size'] for entry in self.dsc['Files'] if entry['name'] == filename] assert len(size) == 1 size = int(size[0]) @@ -174,46 +244,38 @@ class SourcePackage(object): out.close() sys.stdout.write(' done\n') sys.stdout.flush() - if dsc: - if not dsc.verify_file(pathname): + if self.dsc and not url.endswith('.dsc'): + if not self.dsc.verify_file(pathname): Logger.error('Checksum does not match.') return False return True def pull(self): "Pull into workdir" - self.download_file(self._lp_url(self.dsc_name)) - dsc = Dsc(file(self.dsc_pathname, 'rb').read()) - for entry in dsc['Files']: - name = entry['name'] - for mirror in self.mirrors: + if self.dsc is None: + self.pull_dsc() + for entry in self.dsc['Files']: + for url in self._source_urls(entry['name']): try: - if self.download_file(self._mirror_url(mirror, name), dsc): + if self._download_file(url): break except urllib2.HTTPError, e: Logger.normal('HTTP Error %i: %s', e.code, str(e)) except urllib2.URLError, e: Logger.normal('URL Error: %s', e.reason) else: - try: - if not self.download_file(self._lp_url(name), dsc): - raise DownloadError('Could not find %s anywhere.' - % name) - except urllib2.HTTPError, e: - Logger.normal('HTTP Error %i: %s', e.code, str(e)) - except urllib2.URLError, e: - Logger.normal('URL Error: %s', e.reason) + return False return True def unpack(self): "Unpack in workdir" - cmd = ('dpkg-source', '-x', '--require-valid-signature', - self.dsc_name) + cmd = ('dpkg-source', '-x', self.dsc_name) Logger.command(cmd) subprocess.check_call(cmd, cwd=self.workdir) class DebianSourcePackage(SourcePackage): + "Download / unpack a Debian source package" distribution = 'debian' # TODO: Security support # TODO: snapshot support @@ -221,6 +283,7 @@ class DebianSourcePackage(SourcePackage): # TODO: GPG verification fallback class UbuntuSourcePackage(SourcePackage): + "Download / unpack an Ubuntu source package" distribution = 'ubuntu' # TODO: Delete everything after this point. From 1fb2545712a5fcd187db4db2508af6e94b7460fc Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 17:16:58 +0200 Subject: [PATCH 11/43] Use *SourcePackage in syncpackage --- syncpackage | 179 +++++++++++------------------------ ubuntutools/test/pylint.conf | 5 + 2 files changed, 58 insertions(+), 126 deletions(-) diff --git a/syncpackage b/syncpackage index 807bb6c..9c040d5 100755 --- a/syncpackage +++ b/syncpackage @@ -28,9 +28,8 @@ import re import shutil import subprocess import sys -import urllib -import urlparse +from ubuntutools.archive import DebianSourcePackage, UbuntuSourcePackage from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.requestsync.mail import getDebianSrcPkg \ as requestsync_mail_getDebianSrcPkg @@ -38,28 +37,6 @@ from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg from ubuntutools.logger import Logger from ubuntutools.lp import udtexceptions from ubuntutools.lp.lpapicache import Launchpad -from ubuntutools.mirrors import pull_source_pkg - -class File(object): - def __init__(self, url, checksum, size): - self.url = url - self.name = os.path.basename(url) - self.checksum = checksum - self.size = size - - def __repr__(self): - return self.name + " (" + self.checksum + " " + self.size + \ - ") source " + str(bool(self.is_source_file())) - - def __eq__(self, other): - return self.name == other.name and self.checksum == other.checksum and \ - self.size == other.size - - def get_name(self): - return self.name - - def is_source_file(self): - return re.match(".*\.orig.*\.tar\..*", self.name) class Version(debian.debian_support.Version): @@ -88,6 +65,7 @@ class Version(debian.debian_support.Version): def is_modified_in_ubuntu(self): return 'ubuntu' in self.full_version + def remove_signature(dscname): '''Removes the signature from a .dsc file if the .dsc file is signed.''' @@ -110,24 +88,6 @@ def remove_signature(dscname): dsc_file.writelines(unsigned_file) dsc_file.close() -def dsc_getfiles(dscurl): - '''Return list of files in a .dsc file (excluding the .dsc file itself).''' - - basepath = os.path.dirname(dscurl) - dsc = debian.deb822.Dsc(urllib.urlopen(dscurl)) - - if 'Files' not in dsc: - Logger.error('No Files field found in the dsc file. Please check %s!', - os.path.basename(dscurl)) - sys.exit(1) - - files = [] - for source_file in dsc['Files']: - url = os.path.join(basepath, source_file['name']) - if not source_file['name'].endswith('.dsc'): - files.append(File(url, source_file['md5sum'], source_file['size'])) - return files - def add_fixed_bugs(changes, bugs): '''Add additional Launchpad bugs to the list of fixed bugs in changes file.''' @@ -148,44 +108,26 @@ def add_fixed_bugs(changes, bugs): return "\n".join(changes + [""]) -def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, +def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, keyid=None): - assert dscname.endswith(".dsc") - assert os.path.exists(dscname) - assert '/' not in dscname - (srcpkg, new_ver) = dscname.split('_') uploader = name + " <" + email + ">" - dscfile = debian.deb822.Dsc(file(dscname)) - if "Version" not in dscfile: - Logger.error('No Version field found in the dsc file. Please check %s!', - dscname) - sys.exit(1) - new_ver = Version(dscfile["Version"]) + src_pkg.pull_dsc() + new_ver = Version(src_pkg.dsc["Version"]) try: - ubuntu_source = getUbuntuSrcPkg(srcpkg, release) + ubuntu_source = getUbuntuSrcPkg(src_pkg.source, release) ubuntu_ver = Version(ubuntu_source.getVersion()) - ubuntu_dsc = [f for f in ubuntu_source.sourceFileUrls() - if f.endswith(".dsc")] - assert len(ubuntu_dsc) == 1 - ubuntu_dsc = ubuntu_dsc[0] + ubu_pkg = UbuntuSourcePackage(src_pkg.source, ubuntu_ver.full_version, + ubuntu_source.getComponent(), + mirrors=[ubuntu_mirror]) + ubu_pkg.pull_dsc() except udtexceptions.PackageNotFoundException: ubuntu_ver = Version('~') - ubuntu_dsc = None + ubu_pkg = None Logger.debug('Source %s: current version %s, new version %s', - srcpkg, ubuntu_ver, new_ver) - - files = dsc_getfiles(dscname) - source_files = [f for f in files if f.is_source_file()] - Logger.debug('Files: %s', str([x.get_name() for x in files])) - Logger.debug('Source files: %s', str([x.get_name() for x in source_files])) - - if ubuntu_dsc is None: - ubuntu_files = None - else: - ubuntu_files = dsc_getfiles(ubuntu_dsc) + src_pkg.source, ubuntu_ver, new_ver) # do we need the orig.tar.gz? need_orig = True @@ -193,21 +135,25 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, if ubuntu_ver.upstream_version == new_ver.upstream_version: # We need to check if all .orig*.tar.* tarballs exist in Ubuntu need_orig = False - for source_file in source_files: - ubuntu_file = [f for f in ubuntu_files - if f.get_name() == source_file.get_name()] - if len(ubuntu_file) == 0: - # The source file does not exist in Ubuntu + deb_files = dict((e['name'], e['md5sum']) for e in src_pkg.dsc['Files'] + if not e['name'].endswith('.dsc')) + ubu_files = dict((e['name'], e['md5sum']) for e in ubu_pkg.dsc['Files'] + if not e['name'].endswith('.dsc')) + for name in deb_files.iterkeys(): + if not re.match(r'.*\.orig(-[^.]+)?\.tar\.[^.]+$', name): + continue + if name in ubu_files: + if deb_files[name] != ubu_files[name]: + Logger.warn('The checksums of the file %s mismatch. ' + 'A fake sync is required.', name) + fakesync_files.append(name) + Logger.debug('Ubuntu version: %s', deb_files[name]) + Logger.debug('Debian version: %s', ubu_files[name]) + else: Logger.info('%s does not exist in Ubuntu.', - source_file.get_name()) + name) need_orig = True - elif not ubuntu_file[0] == source_file: - # The checksum of the files mismatch -> We need a fake sync - Logger.warn('The checksum of the file %s mismatch. ' - 'A fake sync is required.', source_file.get_name()) - fakesync_files.append(ubuntu_file[0]) - Logger.debug('Ubuntu version: %s', ubuntu_file[0]) - Logger.debug('Debian version: %s', source_file) + Logger.debug('Needs source tarball: %s', str(need_orig)) cur_ver = ubuntu_ver.get_related_debian_version() @@ -216,23 +162,16 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, 'setting current version to %s', ubuntu_ver.full_version, cur_ver.full_version) - # extract package - cmd = ['dpkg-source', '-x', dscname] - env = os.environ - env['DEB_VENDOR'] = 'Ubuntu' - if not Logger.verbose: - cmd.insert(1, "-q") - Logger.command(cmd) - subprocess.check_call(cmd, env=env) + src_pkg.pull() + src_pkg.unpack() # Do a fake sync if required - if len(fakesync_files) > 0: + if fakesync_files: # Download Ubuntu files (override Debian source tarballs) - pull_source_pkg('UBUNTU', ubuntu_mirror, ubuntu_source.getComponent(), - srcpkg, ubuntu_ver.full_version) + ubu_pkg.pull() # change into package directory - directory = srcpkg + '-' + new_ver.upstream_version + directory = src_pkg.source + '-' + new_ver.upstream_version Logger.command(('cd', directory)) os.chdir(directory) @@ -244,7 +183,7 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, if len(fakesync_files) == 0: # create the changes file changes_filename = "%s_%s_source.changes" % \ - (srcpkg, new_ver.strip_epoch()) + (src_pkg.source, new_ver.strip_epoch()) cmd = ["dpkg-genchanges", "-S", "-v" + cur_ver.full_version, "-DDistribution=" + release, "-DOrigin=debian/" + debian_dist, @@ -256,8 +195,7 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, if not Logger.verbose: cmd += ["-q"] Logger.command(cmd + ['>', '../' + changes_filename]) - changes = subprocess.Popen(cmd, stdout=subprocess.PIPE, - env={"DEB_VENDOR": "Ubuntu"}).communicate()[0] + changes = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] # Add additional bug numbers if len(bugs) > 0: @@ -274,7 +212,7 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, changes_file.close() # remove signature and sign package - remove_signature(dscname) + remove_signature(src_pkg.dsc_name) if keyid is not False: cmd = ["debsign", changes_filename] if not keyid is None: @@ -285,14 +223,15 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, # Create fakesync changelog entry new_ver = Version(new_ver.full_version + "fakesync1") changes_filename = "%s_%s_source.changes" % \ - (srcpkg, new_ver.strip_epoch()) + (src_pkg.source, new_ver.strip_epoch()) if len(bugs) > 0: message = "Fake sync due to mismatching orig tarball (LP: %s)." % \ (", ".join(["#" + str(b) for b in bugs])) else: message = "Fake sync due to mismatching orig tarball." - cmd = ["dch", "-v", new_ver.full_version, "-D", release, message] - env = {"DEBFULLNAME": name, "DEBEMAIL": email} + cmd = ['dch', '-v', new_ver.full_version, '--force-distribution', + '-D', release, message] + env = {'DEBFULLNAME': name, 'DEBEMAIL': email} Logger.command(cmd) subprocess.check_call(cmd, env=env) @@ -305,14 +244,12 @@ def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror, # Build source package cmd = ["debuild", "--no-lintian", "-S", "-v" + cur_ver.full_version] - env = os.environ - env['DEB_VENDOR'] = 'Ubuntu' if need_orig: cmd += ['-sa'] if keyid: cmd += ["-k" + keyid] Logger.command(cmd) - returncode = subprocess.call(cmd, env=env) + returncode = subprocess.call(cmd) if returncode != 0: Logger.error('Source-only build with debuild failed. ' 'Please check build log above.') @@ -322,6 +259,9 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): """Download the specified source package. dist, version, component, mirror can all be None. """ + if package.endswith('.dsc'): + return DebianSourcePackage(dscfile=package, mirrors=[mirror]) + if dist is None: dist = "unstable" requested_version = version @@ -352,18 +292,8 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): assert component in ('main', 'contrib', 'non-free') - return pull_source_pkg('DEBIAN', mirror, component, package, - version.full_version) - -def fetch_dsc(dscfile): - "Fetch a dsc" - url = urlparse.urlparse(dscfile) - if not url.scheme: - dscfile = 'file://' + os.path.abspath(dscfile) - cmd = ('dget', '--allow-unauthenticated', '-d', dscfile) - Logger.command(cmd) - subprocess.check_call(cmd) - return os.path.basename(url.path) + return DebianSourcePackage(package, version.full_version, component, + mirrors=[mirror]) def main(): usage = "%prog [options] <.dsc URL/path or package name>" @@ -450,16 +380,13 @@ def main(): if options.release is None: options.release = Launchpad.distributions["ubuntu"].current_series.name - if args[0].endswith(".dsc"): - dscfile = args[0] - if '/' in dscfile: - dscfile = fetch_dsc(dscfile) - else: - dscfile = fetch_source_pkg(args[0], options.dist, options.debversion, - options.component, options.release, - options.debian_mirror) + os.environ['DEB_VENDOR'] = 'Ubuntu' - sync_dsc(dscfile, options.dist, options.release, options.uploader_name, + src_pkg = fetch_source_pkg(args[0], options.dist, options.debversion, + options.component, options.release, + options.debian_mirror) + + sync_dsc(src_pkg, options.dist, options.release, options.uploader_name, options.uploader_email, options.bugs, options.ubuntu_mirror, options.keyid) diff --git a/ubuntutools/test/pylint.conf b/ubuntutools/test/pylint.conf index cf55cbb..e93c872 100644 --- a/ubuntutools/test/pylint.conf +++ b/ubuntutools/test/pylint.conf @@ -13,3 +13,8 @@ max-line-length=80 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' + +[BASIC] + +# Allow variables called e, f, lp +good-names=i,j,k,ex,Run,_,e,f,lp From 3013ee034a77b53383fbb2913ded6b00a223df7f Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 18:05:55 +0200 Subject: [PATCH 12/43] Use *SourcePackage in backportpackage --- backportpackage | 95 ++++++++++++++++-------------------------- ubuntutools/archive.py | 6 ++- 2 files changed, 40 insertions(+), 61 deletions(-) diff --git a/backportpackage b/backportpackage index cb3309c..5c981de 100755 --- a/backportpackage +++ b/backportpackage @@ -25,14 +25,13 @@ import subprocess import sys import tempfile -from debian.deb822 import Dsc from launchpadlib.launchpad import Launchpad import lsb_release +from ubuntutools.archive import UbuntuSourcePackage from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.builder import get_builder from ubuntutools.logger import Logger -from ubuntutools.mirrors import pull_source_pkg from ubuntutools.question import YesNoQuestion def error(msg): @@ -166,42 +165,26 @@ def find_release_package(launchpad, package, version, source_release): return srcpkg -def find_version_package(launchpad, package, version): - ubuntu = launchpad.distributions['ubuntu'] - archive = ubuntu.main_archive - try: - # Might get more than one (i.e. same version in multiple - # releases), but they should all be identical - return archive.getPublishedSources(source_name=package, - version=version)[0] - except IndexError: - error('Version %s of package %s was never published in Ubuntu.' % - (version, package)) - -def fetch_package(launchpad, mirror, workdir, package, version, source_release): - "Returns the path to the .dsc file that was fetched" - +def find_package(launchpad, mirror, workdir, package, version, source_release): + "Returns the SourcePackage" if package.endswith('.dsc'): - cmd = ('dget', '--download-only', '--allow-unauthenticated', package) - Logger.command(cmd) - ret = subprocess.call(cmd, cwd=workdir) - if ret == 0: - return os.path.join(workdir, os.path.basename(package)) - sys.exit(1) + return UbuntuSourcePackage(version=version, dscfile=package, + workdir=workdir, lp=launchpad, + mirrors=[mirror]) if not source_release and not version: source_release = launchpad.distributions['ubuntu'].current_series.name + component = None # If source_release is specified, then version is just for verification if source_release: srcpkg = find_release_package(launchpad, package, version, source_release) - else: - srcpkg = find_version_package(launchpad, package, version) + version = srcpkg.source_package_version + component = srcpkg.component_name - version = srcpkg.source_package_version - return pull_source_pkg('UBUNTU', mirror, srcpkg.component_name, package, - version, workdir=workdir, unpack=False) + return UbuntuSourcePackage(package, version, component, workdir=workdir, + lp=launchpad, mirrors=[mirror]) def get_backport_version(version, suffix, upload, release): backport_version = version + ('~%s1' % release) @@ -217,7 +200,7 @@ def get_backport_dist(upload, release): else: return release -def do_build(workdir, package, release, bp_version, builder, update): +def do_build(workdir, dsc, release, builder, update): builder = get_builder(builder) if not builder: return @@ -226,12 +209,11 @@ def do_build(workdir, package, release, bp_version, builder, update): if 0 != builder.update(release): sys.exit(1) - return builder.build(os.path.join(workdir, - '%s_%s.dsc' % (package, bp_version)), + return builder.build(os.path.join(workdir, dsc), release, os.path.join(workdir, "buildresult")) -def do_upload(workdir, package, bp_version, upload, prompt): +def do_upload(workdir, package, bp_version, changes, upload, prompt): print 'Please check %s %s in file://%s carefully!' % \ (package, bp_version, workdir) if prompt or upload == 'ubuntu': @@ -240,20 +222,21 @@ def do_upload(workdir, package, bp_version, upload, prompt): if answer == "no": return - changes_file = '%s_%s_source.changes' % (package, bp_version) - check_call(['dput', upload, changes_file], cwd=workdir) + check_call(['dput', upload, changes], cwd=workdir) +def do_backport(workdir, pkg, suffix, release, build, builder, update, upload, + prompt): + dirname = '%s-%s' % (pkg.source, release) + pkg.unpack(dirname) + srcdir = os.path.join(workdir, dirname) -def do_backport(workdir, package, dscfile, version, suffix, release, build, - builder, update, upload, prompt): - check_call(['dpkg-source', '-x', dscfile, package], cwd=workdir) - srcdir = os.path.join(workdir, package) - - bp_version = get_backport_version(version, suffix, upload, release) + bp_version = get_backport_version(pkg.version.full_version, suffix, + upload, release) bp_dist = get_backport_dist(upload, release) check_call(['dch', - '--force-bad-version', + '--allow-lower-version', + '--force-distribution', '--preserve', '--newversion', bp_version, '--distribution', bp_dist, @@ -261,15 +244,14 @@ def do_backport(workdir, package, dscfile, version, suffix, release, build, cwd=srcdir) check_call(['debuild', '--no-lintian', '-S', '-sa'], cwd=srcdir) - if ':' in bp_version: - bp_version = bp_version[bp_version.find(':')+1:] + fn_base = pkg.source + '_' + bp_version.split(':', 1)[-1] if build: - if 0 != do_build(workdir, package, release, bp_version, builder, - update): + if 0 != do_build(workdir, fn_base + '.dsc', release, builder, update): sys.exit(1) if upload: - do_upload(workdir, package, bp_version, upload, prompt) + do_upload(workdir, pkg.source, bp_version, fn_base + '.changes', + upload, prompt) shutil.rmtree(srcdir) @@ -298,22 +280,17 @@ def main(args): os.makedirs(workdir) try: - dscfile = fetch_package(launchpad, - opts.ubuntu_mirror, - workdir, - package_or_dsc, - opts.version, - opts.source_release) - - dsc = Dsc(open(os.path.join(workdir, dscfile))) - package = dsc['Source'] - version = dsc['Version'] + pkg = find_package(launchpad, + opts.ubuntu_mirror, + workdir, + package_or_dsc, + opts.version, + opts.source_release) + pkg.pull() for release in opts.dest_releases: do_backport(workdir, - package, - dscfile, - version, + pkg, opts.suffix, release, opts.build, diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 9b91003..1e69428 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -267,9 +267,11 @@ class SourcePackage(object): return False return True - def unpack(self): + def unpack(self, destdir=None): "Unpack in workdir" - cmd = ('dpkg-source', '-x', self.dsc_name) + cmd = ['dpkg-source', '-x', self.dsc_name] + if destdir: + cmd.append(destdir) Logger.command(cmd) subprocess.check_call(cmd, cwd=self.workdir) From a2da2da87bce20aefcddb2597d12aef7a67b1a72 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 19:50:07 +0200 Subject: [PATCH 13/43] Snapshot and rmadison support --- pull-debian-debdiff | 2 +- ubuntutools/archive.py | 283 +++++++++++++++++++---------------------- 2 files changed, 130 insertions(+), 155 deletions(-) diff --git a/pull-debian-debdiff b/pull-debian-debdiff index 3d7b292..6483a1f 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -100,7 +100,7 @@ def main(): oldpkg.unpack() cmd = ['debdiff', oldpkg.dsc_name, newpkg.dsc_name] - difffn = newdsc[:-3] + 'debdiff' + difffn = newpkg.dsc_name[:-3] + 'debdiff' Logger.command(cmd + ['> %s' % difffn]) debdiff_file = open(difffn, 'w') if subprocess.call(cmd, stdout=debdiff_file) > 2: diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 1e69428..816a952 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -39,7 +39,10 @@ import debian.debian_support from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger -from ubuntutools.lp.lpapicache import Launchpad, Distribution +from ubuntutools.lp.lpapicache import (Launchpad, Distribution, + SourcePackagePublishingHistory) +from ubuntutools.requestsync.mail import (SourcePackagePublishingHistory + as rmadison_SPPH) class DownloadError(Exception): "Unable to pull a source package" @@ -137,7 +140,7 @@ class SourcePackage(object): version=self.version.full_version, exact_match=True, )) - self._spph = spph[0] + self._spph = SourcePackagePublishingHistory(spph[0]) return self._spph @property @@ -145,7 +148,7 @@ class SourcePackage(object): "Cached archive component, in available" if not self._component: Logger.debug('Determining component from Launchpad') - self._component = self.lp_spph.component_name + self._component = self.lp_spph.getComponent() return self._component @property @@ -202,10 +205,17 @@ class SourcePackage(object): if (parsed.scheme != 'file' or os.path.realpath(os.path.dirname(parsed.path)) != os.path.realpath(self.workdir)): - self._download_file(self._dsc_source) + if not self._download_file(self._dsc_source, self.dsc_name): + raise DownloadError('dsc not found') else: - self._download_file(self._lp_url(self.dsc_name)) + if not self._download_file(self._lp_url(self.dsc_name), + self.dsc_name): + raise DownloadError('dsc not found') + self._check_dsc() + def _check_dsc(self): + "Check that the dsc matches what we are expecting" + assert os.path.exists(self.dsc_name) self._dsc_fetched = True assert self.source == self.dsc['Source'] @@ -214,9 +224,11 @@ class SourcePackage(object): assert self.version.debian_revision == version.debian_revision self.version = version - def _download_file(self, url): - "Download url to workdir" - filename = os.path.basename(url) + def _download_file(self, url, filename): + "Download url to filename in workdir." + logurl = url + if os.path.basename(url) != filename: + logurl += ' -> ' + filename pathname = os.path.join(self.workdir, filename) if self.dsc and not url.endswith('.dsc'): if self.dsc.verify_file(pathname): @@ -226,12 +238,16 @@ class SourcePackage(object): if entry['name'] == filename] assert len(size) == 1 size = int(size[0]) - Logger.normal('Downloading %s (%0.3f MiB)', url, + Logger.normal('Downloading %s (%0.3f MiB)', logurl, size / 1024.0 / 1024) else: - Logger.normal('Downloading %s', url) + Logger.normal('Downloading %s', logurl) + + try: + in_ = urllib2.urlopen(url) + except urllib2.HTTPError, e: + return False - in_ = urllib2.urlopen(url) out = open(pathname, 'wb') while True: block = in_.read(10240) @@ -255,9 +271,10 @@ class SourcePackage(object): if self.dsc is None: self.pull_dsc() for entry in self.dsc['Files']: - for url in self._source_urls(entry['name']): + name = entry['name'] + for url in self._source_urls(name): try: - if self._download_file(url): + if self._download_file(url, name): break except urllib2.HTTPError, e: Logger.normal('HTTP Error %i: %s', e.code, str(e)) @@ -279,149 +296,107 @@ class SourcePackage(object): class DebianSourcePackage(SourcePackage): "Download / unpack a Debian source package" distribution = 'debian' - # TODO: Security support - # TODO: snapshot support - # TODO: Madison component fallback + + def __init__(self, *args, **kwargs): + super(DebianSourcePackage, self).__init__(*args, **kwargs) + self.masters.append(UDTConfig.defaults['DEBSEC_MIRROR']) + # Cached values: + self._snapshot_list = None + + # Overridden methods: + @property + def lp_spph(self): + "Return the LP Source Package Publishing History entry" + if not self._spph: + try: + return super(DebianSourcePackage, self).lp_spph + except IndexError: + pass + + Logger.normal('Using rmadison for component determination') + p = subprocess.Popen(('rmadison', '-u', 'debian', self.source), + stdout=subprocess.PIPE) + rmadison = p.communicate()[0] + comp = 'main' + for line in rmadison.strip().splitlines(): + pkg, ver, dist, archs = [x.strip() for x in line.split('|')] + comp = 'main' + if '/' in dist: + dist, comp = dist.split('/') + if ver == self.version.full_version: + self._spph = rmadison_SPPH(pkg, ver, comp) + return self._spph + + Logger.normal('Guessing component from most recent upload') + self._spph = rmadison_SPPH(self.source, self.version.full_version, + comp) + return self._spph + + def _source_urls(self, name): + "Generator of sources for name" + it = super(DebianSourcePackage, self)._source_urls(name) + while True: + try: + yield it.next() + except StopIteration: + break + if self._snapshot_list: + yield self._snapshot_url(name) + + def pull_dsc(self): + "Retrieve dscfile and parse" + try: + super(DebianSourcePackage, self).pull_dsc() + return + except DownloadError: + pass + + # Not all Debian Source packages get imported to LP + # (or the importer could be lagging) + for url in self._source_urls(self.dsc_name): + if self._download_file(url, self.dsc_name): + break + else: + raise DownloadError('dsc could not be found anywhere') + self._check_dsc() + + # Local methods: + @property + def snapshot_list(self): + "Return a filename -> hash dictionary from snapshot.debian.org" + if self._snapshot_list is None: + try: + import json + except ImportError: + import simplejson as json + except ImportError: + Logger.error("Please install python-simplejson.") + sys.exit(1) + + try: + srcfiles = json.load(urllib2.urlopen( + 'http://snapshot.debian.org' + '/mr/package/%s/%s/srcfiles?fileinfo=1' + % (self.source, self.version.full_version))) + except urllib2.HTTPError: + Logger.error('Version %s of %s not found on ' + 'snapshot.debian.org', + self.version.full_version, self.source) + self._snapshot_list = False + return False + self._snapshot_list = dict((info[0]['name'], hash_) + for hash_, info + in srcfiles['fileinfo'].iteritems()) + return self._snapshot_list + + def _snapshot_url(self, name): + "Return the snapshot.debian.org URL for name" + return os.path.join('http://snapshot.debian.org/file', + self.snapshot_list[name]) + # TODO: GPG verification fallback + class UbuntuSourcePackage(SourcePackage): "Download / unpack an Ubuntu source package" distribution = 'ubuntu' - -# TODO: Delete everything after this point. -def pull_source_pkg(archives, mirrors, component, package, version, workdir='.', - unpack=False): - """Download a source package or die. - archives may be a list or single item (in which case mirrors can be too) - mirrors should be a dict (keyed on archive) unless archives is single""" - - if not isinstance(archives, (tuple, list)): - if not isinstance(mirrors, dict): - mirrors = {archives: mirrors} - archives = [archives] - assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives) - - for archive in archives: - filename = try_pull_from_archive(archive, mirrors.get(archive), - component, package, version, - workdir, unpack) - if filename: - return filename - - if 'DEBIAN' in archives or 'DEBSEC' in archives: - Logger.info('Trying snapshot.debian.org') - filename = try_pull_from_snapshot(package, version, workdir, unpack) - if filename: - return filename - - if 'UBUNTU' in archives: - Logger.info('Trying Launchpad') - filename = try_pull_from_lp(package, 'ubuntu', version, workdir, unpack) - if filename: - return filename - - raise Exception('Unable to locate %s/%s %s' % (package, component, version)) - -def try_pull_from_archive(archive, mirror, component, package, version, - workdir='.', unpack=False): - """Download a source package from the specified source, return filename. - Try mirror first, then master. - """ - assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU') - urls = [] - if mirror and mirror != UDTConfig.defaults[archive + '_MIRROR']: - urls.append(dsc_url(mirror, component, package, version)) - urls.append(dsc_url(UDTConfig.defaults[archive + '_MIRROR'], component, - package, version)) - - for url in urls: - cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd, cwd=workdir) - if return_code == 0: - return os.path.basename(url) - -def try_pull_from_snapshot(package, version, workdir='.', unpack=False): - """Download Debian source package version version from snapshot.debian.org. - Return filename. - """ - try: - import json - except ImportError: - import simplejson as json - except ImportError: - Logger.error("Please install python-simplejson.") - sys.exit(1) - - try: - srcfiles = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' - % (package, version))) - except urllib2.HTTPError: - Logger.error('Version %s of %s not found on snapshot.debian.org', - version, package) - return - - for hash_ in srcfiles['result']: - hash_ = hash_['hash'] - - try: - info = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) - except urllib2.URLError: - Logger.error('Unable to dowload info for hash.') - return - - filename = info['result'][0]['name'] - if '/' in filename: - Logger.error('Unacceptable file name: %s', filename) - return - pathname = os.path.join(workdir, filename) - - if os.path.exists(pathname): - source_file = open(pathname, 'r') - sha1 = hashlib.sha1() - sha1.update(source_file.read()) - source_file.close() - if sha1.hexdigest() == hash_: - Logger.normal('Using existing %s', filename) - continue - - Logger.normal('Downloading: %s (%0.3f MiB)', filename, - info['result'][0]['size'] / 1024.0 / 1024) - try: - in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) - out = open(pathname, 'w') - while True: - block = in_.read(10240) - if block == '': - break - out.write(block) - sys.stdout.write('.') - sys.stdout.flush() - sys.stdout.write('\n') - sys.stdout.flush() - out.close() - except urllib2.URLError: - Logger.error('Error downloading %s', filename) - return - - filename = dsc_name(package, version) - if unpack: - cmd = ('dpkg-source', '--no-check', '-x', filename) - Logger.command(cmd) - subprocess.check_call(cmd) - return filename - -def try_pull_from_lp(package, distro, version, workdir='.', unpack=False): - """Try to download the specified version of a source package from Launchpad. - Return filename. - """ - filename = dsc_name(package, version) - url = ('https://launchpad.net/%s/+archive/primary/+files/%s' - % (distro, filename)) - cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd, cwd=workdir) - if return_code == 0: - return filename From eed93046646cf2753b6c54d8e8c146e0839feb9e Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 20:11:13 +0200 Subject: [PATCH 14/43] GPG verification support --- ubuntutools/archive.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 816a952..e0bade6 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -213,7 +213,7 @@ class SourcePackage(object): raise DownloadError('dsc not found') self._check_dsc() - def _check_dsc(self): + def _check_dsc(self, verify_signature=False): "Check that the dsc matches what we are expecting" assert os.path.exists(self.dsc_name) self._dsc_fetched = True @@ -224,6 +224,25 @@ class SourcePackage(object): assert self.version.debian_revision == version.debian_revision self.version = version + gpg_info = self.dsc.get_gpg_info() + if gpg_info.valid(): + message = 'Valid signature' + else: + message = 'Signature on %s could not be verified' % self.dsc_name + if 'GOODSIG' in gpg_info: + message = 'Good signature by %s (0x%s)' % (gpg_info['GOODSIG'][1], + gpg_info['GOODSIG'][0]) + elif 'VALIDSIG' in gpg_info: + message = 'Valid signature by 0x%s' % gpg_info['VALIDSIG'][0] + if verify_signature: + if gpg_info.valid(): + Logger.normal(message) + else: + Logger.error(message) + raise DownloadError(message) + else: + Logger.info(message) + def _download_file(self, url, filename): "Download url to filename in workdir." logurl = url @@ -245,7 +264,7 @@ class SourcePackage(object): try: in_ = urllib2.urlopen(url) - except urllib2.HTTPError, e: + except urllib2.HTTPError: return False out = open(pathname, 'wb') @@ -358,7 +377,7 @@ class DebianSourcePackage(SourcePackage): break else: raise DownloadError('dsc could not be found anywhere') - self._check_dsc() + self._check_dsc(verify_signature=True) # Local methods: @property @@ -394,8 +413,6 @@ class DebianSourcePackage(SourcePackage): return os.path.join('http://snapshot.debian.org/file', self.snapshot_list[name]) - # TODO: GPG verification fallback - class UbuntuSourcePackage(SourcePackage): "Download / unpack an Ubuntu source package" From 620f3fb25a1c97eadb5290e536141185ccce6d54 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 30 Dec 2010 20:14:15 +0200 Subject: [PATCH 15/43] Typo in snapshot support --- ubuntutools/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index e0bade6..54f9b9e 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -359,7 +359,7 @@ class DebianSourcePackage(SourcePackage): yield it.next() except StopIteration: break - if self._snapshot_list: + if self.snapshot_list: yield self._snapshot_url(name) def pull_dsc(self): From 36f7f7729f69062a9a3b47e4e5fcd3295355b301 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 12:37:52 +0200 Subject: [PATCH 16/43] Man page update --- doc/backportpackage.1 | 7 +++++ doc/requestsync.1 | 1 + doc/syncpackage.1 | 60 +++++++++++++++++++++++++++++++++++++----- doc/ubuntu-dev-tools.5 | 2 ++ 4 files changed, 63 insertions(+), 7 deletions(-) diff --git a/doc/backportpackage.1 b/doc/backportpackage.1 index 3ffb615..3a21b36 100644 --- a/doc/backportpackage.1 +++ b/doc/backportpackage.1 @@ -94,6 +94,13 @@ the default of "production". Do not read any configuration files, or configuration from environment variables. .SH ENVIRONMENT +.TP +.BR DEBFULLNAME ", " DEBEMAIL ", " UBUMAIL +Used to determine the uploader (if not supplied as options). +See +.BR ubuntu\-dev\-tools (5) +for details. +.P All of the \fBCONFIGURATION VARIABLES\fR below are also supported as environment variables. Variables in the environment take precedence to those in configuration diff --git a/doc/requestsync.1 b/doc/requestsync.1 index c2863e2..a6593e8 100644 --- a/doc/requestsync.1 +++ b/doc/requestsync.1 @@ -121,6 +121,7 @@ The default value for \fB--lpinstance\fR. .SH SEE ALSO .BR rmadison (1), +.BR syncpackage (1), .BR ubuntu\-dev\-tools (5) .SH AUTHOR diff --git a/doc/syncpackage.1 b/doc/syncpackage.1 index 7677860..22f10dc 100644 --- a/doc/syncpackage.1 +++ b/doc/syncpackage.1 @@ -12,7 +12,8 @@ primary archive or PPA starting from a pristine Debian package. Debian ones, as the common script used by Ubuntu archive administrators does, this way you can preserve source files integrity between the two distributions. .PP -\fBsyncpackage\fR will detect source tarballs with mismatching checksums and will automatically create fake syncs instead. +\fBsyncpackage\fR will detect source tarballs with mismatching checksums +and will automatically create fake syncs instead. .SH WARNING The use of \fBsyncpackage\fR is discouraged by the Ubuntu Archive Administrators, as it introduces an unnecessary window for error. @@ -40,12 +41,15 @@ Specify the version to sync from. Specify the component to sync from. .TP \fB\-v\fR, \fB\-\-verbose\fR -print more information +Display more progress information. .TP -\fB\-n\fI UPLOADER\fR, \fB\-\-uploader\fR=\fIUPLOADER\fR -Use UPLOADER as the name and email address of the -maintainer for this upload instead of evaluating -DEBFULLNAME and DEBEMAIL. +\fB\-n\fI UPLOADER_NAME\fR, \fB\-\-uploader\-name\fR=\fIUPLOADER_NAME\fR +Use UPLOADER_NAME as the name of the maintainer for this upload instead +of evaluating DEBFULLNAME and UBUMAIL. +.TP +\fB\-e\fI UPLOADER_EMAIL\fR, \fB\-\-uploader\-email\fR=\fIUPLOADER_EMAIL\fR +Use UPLOADER_EMAIL as the email address of the maintainer for this +upload instead of evaluating DEBEMAIL and UBUMAIL. .TP \fB\-k\fI KEYID\fR, \fB\-\-key\fR=\fIKEYID\fR Specify the key ID to be used for signing. @@ -55,7 +59,49 @@ Do not sign the upload. .TP \fB\-b\fI BUG\fR, \fB\-\-bug\fR=\fIBUG\fR Mark a Launchpad bug as being fixed by this upload. -.PP +.TP +.B \-d \fIDEBIAN_MIRROR\fR, \fB\-\-debian\-mirror\fR=\fIDEBIAN_MIRROR\fR +Use the specified mirror. +Should be in the form \fBhttp://ftp.debian.org/debian\fR. +If the package isn't found on this mirror, \fBsyncpackage\fR will fall +back to the default mirror. +.TP +.B \-s \fIUBUNTU_MIRROR\fR, \fB\-\-debsec\-mirror\fR=\fIUBUNTU_MIRROR\fR +Use the specified Debian security mirror. +Should be in the form \fBhttp://archive.ubuntu.com/ubuntu\fR. +If the package isn't found on this mirror, \fBsyncpackage\fR will fall +back to the default mirror. +.TP +.B \-\-no\-conf +Do not read any configuration files, or configuration from environment +variables. +.SH ENVIRONMENT +.TP +.BR DEBFULLNAME ", " DEBEMAIL ", " UBUMAIL +Used to determine the uploader (if not supplied as options). +See +.BR ubuntu\-dev\-tools (5) +for details. +.P +All of the \fBCONFIGURATION VARIABLES\fR below are also supported as +environment variables. +Variables in the environment take precedence to those in configuration +files. +.SH CONFIGURATION VARIABLES +The following variables can be set in the environment or in +.BR ubuntu\-dev\-tools (5) +configuration files. +In each case, the script\-specific variable takes precedence over the +package\-wide variable. +.TP +.BR SYNCPACKAGE_DEBIAN_MIRROR ", " UBUNTUTOOLS_DEBIAN_MIRROR +The default value for \fB\-\-debian\-mirror\fR. +.TP +.BR SYNCPACKAGE_UBUNTU_MIRROR ", " UBUNTUTOOLS_DEBSEC_MIRROR +The default value for \fB\-\-ubuntu\-mirror\fR. +.SH SEE ALSO +.BR requestsync (1), +.BR ubuntu\-dev\-tools (5) .SH AUTHOR \fBsyncpackage\fR was written by Martin Pitt and Benjamin Drung . .PP diff --git a/doc/ubuntu-dev-tools.5 b/doc/ubuntu-dev-tools.5 index 2c17596..adbd70f 100644 --- a/doc/ubuntu-dev-tools.5 +++ b/doc/ubuntu-dev-tools.5 @@ -43,6 +43,8 @@ In addition, several scripts use the following environment variables: .B UBUMAIL Overrides \fBDEBEMAIL\fR and \fBDEBFULLNAME\fR when the target is clearly Ubuntu. +Can either contain an e-mail address or \fBFull Name +\fR. .TP .BR DEBEMAIL ", " DEBFULLNAME From a3b82ff2e9070c3107ce4d0fd325938259c10f6c Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 17:30:39 +0200 Subject: [PATCH 17/43] Correct dsc location assertion --- ubuntutools/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 54f9b9e..1fba7bd 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -215,7 +215,7 @@ class SourcePackage(object): def _check_dsc(self, verify_signature=False): "Check that the dsc matches what we are expecting" - assert os.path.exists(self.dsc_name) + assert os.path.exists(self.dsc_pathname) self._dsc_fetched = True assert self.source == self.dsc['Source'] From 26c21988387e635868f1fc77fc80aef976b4f203 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 17:36:20 +0200 Subject: [PATCH 18/43] Help testing: Separate rmadison out into a function --- ubuntutools/archive.py | 48 +++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 1fba7bd..798d495 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -333,17 +333,14 @@ class DebianSourcePackage(SourcePackage): pass Logger.normal('Using rmadison for component determination') - p = subprocess.Popen(('rmadison', '-u', 'debian', self.source), - stdout=subprocess.PIPE) - rmadison = p.communicate()[0] comp = 'main' - for line in rmadison.strip().splitlines(): - pkg, ver, dist, archs = [x.strip() for x in line.split('|')] - comp = 'main' - if '/' in dist: - dist, comp = dist.split('/') - if ver == self.version.full_version: - self._spph = rmadison_SPPH(pkg, ver, comp) + for record in rmadison(self.distribution, self.source): + if record.get('source') != self.source: + continue + comp = record['component'] + if record['version'] == self.version.full_version: + self._spph = rmadison_SPPH(record['source'], + record['version'], comp) return self._spph Logger.normal('Guessing component from most recent upload') @@ -417,3 +414,34 @@ class DebianSourcePackage(SourcePackage): class UbuntuSourcePackage(SourcePackage): "Download / unpack an Ubuntu source package" distribution = 'ubuntu' + + +def rmadison(url, package): + "Call rmadison and parse the result" + p = subprocess.Popen(('rmadison', '-u', url, package), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True) + output = p.communicate()[0] + assert p.wait() == 0 + for line in output.strip().splitlines(): + pkg, ver, dist, archs = [x.strip() for x in line.split('|')] + comp = 'main' + if '/' in dist: + dist, comp = dist.split('/') + archs = set(x.strip() for x in archs.split(',')) + if 'source' in archs: + yield { + 'source': pkg, + 'version': ver, + 'distribution': dist, + 'component': comp, + } + archs.discard('source') + if archs: + yield { + 'binary': pkg, + 'version': ver, + 'distribution': dist, + 'component': comp, + 'architectures': archs, + } From b78922b14a5462e2b8000b76dad8d7c204e1572e Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 17:36:38 +0200 Subject: [PATCH 19/43] Help testing: Write download progress to Logger's streams --- ubuntutools/archive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 798d495..53a6fc3 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -273,12 +273,12 @@ class SourcePackage(object): if block == '': break out.write(block) - sys.stdout.write('.') - sys.stdout.flush() + Logger.stdout.write('.') + Logger.stdout.flush() in_.close() out.close() - sys.stdout.write(' done\n') - sys.stdout.flush() + Logger.stdout.write(' done\n') + Logger.stdout.flush() if self.dsc and not url.endswith('.dsc'): if not self.dsc.verify_file(pathname): Logger.error('Checksum does not match.') From 531aaba3a0acab82b6b5280271118d9585729b29 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 17:37:24 +0200 Subject: [PATCH 20/43] Basic test suite for ubuntutools.archive --- test-data/example_1.0-1.debian.tar.gz | Bin 0 -> 1170 bytes test-data/example_1.0-1.dsc | 17 ++++ test-data/example_1.0.orig.tar.gz | Bin 0 -> 170 bytes ubuntutools/test/test_archive.py | 135 ++++++++++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 test-data/example_1.0-1.debian.tar.gz create mode 100644 test-data/example_1.0-1.dsc create mode 100644 test-data/example_1.0.orig.tar.gz create mode 100644 ubuntutools/test/test_archive.py diff --git a/test-data/example_1.0-1.debian.tar.gz b/test-data/example_1.0-1.debian.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..0dfd3a7e1658ee5c3f79f35e259b6ab629b90418 GIT binary patch literal 1170 zcmV;D1a12tiwFP!000021MOIClbbdW_SfoHY}2%pq&5cInAYtj;cR@z3|G&il zfj&O+e~dlFe|bR$_= zOkmH%hAcTNIg>Z)f5{WFNLV59OZnuhm`~2JvPgR~TG1jE6Tsb;(Y&G@Uc5d7OVc$| z|NAA0WKyIFPetz?<)Se^)PB|sLYB$qW3v0iKbvndCeg{7<7krSeqUS_ z+F?s*CExY0vENT=)xV}|)e7}vYWwuMcvDtqTb9!wIK4VE%mF!O5~WF?sywNSO;#nT zR#4eLB!yl|uX?g9b3w=)g^{(d$n?1PCQWV7{kqKJu?;N*+H<2RfhFb0bd@?lmWrTV zP+PhS`!Z!)Y%#&~k>4p3lCyO!k+~TYAj+-0PWXZRNpVAt%!g|xGr&RBmbHW}rGNq% zuQ*sSh-J^ocy6gMF&R6?M=v=6Lw(&EK-N09~(P|vp$_3xK@Z1Z>pE@W=AZVt?L5MR~(XB)m?9h#r z&0rNP@u1O&N_^*_QmuKj4CqFqXX%8v01La(JO6hK9_guZoBw%@WPG&uzeDRj|BuJ2 z{u`KW{dW?4_ieuxyuW5ef1iA$(AyI7{e*mzX7Kziv +Standards-Version: 3.9.1 +Build-Depends: debhelper (>= 7.0.50~) +Checksums-Sha1: + 13c309f70155ab5be50553ce72f3344c48851013 170 example_1.0.orig.tar.gz + ae9dcd225000d24128b8eccbdc472abcfa9b60ee 1170 example_1.0-1.debian.tar.gz +Checksums-Sha256: + 4908cb2ce47c9835fb74476ab5dba0298da7a56a389328b845d07575f5ea6276 170 example_1.0.orig.tar.gz + ca366dc1ae0efa1df6537f6882e87134a0409e1589ca18daf56d065beb9cfce3 1170 example_1.0-1.debian.tar.gz +Files: + 8054fdf3653160e13d9539ca36c2e311 170 example_1.0.orig.tar.gz + 8538e8b6cb8162d5c6886b18c132a1c8 1170 example_1.0-1.debian.tar.gz diff --git a/test-data/example_1.0.orig.tar.gz b/test-data/example_1.0.orig.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1d3482e74e5d142569d08e073d34700829f1d6e1 GIT binary patch literal 170 zcmb2|=3odpBJ0b*{Pvsv+^E0^~y8u$LD{?^G?)ueSLS^ TkpUUhykT4X#;A`$gMk46H^NWl literal 0 HcmV?d00001 diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py new file mode 100644 index 0000000..cafa1c9 --- /dev/null +++ b/ubuntutools/test/test_archive.py @@ -0,0 +1,135 @@ +# test_archive.py - Test suite for ubuntutools.archive +# +# Copyright (C) 2010, Stefano Rivera +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +from __future__ import with_statement + +import __builtin__ +import os.path +import shutil +import StringIO +import subprocess +import sys +import tempfile +import urllib2 +import urlparse + +import mox + +from ubuntutools.archive import Dsc, DebianSourcePackage, UbuntuSourcePackage +from ubuntutools.logger import Logger +from ubuntutools.test import unittest + +class DscVerificationTestCase(mox.MoxTestBase, unittest.TestCase): + def setUp(self): + super(DscVerificationTestCase, self).setUp() + with open('test-data/example_1.0-1.dsc', 'rb') as f: + self.dsc = Dsc(f.read()) + + def tearDown(self): + super(DscVerificationTestCase, self).tearDown() + + def test_good(self): + self.assertTrue(self.dsc.verify_file( + 'test-data/example_1.0.orig.tar.gz')) + self.assertTrue(self.dsc.verify_file( + 'test-data/example_1.0-1.debian.tar.gz')) + + def test_missing(self): + self.assertFalse(self.dsc.verify_file( + 'test-data/does.not.exist')) + + def test_bad(self): + fn = 'test-data/example_1.0.orig.tar.gz' + with open(fn, 'rb') as f: + data = f.read() + data = data[:-1] + chr(ord(data[-1]) ^ 8) + self.mox.StubOutWithMock(__builtin__, 'open') + open(fn, 'rb').AndReturn(StringIO.StringIO(data)) + self.mox.ReplayAll() + self.assertFalse(self.dsc.verify_file(fn)) + + def test_sha1(self): + del self.dsc['Checksums-Sha256'] + self.test_good() + self.test_bad() + + def test_md5(self): + del self.dsc['Checksums-Sha256'] + del self.dsc['Checksums-Sha1'] + self.test_good() + self.test_bad() + + +class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): + SourcePackage = DebianSourcePackage + + def setUp(self): + super(LocalSourcePackageTestCase, self).setUp() + self.workdir = tempfile.mkdtemp(prefix='udt-test') + + for funcname in ('ubuntutools.archive.Distribution', + 'ubuntutools.archive.rmadison', + 'urllib2.urlopen', + ): + mod, func = funcname.rsplit('.', 1) + setattr(self, 'o_' + func, getattr(sys.modules[mod], func)) + self.mox.StubOutWithMock(sys.modules[mod], func) + self.mox.StubOutWithMock(Logger, 'stdout') + + def tearDown(self): + super(LocalSourcePackageTestCase, self).tearDown() + shutil.rmtree(self.workdir) + + def test_local_copy(self): + urllib2.urlopen(mox.Regex('^file://.*\.dsc$') + ).WithSideEffects(self.o_urlopen) + urllib2.urlopen(mox.Regex('^file://.*\.orig\.tar\.gz$') + ).WithSideEffects(self.o_urlopen) + urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') + ).WithSideEffects(self.o_urlopen) + Logger.stdout.write(mox.IsA(basestring)).MultipleTimes() + Logger.stdout.flush().MultipleTimes() + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + dscfile='test-data/example_1.0-1.dsc', + workdir=self.workdir) + pkg.pull() + pkg.unpack() + + def test_verification(self): + shutil.copy2('test-data/example_1.0-1.dsc', self.workdir) + shutil.copy2('test-data/example_1.0.orig.tar.gz', self.workdir) + shutil.copy2('test-data/example_1.0-1.debian.tar.gz', self.workdir) + with open(os.path.join(self.workdir, 'example_1.0-1.debian.tar.gz'), + 'r+b') as f: + f.write('CORRUPTION') + urllib2.urlopen(mox.Regex('^file://.*\.dsc$') + ).WithSideEffects(self.o_urlopen) + urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') + ).WithSideEffects(self.o_urlopen) + Logger.stdout.write(mox.IsA(basestring)).MultipleTimes() + Logger.stdout.flush().MultipleTimes() + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + dscfile='test-data/example_1.0-1.dsc', + workdir=self.workdir) + pkg.pull() + pkg.unpack() + +class UbuntuLocalSourcePackageTestCase(LocalSourcePackageTestCase): + SourcePackage = UbuntuSourcePackage From 6b1ad27167233170a9725b73d9921241dc553712 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 18:52:22 +0200 Subject: [PATCH 21/43] A bit less magic (for pylint) --- ubuntutools/test/test_archive.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index cafa1c9..d1ed0d7 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -28,7 +28,7 @@ import urlparse import mox -from ubuntutools.archive import Dsc, DebianSourcePackage, UbuntuSourcePackage +import ubuntutools.archive from ubuntutools.logger import Logger from ubuntutools.test import unittest @@ -36,7 +36,7 @@ class DscVerificationTestCase(mox.MoxTestBase, unittest.TestCase): def setUp(self): super(DscVerificationTestCase, self).setUp() with open('test-data/example_1.0-1.dsc', 'rb') as f: - self.dsc = Dsc(f.read()) + self.dsc = ubuntutools.archive.Dsc(f.read()) def tearDown(self): super(DscVerificationTestCase, self).tearDown() @@ -74,19 +74,16 @@ class DscVerificationTestCase(mox.MoxTestBase, unittest.TestCase): class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): - SourcePackage = DebianSourcePackage + SourcePackage = ubuntutools.archive.DebianSourcePackage def setUp(self): super(LocalSourcePackageTestCase, self).setUp() self.workdir = tempfile.mkdtemp(prefix='udt-test') - for funcname in ('ubuntutools.archive.Distribution', - 'ubuntutools.archive.rmadison', - 'urllib2.urlopen', - ): - mod, func = funcname.rsplit('.', 1) - setattr(self, 'o_' + func, getattr(sys.modules[mod], func)) - self.mox.StubOutWithMock(sys.modules[mod], func) + self.mox.StubOutWithMock(ubuntutools.archive, 'Distribution') + self.mox.StubOutWithMock(ubuntutools.archive, 'rmadison') + self.urlopen = urllib2.urlopen + self.mox.StubOutWithMock(urllib2, 'urlopen') self.mox.StubOutWithMock(Logger, 'stdout') def tearDown(self): @@ -95,11 +92,11 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): def test_local_copy(self): urllib2.urlopen(mox.Regex('^file://.*\.dsc$') - ).WithSideEffects(self.o_urlopen) + ).WithSideEffects(self.urlopen) urllib2.urlopen(mox.Regex('^file://.*\.orig\.tar\.gz$') - ).WithSideEffects(self.o_urlopen) + ).WithSideEffects(self.urlopen) urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') - ).WithSideEffects(self.o_urlopen) + ).WithSideEffects(self.urlopen) Logger.stdout.write(mox.IsA(basestring)).MultipleTimes() Logger.stdout.flush().MultipleTimes() self.mox.ReplayAll() @@ -118,9 +115,9 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): 'r+b') as f: f.write('CORRUPTION') urllib2.urlopen(mox.Regex('^file://.*\.dsc$') - ).WithSideEffects(self.o_urlopen) + ).WithSideEffects(self.urlopen) urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') - ).WithSideEffects(self.o_urlopen) + ).WithSideEffects(self.urlopen) Logger.stdout.write(mox.IsA(basestring)).MultipleTimes() Logger.stdout.flush().MultipleTimes() self.mox.ReplayAll() @@ -132,4 +129,4 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): pkg.unpack() class UbuntuLocalSourcePackageTestCase(LocalSourcePackageTestCase): - SourcePackage = UbuntuSourcePackage + SourcePackage = ubuntutools.archive.UbuntuSourcePackage From 182cab66c470653dca258ea5ed16a4e0edf492f4 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 18:54:24 +0200 Subject: [PATCH 22/43] distribution -> suite --- ubuntutools/archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 53a6fc3..3206247 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -433,7 +433,7 @@ def rmadison(url, package): yield { 'source': pkg, 'version': ver, - 'distribution': dist, + 'suite': dist, 'component': comp, } archs.discard('source') @@ -441,7 +441,7 @@ def rmadison(url, package): yield { 'binary': pkg, 'version': ver, - 'distribution': dist, + 'suite': dist, 'component': comp, 'architectures': archs, } From c52f0965e7cfc8d9bcfd9eccc42125f354bf118e Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 19:56:31 +0200 Subject: [PATCH 23/43] Fall back to master mirrors --- ubuntutools/archive.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 3206247..a0dd0a6 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -192,6 +192,9 @@ class SourcePackage(object): yield os.path.join(os.path.dirname(self._dsc_source), name) for mirror in self.mirrors: yield self._mirror_url(mirror, name) + for mirror in self.masters: + if mirror not in self.mirrors: + yield self._mirror_url(mirror, name) yield self._lp_url(name) def pull_dsc(self): From 5ad9ee9f5bd9518833c9df381a3115f5592744af Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 21:08:01 +0200 Subject: [PATCH 24/43] SourcPackage.pull() should raise DownloadError instead of returning False --- ubuntutools/archive.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index a0dd0a6..b3822da 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -303,8 +303,7 @@ class SourcePackage(object): except urllib2.URLError, e: Logger.normal('URL Error: %s', e.reason) else: - return False - return True + raise DownloadError('File %s could not be found' % name) def unpack(self, destdir=None): "Unpack in workdir" From d9a9b4cab49f4e818582c296ff14674ae0b78877 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Fri, 31 Dec 2010 21:08:16 +0200 Subject: [PATCH 25/43] Complete test suite for ubuntutools.archive --- ubuntutools/test/test_archive.py | 169 +++++++++++++++++++++++++++++-- ubuntutools/test/test_pylint.py | 2 + 2 files changed, 160 insertions(+), 11 deletions(-) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index d1ed0d7..3412618 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -20,15 +20,14 @@ import __builtin__ import os.path import shutil import StringIO -import subprocess -import sys import tempfile import urllib2 -import urlparse +import debian.deb822 import mox import ubuntutools.archive +from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.test import unittest @@ -74,7 +73,7 @@ class DscVerificationTestCase(mox.MoxTestBase, unittest.TestCase): class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): - SourcePackage = ubuntutools.archive.DebianSourcePackage + SourcePackage = ubuntutools.archive.UbuntuSourcePackage def setUp(self): super(LocalSourcePackageTestCase, self).setUp() @@ -84,12 +83,34 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): self.mox.StubOutWithMock(ubuntutools.archive, 'rmadison') self.urlopen = urllib2.urlopen self.mox.StubOutWithMock(urllib2, 'urlopen') - self.mox.StubOutWithMock(Logger, 'stdout') + # Silence the tests a little: + self.mox.stubs.Set(Logger, 'stdout', StringIO.StringIO()) + self.mox.stubs.Set(Logger, 'stderr', StringIO.StringIO()) def tearDown(self): super(LocalSourcePackageTestCase, self).tearDown() shutil.rmtree(self.workdir) + def urlopen_proxy(self, url, destname=None): + "Grab the file from test-data" + if destname is None: + destname = os.path.basename(url) + return self.urlopen('file://' + + os.path.join(os.path.abspath('test-data'), + destname)) + + def urlopen_file(self, filename): + "Wrapper for urlopen_proxy for named files" + return lambda url: self.urlopen_proxy(url, filename) + + def urlopen_null(self, url): + "urlopen for zero length files" + return StringIO.StringIO('') + + def urlopen_404(self, url): + "urlopen for errors" + raise urllib2.HTTPError(url, 404, "Not Found", {}, None) + def test_local_copy(self): urllib2.urlopen(mox.Regex('^file://.*\.dsc$') ).WithSideEffects(self.urlopen) @@ -97,8 +118,6 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): ).WithSideEffects(self.urlopen) urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') ).WithSideEffects(self.urlopen) - Logger.stdout.write(mox.IsA(basestring)).MultipleTimes() - Logger.stdout.flush().MultipleTimes() self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -118,15 +137,143 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): ).WithSideEffects(self.urlopen) urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') ).WithSideEffects(self.urlopen) - Logger.stdout.write(mox.IsA(basestring)).MultipleTimes() - Logger.stdout.flush().MultipleTimes() self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', dscfile='test-data/example_1.0-1.dsc', workdir=self.workdir) pkg.pull() + + def test_pull(self): + dist = self.SourcePackage.distribution + mirror = UDTConfig.defaults['%s_MIRROR' % dist.upper()] + urlbase = '/pool/main/e/example/' + urllib2.urlopen('https://launchpad.net/%s/+archive/primary/' + '+files/example_1.0-1.dsc' % dist + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + workdir=self.workdir) + pkg.pull() + + def test_mirrors(self): + master = UDTConfig.defaults['UBUNTU_MIRROR'] + mirror = 'http://mirror' + lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/' + urlbase = '/pool/main/e/example/' + urllib2.urlopen(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_null) + urllib2.urlopen(master + urlbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + workdir=self.workdir, mirrors=[mirror]) + pkg.pull() + + def test_dsc_missing(self): + lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/' + urllib2.urlopen(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_404) + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + workdir=self.workdir) + self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) + + +class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): + SourcePackage = ubuntutools.archive.DebianSourcePackage + + def test_mirrors(self): + debian_master = UDTConfig.defaults['DEBIAN_MIRROR'] + debsec_master = UDTConfig.defaults['DEBSEC_MIRROR'] + debian_mirror = 'http://mirror/debian' + debsec_mirror = 'http://mirror/debsec' + lpbase = 'https://launchpad.net/debian/+archive/primary/+files/' + base = '/pool/main/e/example/' + urllib2.urlopen(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(debian_mirror + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_null) + urllib2.urlopen(debsec_mirror + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen(debian_master + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen(debsec_master + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen('http://snapshot.debian.org/mr/package/example/1.0-1/' + 'srcfiles?fileinfo=1' + ).WithSideEffects(lambda x: StringIO.StringIO( + '{"fileinfo": {"hashabc": [{"name": "example_1.0.orig.tar.gz"}]}}' + )) + urllib2.urlopen('http://snapshot.debian.org/file/hashabc' + ).WithSideEffects(self.urlopen_file( + 'example_1.0.orig.tar.gz')) + urllib2.urlopen(debian_mirror + base + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + workdir=self.workdir, mirrors=[debian_mirror, + debsec_mirror]) + pkg.pull() pkg.unpack() -class UbuntuLocalSourcePackageTestCase(LocalSourcePackageTestCase): - SourcePackage = ubuntutools.archive.UbuntuSourcePackage + def test_dsc_missing(self): + mirror = 'http://mirror' + lpbase = 'https://launchpad.net/debian/+archive/primary/+files/' + base = '/pool/main/e/example/' + urllib2.urlopen(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen(mirror + base + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(mirror + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + urllib2.urlopen(mirror + base + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + + self.mox.StubOutWithMock(debian.deb822.GpgInfo, 'from_sequence') + debian.deb822.GpgInfo.from_sequence(mox.IsA(str)).WithSideEffects( + lambda x: debian.deb822.GpgInfo.from_output( + '[GNUPG:] GOODSIG DEADBEEF Joe Developer ' + '')) + + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + workdir=self.workdir, mirrors=[mirror]) + pkg.pull() + + def test_dsc_badsig(self): + mirror = 'http://mirror' + lpbase = 'https://launchpad.net/debian/+archive/primary/+files/' + base = '/pool/main/e/example/' + urllib2.urlopen(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_404) + urllib2.urlopen(mirror + base + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + + self.mox.StubOutWithMock(debian.deb822.GpgInfo, 'from_sequence') + debian.deb822.GpgInfo.from_sequence(mox.IsA(str)).WithSideEffects( + lambda x: debian.deb822.GpgInfo.from_output( + '[GNUPG:] ERRSIG DEADBEEF')) + + self.mox.ReplayAll() + + pkg = self.SourcePackage('example', '1.0-1', 'main', + workdir=self.workdir, mirrors=[mirror]) + self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) diff --git a/ubuntutools/test/test_pylint.py b/ubuntutools/test/test_pylint.py index d6275e9..e467ac4 100644 --- a/ubuntutools/test/test_pylint.py +++ b/ubuntutools/test/test_pylint.py @@ -25,6 +25,8 @@ WHITELIST = [re.compile(': %s$' % x) for x in ( r"No name '\w+Error' in module 'launchpadlib\.errors'", # http://www.logilab.org/ticket/51250: r"Module 'hashlib' has no '(md5|sha(1|224|256|384|512))' member", + # mox: + r"Instance of 'GpgInfo' has no 'WithSideEffects' member", )] class PylintTestCase(unittest.TestCase): From 3ef8324ff711f48ebfea1fad246d0b77c7a9445d Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 1 Jan 2011 19:51:03 +0200 Subject: [PATCH 26/43] Improve error handling --- backportpackage | 4 +++- pull-debian-debdiff | 15 +++++++++++---- syncpackage | 10 +++++++++- ubuntutools/archive.py | 4 +++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/backportpackage b/backportpackage index 5c981de..e349f49 100755 --- a/backportpackage +++ b/backportpackage @@ -28,7 +28,7 @@ import tempfile from launchpadlib.launchpad import Launchpad import lsb_release -from ubuntutools.archive import UbuntuSourcePackage +from ubuntutools.archive import UbuntuSourcePackage, DownloadError from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.builder import get_builder from ubuntutools.logger import Logger @@ -298,6 +298,8 @@ def main(args): opts.update, opts.upload, opts.prompt) + except DownloadError, e: + error(str(e)) finally: if not opts.workdir: shutil.rmtree(workdir) diff --git a/pull-debian-debdiff b/pull-debian-debdiff index 6483a1f..053f5c5 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -25,7 +25,7 @@ import sys import debian.debian_support import debian.changelog -from ubuntutools.archive import DebianSourcePackage +from ubuntutools.archive import DebianSourcePackage, DownloadError from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger @@ -81,9 +81,12 @@ def main(): Logger.normal('Downloading %s %s', package, version) - #TODO: Proper snapshot and security support newpkg = DebianSourcePackage(package, version, mirrors=mirrors) - newpkg.pull() + try: + newpkg.pull() + except DownloadError, e: + Logger.error(str(e)) + sys.exit(1) newpkg.unpack() if opts.fetch_only: @@ -96,7 +99,11 @@ def main(): Logger.normal('Downloading %s %s', package, oldversion) oldpkg = DebianSourcePackage(package, oldversion, mirrors=mirrors) - oldpkg.pull() + try: + oldpkg.pull() + except DownloadError, e: + Logger.error(str(e)) + sys.exit(1) oldpkg.unpack() cmd = ['debdiff', oldpkg.dsc_name, newpkg.dsc_name] diff --git a/syncpackage b/syncpackage index 9c040d5..23bf7bf 100755 --- a/syncpackage +++ b/syncpackage @@ -269,7 +269,12 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): version = Version(version) if version is None or component is None: - debian_srcpkg = getDebianSrcPkg(package, dist) + try: + debian_srcpkg = getDebianSrcPkg(package, dist) + except (udtexceptions.PackageNotFoundException, + udtexceptions.SeriesNotFoundException), e: + Logger.error(str(e)) + sys.exit(1) if version is None: version = Version(debian_srcpkg.getVersion()) try: @@ -277,6 +282,9 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): ubuntu_version = Version(ubuntu_srcpkg.getVersion()) except udtexceptions.PackageNotFoundException: ubuntu_version = Version('~') + except udtexceptions.SeriesNotFoundException, e: + Logger.error(str(e)) + sys.exit(1) if ubuntu_version >= version: # The LP importer is maybe out of date debian_srcpkg = requestsync_mail_getDebianSrcPkg(package, dist) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index b3822da..e144557 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -389,7 +389,9 @@ class DebianSourcePackage(SourcePackage): import simplejson as json except ImportError: Logger.error("Please install python-simplejson.") - sys.exit(1) + raise DownloadError("Unable to dowload from " + "snapshot.debian.org without " + "python-simplejson") try: srcfiles = json.load(urllib2.urlopen( From cdd93a4cd841fe8032e26688ab4867cfd5a09d77 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 13 Jan 2011 22:58:03 +0200 Subject: [PATCH 27/43] backportpackage: check_call: Take first argument (command name) when reporting error. --- backportpackage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backportpackage b/backportpackage index e349f49..9e8b051 100755 --- a/backportpackage +++ b/backportpackage @@ -42,7 +42,7 @@ def check_call(cmd, *args, **kwargs): Logger.command(cmd) ret = subprocess.call(cmd, *args, **kwargs) if ret != 0: - error('%s returned %d.' % (cmd, ret)) + error('%s returned %d.' % (cmd[0], ret)) def parse(args): usage = 'Usage: %prog [options] ' From ed91b960e2d537b00f9b3bcabfc35d1341e96601 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 13:28:44 +0200 Subject: [PATCH 28/43] Move debdiff into SourcePackage --- debian/copyright | 8 +++---- pull-debian-debdiff | 16 ++----------- ubuntutools/archive.py | 52 ++++++++++++++++++++++++++++-------------- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/debian/copyright b/debian/copyright index f1c5b02..2bfa848 100644 --- a/debian/copyright +++ b/debian/copyright @@ -194,10 +194,10 @@ Files: doc/pull-debian-debdiff.1, ubuntutools/update_maintainer.py, update-maintainer, wrap-and-sort -Copyright: 2010, Benjamin Drung - 2010, Evan Broder - 2008, Siegfried-Angel Gevatter Pujals - 2010, Stefano Rivera +Copyright: 2010, Benjamin Drung + 2010, Evan Broder + 2008, Siegfried-Angel Gevatter Pujals + 2010-2011, Stefano Rivera License: ISC Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/pull-debian-debdiff b/pull-debian-debdiff index 053f5c5..5e68237 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -2,7 +2,7 @@ # pull-debian-debdiff - find and download a specific version of a Debian # package and its immediate parent to generate a debdiff. # -# Copyright (C) 2010, Stefano Rivera +# Copyright (C) 2010-2011, Stefano Rivera # Inspired by a tool of the same name by Kees Cook. # # Permission to use, copy, modify, and/or distribute this software for any @@ -105,19 +105,7 @@ def main(): Logger.error(str(e)) sys.exit(1) oldpkg.unpack() - - cmd = ['debdiff', oldpkg.dsc_name, newpkg.dsc_name] - difffn = newpkg.dsc_name[:-3] + 'debdiff' - Logger.command(cmd + ['> %s' % difffn]) - debdiff_file = open(difffn, 'w') - if subprocess.call(cmd, stdout=debdiff_file) > 2: - Logger.error('Debdiff failed.') - sys.exit(1) - debdiff_file.close() - cmd = ('diffstat', '-p0', difffn) - Logger.command(cmd) - subprocess.check_call(cmd) - print 'file://' + os.path.abspath(difffn) + print 'file://' + oldpkg.debdiff(newpkg, diffstat=True) if __name__ == '__main__': main() diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index e144557..f337240 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -1,7 +1,7 @@ # archive.py - Functions for dealing with Debian source packages, archives, # and mirrors. # -# Copyright (C) 2010, Stefano Rivera +# Copyright (C) 2010-2011, Stefano Rivera # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -27,6 +27,8 @@ Approach: 3. Verify checksums. """ +from __future__ import with_statement + import hashlib import os.path import subprocess @@ -74,13 +76,12 @@ class Dsc(debian.deb822.Dsc): if os.path.getsize(pathname) != size: return False hash_func = getattr(hashlib, alg)() - f = open(pathname, 'rb') - while True: - buf = f.read(hash_func.block_size) - if buf == '': - break - hash_func.update(buf) - f.close() + with open(pathname, 'rb') as f: + while True: + buf = f.read(hash_func.block_size) + if buf == '': + break + hash_func.update(buf) return hash_func.hexdigest() == digest return False @@ -270,16 +271,15 @@ class SourcePackage(object): except urllib2.HTTPError: return False - out = open(pathname, 'wb') - while True: - block = in_.read(10240) - if block == '': - break - out.write(block) - Logger.stdout.write('.') - Logger.stdout.flush() + with open(pathname, 'wb') as out: + while True: + block = in_.read(10240) + if block == '': + break + out.write(block) + Logger.stdout.write('.') + Logger.stdout.flush() in_.close() - out.close() Logger.stdout.write(' done\n') Logger.stdout.flush() if self.dsc and not url.endswith('.dsc'): @@ -313,6 +313,24 @@ class SourcePackage(object): Logger.command(cmd) subprocess.check_call(cmd, cwd=self.workdir) + def debdiff(self, newpkg, diffstat=False): + """Write a debdiff comparing this src pkg to a newer one. + Optionally print diffstat. + Return the debdiff filename. + """ + cmd = ['debdiff', self.dsc_name, newpkg.dsc_name] + difffn = newpkg.dsc_name[:-3] + 'debdiff' + Logger.command(cmd + ['> %s' % difffn]) + with open(difffn, 'w') as f: + if subprocess.call(cmd, stdout=f, cwd=self.workdir) > 2: + Logger.error('Debdiff failed.') + sys.exit(1) + if diffstat: + cmd = ('diffstat', '-p1', difffn) + Logger.command(cmd) + subprocess.check_call(cmd) + return os.path.abspath(difffn) + class DebianSourcePackage(SourcePackage): "Download / unpack a Debian source package" From b62377b374eb6034f35ed09902393ed21eaf8203 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 13:46:40 +0200 Subject: [PATCH 29/43] Handle missing debian-keyring more gracefully --- debian/control | 4 +++- debian/copyright | 1 + ubuntutools/archive.py | 33 ++++++++++++++++++++++----------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/debian/control b/debian/control index 67c613e..470e6c8 100644 --- a/debian/control +++ b/debian/control @@ -56,7 +56,9 @@ Recommends: bzr, python-magic, python-soappy, reportbug (>= 3.39ubuntu1) -Suggests: python-simplejson | python (>= 2.7), qemu-kvm-extras-static +Suggests: debian-keyring, + python-simplejson | python (>= 2.7), + qemu-kvm-extras-static Breaks: ${python:Breaks} Description: useful tools for Ubuntu developers This is a collection of useful tools that Ubuntu developers use to make their diff --git a/debian/copyright b/debian/copyright index 2bfa848..bfeb82c 100644 --- a/debian/copyright +++ b/debian/copyright @@ -27,6 +27,7 @@ Copyright: 2007, Albert Damen 2010, Evan Broder 2006-2007, Luke Yelavich 2009-2010, Michael Bienia + 2010-2011, Stefano Rivera 2008, Stephan Hermann 2007, Steve Kowalik License: GPL-2 diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index f337240..eff4244 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -228,18 +228,29 @@ class SourcePackage(object): assert self.version.debian_revision == version.debian_revision self.version = version - gpg_info = self.dsc.get_gpg_info() - if gpg_info.valid(): - message = 'Valid signature' - else: - message = 'Signature on %s could not be verified' % self.dsc_name - if 'GOODSIG' in gpg_info: - message = 'Good signature by %s (0x%s)' % (gpg_info['GOODSIG'][1], - gpg_info['GOODSIG'][0]) - elif 'VALIDSIG' in gpg_info: - message = 'Valid signature by 0x%s' % gpg_info['VALIDSIG'][0] + valid = False + message = None + gpg_info = None + try: + gpg_info = self.dsc.get_gpg_info() + valid = gpg_info.valid() + except IOError: + message = ('Signature on %s could not be verified, install ' + 'debian-keyring' % self.dsc_name) + if message is None: + if valid: + message = 'Valid signature' + else: + message = ('Signature on %s could not be verified' + % self.dsc_name) + if gpg_info is not None: + if 'GOODSIG' in gpg_info: + message = ('Good signature by %s (0x%s)' + % (gpg_info['GOODSIG'][1], gpg_info['GOODSIG'][0])) + elif 'VALIDSIG' in gpg_info: + message = 'Valid signature by 0x%s' % gpg_info['VALIDSIG'][0] if verify_signature: - if gpg_info.valid(): + if valid: Logger.normal(message) else: Logger.error(message) From 394d956a378733f4abd960f1ad457aad8f6d4d47 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 13:48:38 +0200 Subject: [PATCH 30/43] Catch URLError, not HTTPError when pulling sources --- ubuntutools/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index eff4244..2907f28 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -279,7 +279,7 @@ class SourcePackage(object): try: in_ = urllib2.urlopen(url) - except urllib2.HTTPError: + except urllib2.URLError: return False with open(pathname, 'wb') as out: From e48a2c92d4aea26bc7ba1797f70abbca59369f68 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 13:55:20 +0200 Subject: [PATCH 31/43] Cannot use with statement with StringIO --- ubuntutools/archive.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 2907f28..ee43262 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -76,12 +76,13 @@ class Dsc(debian.deb822.Dsc): if os.path.getsize(pathname) != size: return False hash_func = getattr(hashlib, alg)() - with open(pathname, 'rb') as f: - while True: - buf = f.read(hash_func.block_size) - if buf == '': - break - hash_func.update(buf) + f = open(pathname, 'rb') + while True: + buf = f.read(hash_func.block_size) + if buf == '': + break + hash_func.update(buf) + f.close() return hash_func.hexdigest() == digest return False From f4bbff9e9c42b58f89f2b96d5f445adc7205eeb9 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 15:50:07 +0200 Subject: [PATCH 32/43] Move source package verification into ubuntutools.archive --- syncpackage | 39 +++++++++------------------------------ ubuntutools/archive.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/syncpackage b/syncpackage index 23bf7bf..5f2e9d6 100755 --- a/syncpackage +++ b/syncpackage @@ -24,7 +24,6 @@ import debian.deb822 import debian.debian_support import optparse import os -import re import shutil import subprocess import sys @@ -122,38 +121,15 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, ubuntu_source.getComponent(), mirrors=[ubuntu_mirror]) ubu_pkg.pull_dsc() + need_orig = ubuntu_ver.upstream_version != new_ver.upstream_version except udtexceptions.PackageNotFoundException: ubuntu_ver = Version('~') ubu_pkg = None + need_orig = True + Logger.info('%s does not exist in Ubuntu.', name) Logger.debug('Source %s: current version %s, new version %s', src_pkg.source, ubuntu_ver, new_ver) - - # do we need the orig.tar.gz? - need_orig = True - fakesync_files = [] - if ubuntu_ver.upstream_version == new_ver.upstream_version: - # We need to check if all .orig*.tar.* tarballs exist in Ubuntu - need_orig = False - deb_files = dict((e['name'], e['md5sum']) for e in src_pkg.dsc['Files'] - if not e['name'].endswith('.dsc')) - ubu_files = dict((e['name'], e['md5sum']) for e in ubu_pkg.dsc['Files'] - if not e['name'].endswith('.dsc')) - for name in deb_files.iterkeys(): - if not re.match(r'.*\.orig(-[^.]+)?\.tar\.[^.]+$', name): - continue - if name in ubu_files: - if deb_files[name] != ubu_files[name]: - Logger.warn('The checksums of the file %s mismatch. ' - 'A fake sync is required.', name) - fakesync_files.append(name) - Logger.debug('Ubuntu version: %s', deb_files[name]) - Logger.debug('Debian version: %s', ubu_files[name]) - else: - Logger.info('%s does not exist in Ubuntu.', - name) - need_orig = True - Logger.debug('Needs source tarball: %s', str(need_orig)) cur_ver = ubuntu_ver.get_related_debian_version() @@ -165,8 +141,11 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, src_pkg.pull() src_pkg.unpack() - # Do a fake sync if required - if fakesync_files: + fakesync = not ubu_pkg.verify_orig() + + if fakesync: + Logger.warn('The checksums of the Debian and Ubuntu packages mismatch. ' + 'A fake sync is required.') # Download Ubuntu files (override Debian source tarballs) ubu_pkg.pull() @@ -180,7 +159,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, line = open("debian/changelog").readline() debian_dist = line.split(" ")[2].strip(";") - if len(fakesync_files) == 0: + if not fakesync: # create the changes file changes_filename = "%s_%s_source.changes" % \ (src_pkg.source, new_ver.strip_epoch()) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index ee43262..5b753c9 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -34,6 +34,7 @@ import os.path import subprocess import urllib2 import urlparse +import re import sys import debian.deb822 @@ -317,6 +318,24 @@ class SourcePackage(object): else: raise DownloadError('File %s could not be found' % name) + def verify(self): + """Verify that the source package in workdir matches the dsc. + Return boolean + """ + return all(self.dsc.verify_file(os.path.join(self.workdir, + entry['name'])) + for entry in self.dsc['Files']) + + def verify_orig(self): + """Verify that the .orig files in workdir match the dsc. + Return boolean + """ + orig_re = re.compile(r'.*\.orig(-[^.]+)?\.tar\.[^.]+$') + return all(self.dsc.verify_file(os.path.join(self.workdir, + entry['name'])) + for entry in self.dsc['Files'] + if orig_re.match(entry['name'])) + def unpack(self, destdir=None): "Unpack in workdir" cmd = ['dpkg-source', '-x', self.dsc_name] From baf7a803ae678510b4a91d2badea1ccb14329510 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 15:59:59 +0200 Subject: [PATCH 33/43] Read source and version from dsc, not dsc filename --- ubuntutools/archive.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 5b753c9..1f005ea 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -120,11 +120,10 @@ class SourcePackage(object): self.masters = [UDTConfig.defaults['%s_MIRROR' % self.distribution.upper()]] if dscfile is not None: - d_source, d_version = os.path.basename(dscfile)[:-4].split('_') if self.source is None: - self.source = d_source + self.source = 'unknown' if self.version is None: - self.version = d_version + self.version = 'unknown' self.version = debian.debian_support.Version(self.version) @@ -223,12 +222,14 @@ class SourcePackage(object): "Check that the dsc matches what we are expecting" assert os.path.exists(self.dsc_pathname) self._dsc_fetched = True + old_pathname = self.dsc_pathname - assert self.source == self.dsc['Source'] - version = debian.debian_support.Version(self.dsc['Version']) - assert self.version.upstream_version == version.upstream_version - assert self.version.debian_revision == version.debian_revision - self.version = version + self.source = self.dsc['Source'] + self.version = debian.debian_support.Version(self.dsc['Version']) + + # If source or version was previously unknown + if self.dsc_pathname != old_pathname: + os.rename(old_pathname, self.dsc_pathname) valid = False message = None From 495d14b69d5960c451da3ff1e6822b620cfe3275 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 16:05:15 +0200 Subject: [PATCH 34/43] Help pylint --- ubuntutools/archive.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 1f005ea..7f52511 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -102,7 +102,6 @@ class SourcePackage(object): or dscfile is not None) self.source = package - self.version = version self._lp = lp self.workdir = workdir @@ -122,10 +121,10 @@ class SourcePackage(object): if dscfile is not None: if self.source is None: self.source = 'unknown' - if self.version is None: - self.version = 'unknown' + if version is None: + version = 'unknown' - self.version = debian.debian_support.Version(self.version) + self.version = debian.debian_support.Version(version) @property def lp_spph(self): From b11d08f6b03b3317eeefcca2519bb97b05d730b9 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 17:56:48 +0200 Subject: [PATCH 35/43] Generate example package --- test-data/example_1.0-1.debian.tar.gz | Bin 1170 -> 0 bytes test-data/example_1.0-1.dsc | 17 ---- test-data/example_1.0.orig.tar.gz | Bin 170 -> 0 bytes ubuntutools/test/example_package.py | 133 ++++++++++++++++++++++++++ ubuntutools/test/test_archive.py | 38 +++++--- 5 files changed, 156 insertions(+), 32 deletions(-) delete mode 100644 test-data/example_1.0-1.debian.tar.gz delete mode 100644 test-data/example_1.0-1.dsc delete mode 100644 test-data/example_1.0.orig.tar.gz create mode 100644 ubuntutools/test/example_package.py diff --git a/test-data/example_1.0-1.debian.tar.gz b/test-data/example_1.0-1.debian.tar.gz deleted file mode 100644 index 0dfd3a7e1658ee5c3f79f35e259b6ab629b90418..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1170 zcmV;D1a12tiwFP!000021MOIClbbdW_SfoHY}2%pq&5cInAYtj;cR@z3|G&il zfj&O+e~dlFe|bR$_= zOkmH%hAcTNIg>Z)f5{WFNLV59OZnuhm`~2JvPgR~TG1jE6Tsb;(Y&G@Uc5d7OVc$| z|NAA0WKyIFPetz?<)Se^)PB|sLYB$qW3v0iKbvndCeg{7<7krSeqUS_ z+F?s*CExY0vENT=)xV}|)e7}vYWwuMcvDtqTb9!wIK4VE%mF!O5~WF?sywNSO;#nT zR#4eLB!yl|uX?g9b3w=)g^{(d$n?1PCQWV7{kqKJu?;N*+H<2RfhFb0bd@?lmWrTV zP+PhS`!Z!)Y%#&~k>4p3lCyO!k+~TYAj+-0PWXZRNpVAt%!g|xGr&RBmbHW}rGNq% zuQ*sSh-J^ocy6gMF&R6?M=v=6Lw(&EK-N09~(P|vp$_3xK@Z1Z>pE@W=AZVt?L5MR~(XB)m?9h#r z&0rNP@u1O&N_^*_QmuKj4CqFqXX%8v01La(JO6hK9_guZoBw%@WPG&uzeDRj|BuJ2 z{u`KW{dW?4_ieuxyuW5ef1iA$(AyI7{e*mzX7Kziv -Standards-Version: 3.9.1 -Build-Depends: debhelper (>= 7.0.50~) -Checksums-Sha1: - 13c309f70155ab5be50553ce72f3344c48851013 170 example_1.0.orig.tar.gz - ae9dcd225000d24128b8eccbdc472abcfa9b60ee 1170 example_1.0-1.debian.tar.gz -Checksums-Sha256: - 4908cb2ce47c9835fb74476ab5dba0298da7a56a389328b845d07575f5ea6276 170 example_1.0.orig.tar.gz - ca366dc1ae0efa1df6537f6882e87134a0409e1589ca18daf56d065beb9cfce3 1170 example_1.0-1.debian.tar.gz -Files: - 8054fdf3653160e13d9539ca36c2e311 170 example_1.0.orig.tar.gz - 8538e8b6cb8162d5c6886b18c132a1c8 1170 example_1.0-1.debian.tar.gz diff --git a/test-data/example_1.0.orig.tar.gz b/test-data/example_1.0.orig.tar.gz deleted file mode 100644 index 1d3482e74e5d142569d08e073d34700829f1d6e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 170 zcmb2|=3odpBJ0b*{Pvsv+^E0^~y8u$LD{?^G?)ueSLS^ TkpUUhykT4X#;A`$gMk46H^NWl diff --git a/ubuntutools/test/example_package.py b/ubuntutools/test/example_package.py new file mode 100644 index 0000000..fffdb71 --- /dev/null +++ b/ubuntutools/test/example_package.py @@ -0,0 +1,133 @@ +# example_package.py - Creates an example package +# +# Copyright (C) 2010-2011, Stefano Rivera +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +import os.path +import shutil +import subprocess +import tempfile + +import debian.debian_support + +_base_pkg = { + 'content': 'Boring file from upstream', + 'debian/control': +"""Source: example +Section: misc +Priority: extra +Maintainer: Example +Build-Depends: debhelper (>= 7.0.50~) +Standards-Version: 3.9.1 + +Package: example +Architecture: all +Depends: ${shlibs:Depends}, ${misc:Depends} +Description: Example package for testing purposes + An example package used by the test suite. Useless. +""", + 'debian/copyright': +"""Format: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=152 +Source: https://launchpad.net/ubuntu-dev-tools + +Files: * +Copyright: 2010-2011, Stefano Rivera +License: ISC + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + . + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. +""", + 'debian/compat': '7', + 'debian/rules': +"""#!/usr/bin/make -f +%: +\tdh $@ +""", + 'debian/source/format': '3.0 (quilt)', + 'debian/source/local-options': 'abort-on-upstream-changes', +} + +class ExamplePackage(object): + def __init__(self, source='example', version='1.0-1', workdir=None, + files=None): + self.source = source + self.version = debian.debian_support.Version(version) + + self.pkg = dict(_base_pkg) + if files is not None: + self.pkg.update(files) + + self.workdir = workdir or tempfile.mkdtemp(prefix='examplepkg') + self.srcdir = os.path.join(self.workdir, '%s-%s' % (source, + self.version.upstream_version)) + + def create_orig(self): + "Create .orig.tar.gz" + self._write_files(filter_=lambda fn: not fn.startswith('debian/')) + orig = '%s_%s.orig.tar.gz' % (self.source, + self.version.upstream_version) + subprocess.check_call(('tar', '-czf', orig, + os.path.basename(self.srcdir)), + cwd=self.workdir) + + def changelog_entry(self, version=None, create=False): + "Add a changelog entry" + cmd = ['dch', '--noconf', '--package', self.source] + if create: + cmd.append('--create') + cmd += ['--newversion', version or self.version.full_version] + cmd.append('') + env = dict(os.environ) + env['DEBFULLNAME'] = 'Example' + env['DEBEMAIL'] = 'example@example.net' + subprocess.check_call(cmd, env=env, cwd=self.srcdir) + + def create(self): + "Build source package" + self._write_files() + self.changelog_entry(create=True) + subprocess.check_call(('dpkg-buildpackage', '-rfakeroot', '-S', + '-uc', '-us'), + cwd=self.srcdir) + + def cleanup(self): + "Remove workdir" + shutil.rmtree(self.workdir) + + def pathname(self, fn): + "Return path to file in workdir" + return os.path.join(self.workdir, fn) + + def _write_files(self, filter_=None): + "Write files from self.pkg into src pkg dir, if filter_(fn)" + if filter_ is None: + filter_ = lambda x: True + + for fn, content in self.pkg.iteritems(): + if not filter_(fn): + continue + pathname = os.path.join(self.srcdir, fn) + dirname = os.path.dirname(pathname) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(pathname, 'wb') as f: + f.write(content) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index 3412618..e5a07d9 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -30,28 +30,37 @@ import ubuntutools.archive from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.test import unittest +from ubuntutools.test.example_package import ExamplePackage + +ex_pkg = ExamplePackage() +def setUpModule(): + ex_pkg.create_orig() + ex_pkg.create() + +def tearDownModule(): + ex_pkg.cleanup() class DscVerificationTestCase(mox.MoxTestBase, unittest.TestCase): def setUp(self): super(DscVerificationTestCase, self).setUp() - with open('test-data/example_1.0-1.dsc', 'rb') as f: + with open(ex_pkg.pathname('example_1.0-1.dsc'), 'rb') as f: self.dsc = ubuntutools.archive.Dsc(f.read()) def tearDown(self): super(DscVerificationTestCase, self).tearDown() def test_good(self): - self.assertTrue(self.dsc.verify_file( - 'test-data/example_1.0.orig.tar.gz')) - self.assertTrue(self.dsc.verify_file( - 'test-data/example_1.0-1.debian.tar.gz')) + self.assertTrue(self.dsc.verify_file(ex_pkg.pathname( + 'example_1.0.orig.tar.gz'))) + self.assertTrue(self.dsc.verify_file(ex_pkg.pathname( + 'example_1.0-1.debian.tar.gz'))) def test_missing(self): - self.assertFalse(self.dsc.verify_file( - 'test-data/does.not.exist')) + self.assertFalse(self.dsc.verify_file(ex_pkg.pathname( + 'does.not.exist'))) def test_bad(self): - fn = 'test-data/example_1.0.orig.tar.gz' + fn = ex_pkg.pathname('example_1.0.orig.tar.gz') with open(fn, 'rb') as f: data = f.read() data = data[:-1] + chr(ord(data[-1]) ^ 8) @@ -96,8 +105,7 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): if destname is None: destname = os.path.basename(url) return self.urlopen('file://' - + os.path.join(os.path.abspath('test-data'), - destname)) + + os.path.abspath(ex_pkg.pathname(destname))) def urlopen_file(self, filename): "Wrapper for urlopen_proxy for named files" @@ -121,15 +129,15 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', - dscfile='test-data/example_1.0-1.dsc', + dscfile=ex_pkg.pathname('example_1.0-1.dsc'), workdir=self.workdir) pkg.pull() pkg.unpack() def test_verification(self): - shutil.copy2('test-data/example_1.0-1.dsc', self.workdir) - shutil.copy2('test-data/example_1.0.orig.tar.gz', self.workdir) - shutil.copy2('test-data/example_1.0-1.debian.tar.gz', self.workdir) + for fn in ('example_1.0-1.dsc', 'example_1.0.orig.tar.gz', + 'example_1.0-1.debian.tar.gz'): + shutil.copy2(ex_pkg.pathname(fn), self.workdir) with open(os.path.join(self.workdir, 'example_1.0-1.debian.tar.gz'), 'r+b') as f: f.write('CORRUPTION') @@ -140,7 +148,7 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', - dscfile='test-data/example_1.0-1.dsc', + dscfile=ex_pkg.pathname('example_1.0-1.dsc'), workdir=self.workdir) pkg.pull() From bb718951ed27450148f84d31ca94a4272741d135 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 19:47:13 +0200 Subject: [PATCH 36/43] Store blank example package extracted --- debian/clean | 1 + debian/copyright | 1 + test-data/blank-example/content | 1 + test-data/blank-example/debian/compat | 1 + test-data/blank-example/debian/control | 12 +++ test-data/blank-example/debian/copyright | 17 ++++ test-data/blank-example/debian/rules | 4 + test-data/blank-example/debian/source/format | 1 + .../blank-example/debian/source/local-options | 1 + ubuntutools/test/example_package.py | 94 +++---------------- ubuntutools/test/test_archive.py | 40 ++++---- 11 files changed, 72 insertions(+), 101 deletions(-) create mode 100644 test-data/blank-example/content create mode 100644 test-data/blank-example/debian/compat create mode 100644 test-data/blank-example/debian/control create mode 100644 test-data/blank-example/debian/copyright create mode 100755 test-data/blank-example/debian/rules create mode 100644 test-data/blank-example/debian/source/format create mode 100644 test-data/blank-example/debian/source/local-options diff --git a/debian/clean b/debian/clean index 45149aa..a0ad765 100644 --- a/debian/clean +++ b/debian/clean @@ -1 +1,2 @@ *.egg-info/* +test-data/example_* diff --git a/debian/copyright b/debian/copyright index bfeb82c..74a5ac4 100644 --- a/debian/copyright +++ b/debian/copyright @@ -184,6 +184,7 @@ Files: doc/pull-debian-debdiff.1, pull-debian-debdiff, sponsor-patch, suspicious-source, + test-data/*, ubuntutools/archive.py, ubuntutools/builder.py, ubuntutools/config.py, diff --git a/test-data/blank-example/content b/test-data/blank-example/content new file mode 100644 index 0000000..0459513 --- /dev/null +++ b/test-data/blank-example/content @@ -0,0 +1 @@ +upstream diff --git a/test-data/blank-example/debian/compat b/test-data/blank-example/debian/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/test-data/blank-example/debian/compat @@ -0,0 +1 @@ +7 diff --git a/test-data/blank-example/debian/control b/test-data/blank-example/debian/control new file mode 100644 index 0000000..192c897 --- /dev/null +++ b/test-data/blank-example/debian/control @@ -0,0 +1,12 @@ +Source: example +Section: misc +Priority: extra +Maintainer: Ubuntu Developers +Build-Depends: debhelper (>= 7.0.50~) +Standards-Version: 3.9.1 + +Package: example +Architecture: all +Depends: ${shlibs:Depends}, ${misc:Depends} +Description: Example package for testing purposes + An example package used by the test suite. Useless. diff --git a/test-data/blank-example/debian/copyright b/test-data/blank-example/debian/copyright new file mode 100644 index 0000000..c19a068 --- /dev/null +++ b/test-data/blank-example/debian/copyright @@ -0,0 +1,17 @@ +Format: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=152 +Source: https://launchpad.net/ubuntu-dev-tools + +Files: * +Copyright: 2010-2011, Stefano Rivera +License: ISC + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + . + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. diff --git a/test-data/blank-example/debian/rules b/test-data/blank-example/debian/rules new file mode 100755 index 0000000..4067ba0 --- /dev/null +++ b/test-data/blank-example/debian/rules @@ -0,0 +1,4 @@ +#!/usr/bin/make -f + +%: + dh $@ diff --git a/test-data/blank-example/debian/source/format b/test-data/blank-example/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/test-data/blank-example/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/test-data/blank-example/debian/source/local-options b/test-data/blank-example/debian/source/local-options new file mode 100644 index 0000000..2ee6f0f --- /dev/null +++ b/test-data/blank-example/debian/source/local-options @@ -0,0 +1 @@ +abort-on-upstream-changes diff --git a/ubuntutools/test/example_package.py b/ubuntutools/test/example_package.py index fffdb71..53c236c 100644 --- a/ubuntutools/test/example_package.py +++ b/ubuntutools/test/example_package.py @@ -17,81 +17,31 @@ import os.path import shutil import subprocess -import tempfile import debian.debian_support -_base_pkg = { - 'content': 'Boring file from upstream', - 'debian/control': -"""Source: example -Section: misc -Priority: extra -Maintainer: Example -Build-Depends: debhelper (>= 7.0.50~) -Standards-Version: 3.9.1 - -Package: example -Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends} -Description: Example package for testing purposes - An example package used by the test suite. Useless. -""", - 'debian/copyright': -"""Format: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=152 -Source: https://launchpad.net/ubuntu-dev-tools - -Files: * -Copyright: 2010-2011, Stefano Rivera -License: ISC - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - . - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR - OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THIS SOFTWARE. -""", - 'debian/compat': '7', - 'debian/rules': -"""#!/usr/bin/make -f -%: -\tdh $@ -""", - 'debian/source/format': '3.0 (quilt)', - 'debian/source/local-options': 'abort-on-upstream-changes', -} - class ExamplePackage(object): - def __init__(self, source='example', version='1.0-1', workdir=None, - files=None): + def __init__(self, source='example', version='1.0-1'): self.source = source self.version = debian.debian_support.Version(version) - - self.pkg = dict(_base_pkg) - if files is not None: - self.pkg.update(files) - - self.workdir = workdir or tempfile.mkdtemp(prefix='examplepkg') - self.srcdir = os.path.join(self.workdir, '%s-%s' % (source, - self.version.upstream_version)) + self.srcdir = os.path.join('test-data', '%s-%s' % (source, + self.version.upstream_version)) + if os.path.exists(self.srcdir): + shutil.rmtree(self.srcdir) + shutil.copytree('test-data/blank-example', self.srcdir) def create_orig(self): "Create .orig.tar.gz" - self._write_files(filter_=lambda fn: not fn.startswith('debian/')) orig = '%s_%s.orig.tar.gz' % (self.source, self.version.upstream_version) subprocess.check_call(('tar', '-czf', orig, - os.path.basename(self.srcdir)), - cwd=self.workdir) + os.path.basename(self.srcdir), + '--exclude', 'debian'), + cwd='test-data') def changelog_entry(self, version=None, create=False): "Add a changelog entry" - cmd = ['dch', '--noconf', '--package', self.source] + cmd = ['dch', '--noconf', '--preserve', '--package', self.source] if create: cmd.append('--create') cmd += ['--newversion', version or self.version.full_version] @@ -103,31 +53,11 @@ class ExamplePackage(object): def create(self): "Build source package" - self._write_files() self.changelog_entry(create=True) subprocess.check_call(('dpkg-buildpackage', '-rfakeroot', '-S', '-uc', '-us'), cwd=self.srcdir) def cleanup(self): - "Remove workdir" - shutil.rmtree(self.workdir) - - def pathname(self, fn): - "Return path to file in workdir" - return os.path.join(self.workdir, fn) - - def _write_files(self, filter_=None): - "Write files from self.pkg into src pkg dir, if filter_(fn)" - if filter_ is None: - filter_ = lambda x: True - - for fn, content in self.pkg.iteritems(): - if not filter_(fn): - continue - pathname = os.path.join(self.srcdir, fn) - dirname = os.path.dirname(pathname) - if not os.path.exists(dirname): - os.makedirs(dirname) - with open(pathname, 'wb') as f: - f.write(content) + "Remove srcdir" + shutil.rmtree(self.srcdir) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index e5a07d9..af12d51 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -30,37 +30,38 @@ import ubuntutools.archive from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.test import unittest + from ubuntutools.test.example_package import ExamplePackage -ex_pkg = ExamplePackage() def setUpModule(): - ex_pkg.create_orig() - ex_pkg.create() + if not os.path.exists('test-data/example-0.1-1.dsc'): + ex_pkg = ExamplePackage() + ex_pkg.create_orig() + ex_pkg.create() + ex_pkg.cleanup() -def tearDownModule(): - ex_pkg.cleanup() class DscVerificationTestCase(mox.MoxTestBase, unittest.TestCase): def setUp(self): super(DscVerificationTestCase, self).setUp() - with open(ex_pkg.pathname('example_1.0-1.dsc'), 'rb') as f: + with open('test-data/example_1.0-1.dsc', 'rb') as f: self.dsc = ubuntutools.archive.Dsc(f.read()) def tearDown(self): super(DscVerificationTestCase, self).tearDown() def test_good(self): - self.assertTrue(self.dsc.verify_file(ex_pkg.pathname( - 'example_1.0.orig.tar.gz'))) - self.assertTrue(self.dsc.verify_file(ex_pkg.pathname( - 'example_1.0-1.debian.tar.gz'))) + self.assertTrue(self.dsc.verify_file( + 'test-data/example_1.0.orig.tar.gz')) + self.assertTrue(self.dsc.verify_file( + 'test-data/example_1.0-1.debian.tar.gz')) def test_missing(self): - self.assertFalse(self.dsc.verify_file(ex_pkg.pathname( - 'does.not.exist'))) + self.assertFalse(self.dsc.verify_file( + 'test-data/does.not.exist')) def test_bad(self): - fn = ex_pkg.pathname('example_1.0.orig.tar.gz') + fn = 'test-data/example_1.0.orig.tar.gz' with open(fn, 'rb') as f: data = f.read() data = data[:-1] + chr(ord(data[-1]) ^ 8) @@ -105,7 +106,8 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): if destname is None: destname = os.path.basename(url) return self.urlopen('file://' - + os.path.abspath(ex_pkg.pathname(destname))) + + os.path.join(os.path.abspath('test-data'), + destname)) def urlopen_file(self, filename): "Wrapper for urlopen_proxy for named files" @@ -129,15 +131,15 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', - dscfile=ex_pkg.pathname('example_1.0-1.dsc'), + dscfile='test-data/example_1.0-1.dsc', workdir=self.workdir) pkg.pull() pkg.unpack() def test_verification(self): - for fn in ('example_1.0-1.dsc', 'example_1.0.orig.tar.gz', - 'example_1.0-1.debian.tar.gz'): - shutil.copy2(ex_pkg.pathname(fn), self.workdir) + shutil.copy2('test-data/example_1.0-1.dsc', self.workdir) + shutil.copy2('test-data/example_1.0.orig.tar.gz', self.workdir) + shutil.copy2('test-data/example_1.0-1.debian.tar.gz', self.workdir) with open(os.path.join(self.workdir, 'example_1.0-1.debian.tar.gz'), 'r+b') as f: f.write('CORRUPTION') @@ -148,7 +150,7 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', - dscfile=ex_pkg.pathname('example_1.0-1.dsc'), + dscfile='test-data/example_1.0-1.dsc', workdir=self.workdir) pkg.pull() From 1bc68bd2cfd0a125c2ce57c28ef4697b2d5282cf Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 20:14:34 +0200 Subject: [PATCH 37/43] Tidy example package --- test-data/blank-example/debian/control | 2 +- test-data/blank-example/debian/rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/blank-example/debian/control b/test-data/blank-example/debian/control index 192c897..a101fad 100644 --- a/test-data/blank-example/debian/control +++ b/test-data/blank-example/debian/control @@ -7,6 +7,6 @@ Standards-Version: 3.9.1 Package: example Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends} +Depends: ${misc:Depends}, ${shlibs:Depends} Description: Example package for testing purposes An example package used by the test suite. Useless. diff --git a/test-data/blank-example/debian/rules b/test-data/blank-example/debian/rules index 4067ba0..2d33f6a 100755 --- a/test-data/blank-example/debian/rules +++ b/test-data/blank-example/debian/rules @@ -1,4 +1,4 @@ #!/usr/bin/make -f %: - dh $@ + dh $@ From b1b1c9a91a8fb8cf6e95f68a5ccdff0ac0600bde Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 20:14:41 +0200 Subject: [PATCH 38/43] Please pylint --- ubuntutools/archive.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 7f52511..fc620f4 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -401,10 +401,10 @@ class DebianSourcePackage(SourcePackage): def _source_urls(self, name): "Generator of sources for name" - it = super(DebianSourcePackage, self)._source_urls(name) + wrapped_iterator = super(DebianSourcePackage, self)._source_urls(name) while True: try: - yield it.next() + yield wrapped_iterator.next() except StopIteration: break if self.snapshot_list: @@ -471,11 +471,11 @@ class UbuntuSourcePackage(SourcePackage): def rmadison(url, package): "Call rmadison and parse the result" - p = subprocess.Popen(('rmadison', '-u', url, package), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True) - output = p.communicate()[0] - assert p.wait() == 0 + process = subprocess.Popen(('rmadison', '-u', url, package), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True) + output = process.communicate()[0] + assert process.wait() == 0 for line in output.strip().splitlines(): pkg, ver, dist, archs = [x.strip() for x in line.split('|')] comp = 'main' From 038cd428d3377dbc0c4079ddc8786d94ea506fa8 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sat, 15 Jan 2011 20:54:45 +0200 Subject: [PATCH 39/43] Use ubuntutools.archive's rmadison code in requestsync.mail --- ubuntutools/archive.py | 41 ++++++++++++++++----- ubuntutools/requestsync/mail.py | 65 ++++----------------------------- 2 files changed, 38 insertions(+), 68 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index fc620f4..3c24b12 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -44,8 +44,6 @@ from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.lp.lpapicache import (Launchpad, Distribution, SourcePackagePublishingHistory) -from ubuntutools.requestsync.mail import (SourcePackagePublishingHistory - as rmadison_SPPH) class DownloadError(Exception): "Unable to pull a source package" @@ -390,13 +388,12 @@ class DebianSourcePackage(SourcePackage): continue comp = record['component'] if record['version'] == self.version.full_version: - self._spph = rmadison_SPPH(record['source'], - record['version'], comp) + self._spph = FakeSPPH(record['source'], record['version'], + comp) return self._spph Logger.normal('Guessing component from most recent upload') - self._spph = rmadison_SPPH(self.source, self.version.full_version, - comp) + self._spph = FakeSPPH(self.source, self.version.full_version, comp) return self._spph def _source_urls(self, name): @@ -469,11 +466,35 @@ class UbuntuSourcePackage(SourcePackage): distribution = 'ubuntu' -def rmadison(url, package): +class FakeSPPH(object): + """Provide the same interface as + ubuntutools.lpapicache.SourcePackagePublishingHistory + """ + def __init__(self, name, version, component): + self.name = name + self.version = version + self.component = component + + def getPackageName(self): + return self.name + + def getVersion(self): + return self.version + + def getComponent(self): + return self.component + + +def rmadison(url, package, suite=None, arch=None): "Call rmadison and parse the result" - process = subprocess.Popen(('rmadison', '-u', url, package), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True) + cmd = ['rmadison', '-u', url] + if suite: + cmd += ['-s', suite] + if arch: + cmd += ['-a', arch] + cmd.append(package) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) output = process.communicate()[0] assert process.wait() == 0 for line in output.strip().splitlines(): diff --git a/ubuntutools/requestsync/mail.py b/ubuntutools/requestsync/mail.py index 157d6e3..21c8669 100644 --- a/ubuntutools/requestsync/mail.py +++ b/ubuntutools/requestsync/mail.py @@ -2,7 +2,8 @@ # # mail.py - methods used by requestsync when used in "mail" mode # -# Copyright © 2009 Michael Bienia +# Copyright © 2009 Michael Bienia , +# 2011 Stefano Rivera # # This module may contain code written by other authors/contributors to # the main requestsync script. See there for their names. @@ -25,6 +26,7 @@ import subprocess import smtplib import socket from debian.changelog import Version +from ubuntutools.archive import rmadison, FakeSPPH from ubuntutools.requestsync.common import raw_input_exit_on_ctrlc from ubuntutools.lp.udtexceptions import PackageNotFoundException @@ -36,68 +38,15 @@ __all__ = [ 'mailBug', ] -class SourcePackagePublishingHistory(object): - ''' - Simulate a SourcePackagePublishingHistory class from the LP API caching - module. - ''' - def __init__(self, name, version, component): - self.name = name - self.version = version - self.component = component - - def getPackageName(self): - return self.name - - def getVersion(self): - return self.version - - def getComponent(self): - return self.component - -def rmadison(distro, package, release): - # Map 'sid' and 'squeeze' to their releasenames else rmadison gets a python - # traceback back from the remote script - releasenames = { - 'sid': 'unstable', - 'squeeze': 'testing', # Needs updating after each Debian release - } - release = releasenames.get(release, release) - - rmadison_cmd = subprocess.Popen(['rmadison', '-u', distro, '-a', 'source', - '-s', release, package], - stdout=subprocess.PIPE) - - rmadison_out = rmadison_cmd.communicate()[0] - assert (rmadison_cmd.returncode == 0) - - # Return the most recent source line - lines = rmadison_out.splitlines() - if not lines: - # no output - return None - lines = [map(lambda x: x.strip(), line.split('|')) for line in lines] - lines = [line for line in lines if line[3].find('source') != -1] - if lines: - return max(lines, key = lambda x: Version(x[1])) - else: - # no source line - return None - def getSrcPkg(distro, name, release): - out = rmadison(distro, name, release) - if not out: + lines = list(rmadison(distro, name, suite=release, arch='source')) + if not lines: raise PackageNotFoundException("'%s' doesn't appear to exist " "in %s '%s'" % (name, distro.capitalize(), release)) + pkg = max(lines, key=lambda x: Version(x['version'])) - version = out[1] - component = 'main' - raw_comp = out[2].split('/') - if len(raw_comp) == 2: - component = raw_comp[1] - - return SourcePackagePublishingHistory(name, version, component) + return FakeSPPH(pkg['source'], pkg['version'], pkg['component']) def getDebianSrcPkg(name, release): return getSrcPkg('debian', name, release) From 19221fd2bce825609b5987c1516befee3338a2c6 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 20 Jan 2011 01:09:07 +0200 Subject: [PATCH 40/43] Drop unused imports --- pull-debian-debdiff | 2 -- 1 file changed, 2 deletions(-) diff --git a/pull-debian-debdiff b/pull-debian-debdiff index 5e68237..da2b86e 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -18,8 +18,6 @@ # PERFORMANCE OF THIS SOFTWARE. import optparse -import os.path -import subprocess import sys import debian.debian_support From d68b83af1d7cac02c50747c7a4a3fc2d8273d3a5 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 20 Jan 2011 01:10:06 +0200 Subject: [PATCH 41/43] Typo in comment --- ubuntutools/test/pylint.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntutools/test/pylint.conf b/ubuntutools/test/pylint.conf index e93c872..984e234 100644 --- a/ubuntutools/test/pylint.conf +++ b/ubuntutools/test/pylint.conf @@ -2,7 +2,7 @@ # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). -# lpapicache clasess, urlparse +# lpapicache classes, urlparse ignored-classes=Launchpad,BaseWrapper,PersonTeam,Distribution,Consumer,Credentials,ParseResult [FORMAT] From a5c5215959d296878da1dba2fef0f6db7c1864b5 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 20 Jan 2011 01:16:11 +0200 Subject: [PATCH 42/43] Replace check_call with error and quit --- ubuntutools/archive.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 3c24b12..e4cc609 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -340,7 +340,9 @@ class SourcePackage(object): if destdir: cmd.append(destdir) Logger.command(cmd) - subprocess.check_call(cmd, cwd=self.workdir) + if subprocess.call(cmd, cwd=self.workdir): + Logger.error('Source unpack failed.') + sys.exit(1) def debdiff(self, newpkg, diffstat=False): """Write a debdiff comparing this src pkg to a newer one. @@ -357,7 +359,9 @@ class SourcePackage(object): if diffstat: cmd = ('diffstat', '-p1', difffn) Logger.command(cmd) - subprocess.check_call(cmd) + if subprocess.call(cmd): + Logger.error('diffstat failed.') + sys.exit(1) return os.path.abspath(difffn) From 4a97dee930d645ad3d8b3b3056b30a2a8419ad47 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 20 Jan 2011 01:28:58 +0200 Subject: [PATCH 43/43] Break source_is_workdir out of long if statement --- ubuntutools/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index e4cc609..3388d31 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -204,9 +204,9 @@ class SourcePackage(object): self._dsc_source = 'file://' + os.path.abspath(self._dsc_source) parsed = urlparse.urlparse(self._dsc_source) - if (parsed.scheme != 'file' - or os.path.realpath(os.path.dirname(parsed.path)) - != os.path.realpath(self.workdir)): + source_is_workdir = (os.path.realpath(os.path.dirname(parsed.path)) + == os.path.realpath(self.workdir)) + if not (parsed.scheme == 'file' and source_is_workdir): if not self._download_file(self._dsc_source, self.dsc_name): raise DownloadError('dsc not found') else: