diff --git a/backportpackage b/backportpackage index 716883c..39a4792 100755 --- a/backportpackage +++ b/backportpackage @@ -24,7 +24,6 @@ import shutil import subprocess import sys import tempfile -import urllib from debian.deb822 import Dsc from launchpadlib.launchpad import Launchpad @@ -33,8 +32,8 @@ import lsb_release from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.builder import get_builder from ubuntutools.logger import Logger +from ubuntutools.mirrors import dsc_name, pull_source_pkg from ubuntutools.question import YesNoQuestion -from ubuntutools.misc import dsc_url def error(msg): Logger.error(msg) @@ -179,7 +178,17 @@ def find_version_package(launchpad, package, version): error('Version %s of package %s was never published in Ubuntu.' % (version, package)) -def dscurls_from_package(launchpad, mirror, package, version, source_release): +def fetch_package(launchpad, mirror, workdir, package, version, source_release): + "Returns the path to the .dsc file that was fetched" + + if package.endswith('.dsc'): + cmd = ('dget', '--download-only', '--allow-unauthenticated', package) + Logger.command(cmd) + ret = subprocess.call(cmd, cwd=workdir) + if ret == 0: + return os.path.join(workdir, os.path.basename(package)) + sys.exit(1) + if not source_release and not version: source_release = launchpad.distributions['ubuntu'].current_series.name @@ -190,40 +199,10 @@ def dscurls_from_package(launchpad, mirror, package, version, source_release): else: srcpkg = find_version_package(launchpad, package, version) - urls = [] - if mirror: - urls.append(dsc_url(mirror, srcpkg.component_name, package, - srcpkg.source_package_version)) - - for source_file in srcpkg.sourceFileUrls(): - if source_file.endswith('.dsc'): - urls.append(urllib.unquote(source_file)) - return urls - else: - error('Package %s contains no .dsc file.' % package) - -def dscurl_from_dsc(package): - path = os.path.abspath(os.path.expanduser(package)) - if os.path.exists(path): - return 'file://%s' % path - else: - # Can't resolve it as a local path? Let's just hope it's good as-is - return package - -def fetch_package(launchpad, mirror, workdir, package, version, source_release): - # Returns the path to the .dsc file that was fetched - if package.endswith('.dsc'): - dscs = [dscurl_from_dsc(package)] - else: - dscs = dscurls_from_package(launchpad, mirror, package, version, - source_release) - - for dsc in dscs: - cmd = ('dget', '--download-only', '--allow-unauthenticated', dsc) - Logger.command(cmd) - ret = subprocess.call(cmd, cwd=workdir) - if ret == 0: - return os.path.join(workdir, os.path.basename(dsc)) + version = srcpkg.source_package_version + pull_source_pkg('UBUNTU', mirror, srcpkg.component_name, package, version, + workdir=workdir, unpack=False) + return dsc_name(package, version) def get_backport_version(version, suffix, upload, release): backport_version = version + ('~%s1' % release) diff --git a/debian/copyright b/debian/copyright index e6bd445..8e9f677 100644 --- a/debian/copyright +++ b/debian/copyright @@ -187,6 +187,7 @@ Files: doc/pull-debian-debdiff.1, ubuntutools/config.py, ubuntutools/control.py, ubuntutools/logger.py, + ubuntutools/mirrors.py, ubuntutools/question.py, ubuntutools/sponsor_patch/*, ubuntutools/test/*, diff --git a/pull-debian-debdiff b/pull-debian-debdiff index aa4ac5c..bcccd58 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -17,112 +17,15 @@ # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. -import hashlib import optparse -import os.path import subprocess import sys -import urllib2 import debian.changelog from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger -from ubuntutools.misc import dsc_name, dsc_url - -DEFAULT_DEBIAN_MIRROR = 'http://ftp.debian.org/debian' -DEFAULT_DEBSEC_MIRROR = 'http://security.debian.org' - -def pull(package, version, opts, unpack=False): - "Download Debian source package version version" - urls = [] - # TODO: Not all packages are main :) - # Practically this is fine, as it'll be found on snapshot, but still ugly. - if opts.debsec_mirror and opts.debsec_mirror != DEFAULT_DEBSEC_MIRROR: - urls.append(dsc_url(opts.debsec_mirror, 'main', package, version)) - urls.append(dsc_url(DEFAULT_DEBSEC_MIRROR, 'main', package, version)) - if opts.debian_mirror and opts.debian_mirror != DEFAULT_DEBIAN_MIRROR: - urls.append(dsc_url(opts.debian_mirror, 'main', package, version)) - urls.append(dsc_url(DEFAULT_DEBIAN_MIRROR, 'main', package, version)) - - for url in urls: - cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd) - if return_code == 0: - return True - - Logger.normal('Trying snapshot.debian.org') - return pull_from_snapshot(package, version, unpack) - -def pull_from_snapshot(package, version, unpack=False): - "Download Debian source package version version from snapshot.debian.org" - try: - import json - except ImportError: - import simplejson as json - except ImportError: - Logger.error("Please install python-simplejson.") - sys.exit(1) - - try: - srcfiles = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' - % (package, version))) - except urllib2.HTTPError: - Logger.error('Version %s of %s not found on snapshot.debian.org', - version, package) - return False - - for hash_ in srcfiles['result']: - hash_ = hash_['hash'] - - try: - info = json.load(urllib2.urlopen( - 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) - except urllib2.URLError: - Logger.error('Unable to dowload info for hash.') - return False - - filename = info['result'][0]['name'] - if '/' in filename: - Logger.error('Unacceptable file name: %s', filename) - return False - - if os.path.exists(filename): - source_file = open(filename, 'r') - sha1 = hashlib.sha1() - sha1.update(source_file.read()) - source_file.close() - if sha1.hexdigest() == hash_: - Logger.normal('Using existing %s', filename) - continue - - Logger.normal('Downloading: %s (%0.3f MiB)', filename, - info['result'][0]['size'] / 1024.0 / 1024) - try: - in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) - out = open(filename, 'w') - while True: - block = in_.read(10240) - if block == '': - break - out.write(block) - sys.stdout.write('.') - sys.stdout.flush() - sys.stdout.write('\n') - sys.stdout.flush() - out.close() - except urllib2.URLError: - Logger.error('Error downloading %s', filename) - return False - - if unpack: - cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version)) - Logger.command(cmd) - subprocess.check_call(cmd) - - return True +from ubuntutools.mirrors import dsc_name, pull_source_pkg def previous_version(package, version, distance): "Given an (extracted) package, determine the version distance versions ago" @@ -177,9 +80,13 @@ def main(): opts.debsec_mirror = config.get_value('DEBSEC_MIRROR') Logger.normal('Downloading %s %s', package, version) - if not pull(package, version, opts, unpack=not opts.fetch_only): - Logger.error("Couldn't locate version %s of %s.", version, package) - sys.exit(1) + + # TODO: Not all packages are main, but snapshot.debian.org should save + # the day, as it doesn't care about component. + pull_source_pkg(('DEBSEC', 'DEBIAN'), + {'DEBSEC': opts.debsec_mirror, + 'DEBIAN': opts.debian_mirror}, + 'main', package, version, unpack=True) if opts.fetch_only: sys.exit(0) @@ -189,9 +96,10 @@ def main(): Logger.error('No previous version could be found') sys.exit(1) Logger.normal('Downloading %s %s', package, oldversion) - if not pull(package, oldversion, opts, unpack=True): - Logger.error("Couldn't locate version %s of %s.", oldversion, package) - sys.exit(1) + pull_source_pkg(('DEBSEC', 'DEBIAN'), + {'DEBSEC': opts.debsec_mirror, + 'DEBIAN': opts.debian_mirror}, + 'main', package, oldversion, unpack=True) cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version)) Logger.command(cmd) diff --git a/pull-lp-source b/pull-lp-source index f11fb92..feb060c 100755 --- a/pull-lp-source +++ b/pull-lp-source @@ -25,16 +25,16 @@ import os import sys -import subprocess -import urllib from optparse import OptionParser from ubuntutools.config import UDTConfig from ubuntutools.logger import Logger from ubuntutools.lp.lpapicache import Distribution, Launchpad from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, - PackageNotFoundException, PocketDoesNotExistError) -from ubuntutools.misc import split_release_pocket, dsc_url + PackageNotFoundException, + PocketDoesNotExistError) +from ubuntutools.mirrors import pull_source_pkg +from ubuntutools.misc import split_release_pocket def main(): usage = "Usage: %prog [release]" @@ -83,27 +83,9 @@ def main(): Logger.error(error) sys.exit(1) - urls = [] - if options.ubuntu_mirror: - urls.append(dsc_url(options.ubuntu_mirror, spph.getComponent(), - package, spph.getVersion())) - dsc_url = [url for url in spph.sourceFileUrls() if url.endswith('.dsc')] - assert dsc_url, 'No .dsc file found' - urls.append(urllib.unquote(dsc_url[0])) - - Logger.normal('Fetching the source for %s from %s (%s)...', - package, release.capitalize(), pocket) - for url in urls: - cmd = ('dget', '-u' + ('d' if options.download_only else 'x'), url) - Logger.command(cmd) - return_code = subprocess.call(cmd) - if return_code == 0: - Logger.normal("Success!") - sys.exit(0) - - Logger.error('Failed to fetch and extrace the source. ' - 'Please check the output for the error.') - sys.exit(1) + pull_source_pkg('UBUNTU', options.ubuntu_mirror, spph.getComponent(), + package, spph.getVersion(), + unpack=not options.download_only) if __name__ == '__main__': main() diff --git a/ubuntutools/config.py b/ubuntutools/config.py index 1445b3a..8793aa6 100644 --- a/ubuntutools/config.py +++ b/ubuntutools/config.py @@ -34,11 +34,11 @@ class UDTConfig(object): # These are reqired to be used by at least two scripts. defaults = { 'BUILDER': 'pbuilder', - 'DEBIAN_MIRROR': None, - 'DEBSEC_MIRROR': None, + 'DEBIAN_MIRROR': 'http://ftp.debian.org/debian', + 'DEBSEC_MIRROR': 'http://security.debian.org', 'LPINSTANCE': 'production', 'MIRROR_FALLBACK': True, - 'UBUNTU_MIRROR': None, + 'UBUNTU_MIRROR': 'http://archive.ubuntu.com/ubuntu', 'UPDATE_BUILDER': False, 'WORKDIR': None, } @@ -74,7 +74,7 @@ class UDTConfig(object): f.close() return config - def get_value(self, key, default=None, boolean=False, compat_keys=[]): + def get_value(self, key, default=None, boolean=False, compat_keys=()): """Retrieve a value from the environment or configuration files. keys are prefixed with the script name, falling back to UBUNTUTOOLS for package-wide keys. diff --git a/ubuntutools/mirrors.py b/ubuntutools/mirrors.py new file mode 100644 index 0000000..4cdc33e --- /dev/null +++ b/ubuntutools/mirrors.py @@ -0,0 +1,170 @@ +# mirrors.py - Functions for dealing with Debian source packages and mirrors. +# +# Copyright (C) 2010, Stefano Rivera +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +import hashlib +import os.path +import subprocess +import urllib2 +import sys + +from ubuntutools.config import UDTConfig +from ubuntutools.logger import Logger + +def dsc_name(package, version): + "Return the source package dsc filename for the given package" + if ':' in version: + version = version.split(':', 1)[1] + return '%s_%s.dsc' % (package, version) + +def dsc_url(mirror, component, package, version): + "Build a source package URL" + group = package[:4] if package.startswith('lib') else package[0] + filename = dsc_name(package, version) + return os.path.join(mirror, 'pool', component, group, package, filename) + +def pull_source_pkg(archives, mirrors, component, package, version, workdir='.', + unpack=False): + """Download a source package or die. + archives may be a list or single item (in which case mirrors can be too) + mirrors should be a dict (keyed on archive) unless archives is single""" + + if not isinstance(archives, (tuple, list)): + if not isinstance(mirrors, dict): + mirrors = {archives: mirrors} + archives = [archives] + assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives) + + for archive in archives: + if try_pull_from_archive(archive, mirrors.get(archive), component, + package, version, workdir, unpack): + return + + if 'DEBIAN' in archives or 'DEBSEC' in archives: + Logger.info('Trying snapshot.debian.org') + if try_pull_from_snapshot(package, version, workdir, unpack): + return + + if 'UBUNTU' in archives: + Logger.info('Trying Launchpad') + if try_pull_from_lp(package, 'ubuntu', version, workdir, unpack): + return + + raise Exception('Unable to locate %s/%s %s' % (package, component, version)) + +def try_pull_from_archive(archive, mirror, component, package, version, + workdir='.', unpack=False): + """Download a source package from the specified source or return False. + Try mirror first, then master. + """ + assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU') + urls = [] + if mirror and mirror != UDTConfig.defaults[archive + '_MIRROR']: + urls.append(dsc_url(mirror, component, package, version)) + urls.append(dsc_url(UDTConfig.defaults[archive + '_MIRROR'], component, + package, version)) + + for url in urls: + cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) + Logger.command(cmd) + return_code = subprocess.call(cmd, cwd=workdir) + if return_code == 0: + return True + + return False + +def try_pull_from_snapshot(package, version, workdir='.', unpack=False): + """Download Debian source package version version from snapshot.debian.org + or return False + """ + try: + import json + except ImportError: + import simplejson as json + except ImportError: + Logger.error("Please install python-simplejson.") + sys.exit(1) + + try: + srcfiles = json.load(urllib2.urlopen( + 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles' + % (package, version))) + except urllib2.HTTPError: + Logger.error('Version %s of %s not found on snapshot.debian.org', + version, package) + return False + + for hash_ in srcfiles['result']: + hash_ = hash_['hash'] + + try: + info = json.load(urllib2.urlopen( + 'http://snapshot.debian.org/mr/file/%s/info' % hash_)) + except urllib2.URLError: + Logger.error('Unable to dowload info for hash.') + return False + + filename = info['result'][0]['name'] + if '/' in filename: + Logger.error('Unacceptable file name: %s', filename) + return False + pathname = os.path.join(workdir, filename) + + if os.path.exists(pathname): + source_file = open(pathname, 'r') + sha1 = hashlib.sha1() + sha1.update(source_file.read()) + source_file.close() + if sha1.hexdigest() == hash_: + Logger.normal('Using existing %s', filename) + continue + + Logger.normal('Downloading: %s (%0.3f MiB)', filename, + info['result'][0]['size'] / 1024.0 / 1024) + try: + in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_) + out = open(pathname, 'w') + while True: + block = in_.read(10240) + if block == '': + break + out.write(block) + sys.stdout.write('.') + sys.stdout.flush() + sys.stdout.write('\n') + sys.stdout.flush() + out.close() + except urllib2.URLError: + Logger.error('Error downloading %s', filename) + return False + + if unpack: + cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version)) + Logger.command(cmd) + subprocess.check_call(cmd) + return True + +def try_pull_from_lp(package, distro, version, workdir='.', unpack=False): + """Try to download the specified version of a source package from Launchpad + or return False + """ + url = ('https://launchpad.net/%s/+archive/primary/+files/%s' + % (distro, dsc_name(package, version))) + cmd = ('dget', '-u' + ('x' if unpack else 'd'), url) + Logger.command(cmd) + return_code = subprocess.call(cmd, cwd=workdir) + if return_code == 0: + return True + return False diff --git a/ubuntutools/misc.py b/ubuntutools/misc.py index 84ad93e..dd581d3 100644 --- a/ubuntutools/misc.py +++ b/ubuntutools/misc.py @@ -119,15 +119,3 @@ def split_release_pocket(release): pocket) return (release, pocket) - -def dsc_name(package, version): - "Return the source package dsc filename for the given package" - if ':' in version: - version = version.split(':', 1)[1] - return '%s_%s.dsc' % (package, version) - -def dsc_url(mirror, component, package, version): - "Build a source package URL" - group = package[:4] if package.startswith('lib') else package[0] - filename = dsc_name(package, version) - return os.path.join(mirror, 'pool', component, group, package, filename)