ubuntu-dev-tools/pull-debian-debdiff

#!/usr/bin/python
# pull-debian-debdiff - find and download a specific version of a Debian
#                       package and its immediate parent to generate a debdiff.
#
# Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
# Inspired by a tool of the same name by Kees Cook.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

import optparse
import os.path
import subprocess
import sys
import urllib2

import debian.changelog
try:
    import json
except ImportError:
    import simplejson as json

from ubuntutools.config import UDTConfig
from ubuntutools.logger import Logger

DEFAULT_DEBIAN_MIRROR = 'http://ftp.debian.org/debian'
DEFAULT_DEBSEC_MIRROR = 'http://security.debian.org'

opts = None

def dsc_name(package, version):
    "Return the source package dsc filename for the given package"
    if ':' in version:
        version = version.split(':', 1)[1]
    return '%s_%s.dsc' % (package, version)

def build_url(mirror, package, version):
    "Build a source package URL"
    group = package[:4] if package.startswith('lib') else package[0]
    fn = dsc_name(package, version)
    # TODO: Not all packages are main :)
    # Practically this is fine, as it'll be found on snapshot, but still ugly.
    return os.path.join(mirror, 'pool', 'main', group, package, fn)

def pull(package, version, unpack=False):
    "Download Debian source package version version"
    urls = []
    if opts.debsec_mirror and opts.debsec_mirror != DEFAULT_DEBSEC_MIRROR:
        urls.append(build_url(opts.debsec_mirror, package, version))
    if opts.fallback:
        urls.append(build_url(DEFAULT_DEBSEC_MIRROR, package, version))
    if opts.debian_mirror and opts.debian_mirror != DEFAULT_DEBIAN_MIRROR:
        urls.append(build_url(opts.debian_mirror, package, version))
    if opts.fallback:
        urls.append(build_url(DEFAULT_DEBIAN_MIRROR, package, version))

    for url in urls:
        cmd = ('dget', '-u' + 'x' if unpack else 'd', url)
        Logger.command(cmd)
        p = subprocess.call(cmd)
        if p == 0:
            return True

    Logger.normal('Trying snapshot.debian.org')
    return pull_from_snapshot(package, version, unpack)

def pull_from_snapshot(package, version, unpack=False):
    "Download Debian source package version version from snapshot.debian.org"
    try:
        srcfiles = json.load(urllib2.urlopen(
                'http://snapshot.debian.org/mr/package/%s/%s/srcfiles'
                % (package, version)))
    except urllib2.HTTPError:
        Logger.error('Version %s of %s not found on snapshot.debian.org',
                     version, package)
        return False
    for hash_ in srcfiles['result']:
        hash_ = hash_['hash']
        try:
            info = json.load(urllib2.urlopen(
                'http://snapshot.debian.org/mr/file/%s/info' % hash_))
        except urllib2.URLError:
            Logger.error('Unable to dowload info for hash.')
            return False
        fn = info['result'][0]['name']
        if '/' in fn:
            Logger.error('Unacceptable file name: %s', fn)
            return False
        Logger.normal('Downloading: %s (%0.3f MiB)', fn,
                      info['result'][0]['size'] / 1024.0 / 1024)
        try:
            in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_)
            out = open(fn, 'w')
            while True:
                b = in_.read(10240)
                if b == '':
                    break
                out.write(b)
                sys.stdout.write('.')
                sys.stdout.flush()
            sys.stdout.write('\n')
            sys.stdout.flush()
            out.close()
        except urllib2.URLError:
            Logger.error('Error downloading %s', fn)
            return False

    if unpack:
        cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version))
        Logger.command(cmd)
        subprocess.check_call(cmd)

    return True

def previous_version(package, version, distance):
    "Given an (extracted) package, determine the version distance versions ago"
    upver = version
    if ':' in upver:
        upver = upver.split(':', 1)[1]
    upver = upver.split('-')[0]
    fn = '%s-%s/debian/changelog' % (package, upver)
    f = open(fn, 'r')
    c = debian.changelog.Changelog(f.read())
    f.close()
    seen = 0
    for entry in c:
        if entry.distributions == 'UNRELEASED':
            continue
        if seen == distance:
            return entry.version.full_version
        seen += 1
    return False

def main():
    global opts
    p = optparse.OptionParser('%prog [options] <package> <version> [distance]')
    p.add_option('-f', '--fetch',
                 dest='fetch_only', default=False, action='store_true',
                 help="Only fetch the source packages, don't diff.")
    p.add_option('-d', '--debian-mirror', metavar='DEBIAN_MIRROR',
                 dest='debian_mirror',
                 help='Preferred Debian mirror '
                      '(default: http://ftp.debian.org/debian)')
    p.add_option('-s', '--debsec-mirror', metavar='DEBSEC_MIRROR',
                 dest='debsec_mirror',
                 help='Preferred Debian Security mirror '
                      '(default: http://security.debian.org)')
    p.add_option('-n', '--no-fallback',
                 dest='fallback', default=None, action='store_false',
                 help="If a custom mirror is provided and an error occurs "
                      "while downloading, don't fall back to the default")
    p.add_option('--no-conf',
                 dest='no_conf', default=False, action='store_true',
                 help="Don't read config files or environment variables")

    opts, args = p.parse_args()
    if len(args) < 2:
        p.error('Must specify package and version')
    elif len(args) > 3:
        p.error('Too many arguments')
    package = args[0]
    version = args[1]
    distance = args[2] if len(args) > 2 else 1

    config = UDTConfig(opts.no_conf)
    if opts.debian_mirror is None:
        opts.debian_mirror = config.get_value('DEBIAN_MIRROR')
    if opts.debsec_mirror is None:
        opts.debsec_mirror = config.get_value('DEBSEC_MIRROR')
    if opts.fallback is None:
        opts.fallback = config.get_value('MIRROR_FALLBACK', boolean=True)

    Logger.normal('Downloading %s %s', package, version)
    if not pull(package, version, unpack=not opts.fetch_only):
        Logger.error("Couldn't locate version %s of %s.", version, package)
        sys.exit(1)

    if opts.fetch_only:
        sys.exit(0)

    oldversion = previous_version(package, version, distance)
    if not oldversion:
        Logger.error('No previous version could be found')
        sys.exit(1)
    Logger.normal('Downloading %s %s', package, oldversion)
    if not pull(package, oldversion, unpack=True):
        Logger.error("Couldn't locate version %s of %s.", oldversion, package)
        sys.exit(1)

    cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version))
    Logger.command(cmd)
    difffn = dsc_name(package, version)[:-3] + 'debdiff'
    f = open(difffn, 'w')
    if subprocess.call(cmd, stdout=f) > 2:
        Logger.error('Debdiff failed.')
        sys.exit(1)
    f.close()
    cmd = ('diffstat', '-p0', difffn)
    Logger.command(cmd)
    subprocess.check_call(cmd)
    print difffn

if __name__ == '__main__':
    main()