ubuntu-dev-tools/pull-debian-debdiff

#!/usr/bin/python
# pull-debian-debdiff - find and download a specific version of a Debian
#                       package and its immediate parent to generate a debdiff.
#
# Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
# Inspired by a tool of the same name by Kees Cook.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

import hashlib
import optparse
import os.path
import subprocess
import sys
import urllib2

import debian.changelog

from ubuntutools.config import UDTConfig
from ubuntutools.logger import Logger
from ubuntutools.misc import dsc_name, dsc_url

DEFAULT_DEBIAN_MIRROR = 'http://ftp.debian.org/debian'
DEFAULT_DEBSEC_MIRROR = 'http://security.debian.org'

def pull(package, version, opts, unpack=False):
    "Download Debian source package version version"
    urls = []
    # TODO: Not all packages are main :)
    # Practically this is fine, as it'll be found on snapshot, but still ugly.
    if opts.debsec_mirror and opts.debsec_mirror != DEFAULT_DEBSEC_MIRROR:
        urls.append(dsc_url(opts.debsec_mirror, 'main', package, version))
    urls.append(dsc_url(DEFAULT_DEBSEC_MIRROR, 'main', package, version))
    if opts.debian_mirror and opts.debian_mirror != DEFAULT_DEBIAN_MIRROR:
        urls.append(dsc_url(opts.debian_mirror, 'main', package, version))
    urls.append(dsc_url(DEFAULT_DEBIAN_MIRROR, 'main', package, version))

    for url in urls:
        cmd = ('dget', '-u' + ('x' if unpack else 'd'), url)
        Logger.command(cmd)
        return_code = subprocess.call(cmd)
        if return_code == 0:
            return True

    Logger.normal('Trying snapshot.debian.org')
    return pull_from_snapshot(package, version, unpack)

def pull_from_snapshot(package, version, unpack=False):
    "Download Debian source package version version from snapshot.debian.org"
    try:
        import json
    except ImportError:
        import simplejson as json
    except ImportError:
        Logger.error("Please install python-simplejson.")
        sys.exit(1)

    try:
        srcfiles = json.load(urllib2.urlopen(
                'http://snapshot.debian.org/mr/package/%s/%s/srcfiles'
                % (package, version)))
    except urllib2.HTTPError:
        Logger.error('Version %s of %s not found on snapshot.debian.org',
                     version, package)
        return False

    for hash_ in srcfiles['result']:
        hash_ = hash_['hash']

        try:
            info = json.load(urllib2.urlopen(
                'http://snapshot.debian.org/mr/file/%s/info' % hash_))
        except urllib2.URLError:
            Logger.error('Unable to dowload info for hash.')
            return False

        filename = info['result'][0]['name']
        if '/' in filename:
            Logger.error('Unacceptable file name: %s', filename)
            return False

        if os.path.exists(filename):
            source_file = open(filename, 'r')
            sha1 = hashlib.sha1()
            sha1.update(source_file.read())
            source_file.close()
            if sha1.hexdigest() == hash_:
                Logger.normal('Using existing %s', filename)
                continue

        Logger.normal('Downloading: %s (%0.3f MiB)', filename,
                      info['result'][0]['size'] / 1024.0 / 1024)
        try:
            in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_)
            out = open(filename, 'w')
            while True:
                block = in_.read(10240)
                if block == '':
                    break
                out.write(block)
                sys.stdout.write('.')
                sys.stdout.flush()
            sys.stdout.write('\n')
            sys.stdout.flush()
            out.close()
        except urllib2.URLError:
            Logger.error('Error downloading %s', filename)
            return False

    if unpack:
        cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version))
        Logger.command(cmd)
        subprocess.check_call(cmd)

    return True

def previous_version(package, version, distance):
    "Given an (extracted) package, determine the version distance versions ago"
    upver = version
    if ':' in upver:
        upver = upver.split(':', 1)[1]
    upver = upver.split('-')[0]
    filename = '%s-%s/debian/changelog' % (package, upver)
    changelog_file = open(filename, 'r')
    changelog = debian.changelog.Changelog(changelog_file.read())
    changelog_file.close()
    seen = 0
    for entry in changelog:
        if entry.distributions == 'UNRELEASED':
            continue
        if seen == distance:
            return entry.version.full_version
        seen += 1
    return False

def main():
    parser = optparse.OptionParser('%prog [options] <package> <version> '
                                   '[distance]')
    parser.add_option('-f', '--fetch',
                      dest='fetch_only', default=False, action='store_true',
                      help="Only fetch the source packages, don't diff.")
    parser.add_option('-d', '--debian-mirror', metavar='DEBIAN_MIRROR',
                      dest='debian_mirror',
                      help='Preferred Debian mirror '
                           '(default: http://ftp.debian.org/debian)')
    parser.add_option('-s', '--debsec-mirror', metavar='DEBSEC_MIRROR',
                      dest='debsec_mirror',
                      help='Preferred Debian Security mirror '
                           '(default: http://security.debian.org)')
    parser.add_option('--no-conf',
                      dest='no_conf', default=False, action='store_true',
                      help="Don't read config files or environment variables")

    opts, args = parser.parse_args()
    if len(args) < 2:
        parser.error('Must specify package and version')
    elif len(args) > 3:
        parser.error('Too many arguments')
    package = args[0]
    version = args[1]
    distance = args[2] if len(args) > 2 else 1

    config = UDTConfig(opts.no_conf)
    if opts.debian_mirror is None:
        opts.debian_mirror = config.get_value('DEBIAN_MIRROR')
    if opts.debsec_mirror is None:
        opts.debsec_mirror = config.get_value('DEBSEC_MIRROR')

    Logger.normal('Downloading %s %s', package, version)
    if not pull(package, version, opts, unpack=not opts.fetch_only):
        Logger.error("Couldn't locate version %s of %s.", version, package)
        sys.exit(1)

    if opts.fetch_only:
        sys.exit(0)

    oldversion = previous_version(package, version, distance)
    if not oldversion:
        Logger.error('No previous version could be found')
        sys.exit(1)
    Logger.normal('Downloading %s %s', package, oldversion)
    if not pull(package, oldversion, opts, unpack=True):
        Logger.error("Couldn't locate version %s of %s.", oldversion, package)
        sys.exit(1)

    cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version))
    Logger.command(cmd)
    difffn = dsc_name(package, version)[:-3] + 'debdiff'
    debdiff_file = open(difffn, 'w')
    if subprocess.call(cmd, stdout=debdiff_file) > 2:
        Logger.error('Debdiff failed.')
        sys.exit(1)
    debdiff_file.close()
    cmd = ('diffstat', '-p0', difffn)
    Logger.command(cmd)
    subprocess.check_call(cmd)
    print difffn

if __name__ == '__main__':
    main()