- Support this in many u-d-t scripts, and update manpages.

- Deprecate old configuration environment variables.
* pull-debian-debdiff: Rewrite in Python, and use snapshot.debian.org.

debian/changelog

@@ -11,6 +11,8 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
   * Support reading configuration variables from devscripts configuration
     files. (LP: #681693)
     - Added ubuntu-dev-tools.5
+    - Support this in many u-d-t scripts, and update manpages.
+    - Deprecate old configuration environment variables.
   * Support the combined "Name <email>" format in UBUMAIL, DEBFULLNAME, and
     DEBEMAIL. (LP: #665202)
   * Add the beginnings of a test suite. (LP: #690386)
@@ -19,12 +21,11 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
     - 404main, merge-changelog, pull-debian-debdiff, pull-debian-source,
       pull-revu-source:
       + Return 0 after showing help.
-    - Support this in many u-d-t scripts, and update manpages.
-    - Deprecate old configuration environment variables.
   * ubuntutools/common.py: Remove https_proxy unsetting code, working around
     LP: #94130.
   * edit-patch: Don't let cat error through if debian/source/format doesn't
     exist.
+  * pull-debian-debdiff: Rewrite in Python, and use snapshot.debian.org.
 
   [ Michael Bienia ]
   * ubuntutools/lp/lpapicache.py: Allow easier selection of 'staging' as LP
@@ -46,7 +47,7 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
   * add "add-patch" that provides the non-interactive version of
     edit-patch
 
- -- Stefano Rivera <stefanor@ubuntu.com>  Thu, 23 Dec 2010 17:37:53 +0200
+ -- Stefano Rivera <stefanor@ubuntu.com>  Fri, 24 Dec 2010 02:57:36 +0200
 
 ubuntu-dev-tools (0.108) experimental; urgency=low
 

debian/control

@@ -16,6 +16,7 @@ Build-Depends: dctrl-tools,
                python-apt (>= 0.7.93~),
                python-debian (>= 0.1.15),
                python-gnupginterface,
+               python-simplejson,
                python-launchpadlib (>= 1.5.7),
                python-magic,
                python-setuptools,

doc/pull-debian-debdiff.1

@@ -1,4 +1,17 @@
-.TH PULL-DEBIAN-DEBDIFF "1" "June 2010" "ubuntu-dev-tools"
+.\" Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
+.\"
+.\" Permission to use, copy, modify, and/or distribute this software for any
+.\" purpose with or without fee is hereby granted, provided that the above
+.\" copyright notice and this permission notice appear in all copies.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+.\" REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+.\" AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+.\" INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+.\" LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+.\" OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+.\" PERFORMANCE OF THIS SOFTWARE.
+.TH PULL-DEBIAN-DEBDIFF "1" "December 2010" "ubuntu-dev-tools"
 .SH NAME
 \fBpull-debian-debdiff\fR \- find, download, and generate a debdiff
 
@@ -76,8 +89,8 @@ The default value for \fB\-\-no\-fallback\fR.
 .BR ubuntu\-dev\-tools (5)
 
 .SH AUTHORS
-\fBpull-debian-debdiff\fR was written by Kees Cook <kees@ubuntu.com>.
+\fBpull-debian-debdiff\fR was written by Stefano Rivera
+<stefanor@ubuntu.com>, a clone of a tool by Kees Cook <kees@ubuntu.com>.
 
-This manual page was written by Andrew Starr-Bochicchio <a.starr.b@gmail.com>.
+This manual page was written by Stefano Rivera, based on the original by
-.PP
+Andrew Starr\-Bochicchio <a.starr.b@gmail.com>.
-Both are released under the terms of the GNU General Public License, version 3, or (at your option) any later version.

pull-debian-debdiff

@@ -1,237 +1,213 @@
-#!/usr/bin/perl
+#!/usr/bin/python
+# pull-debian-debdiff - find and download a specific version of a Debian
+#                       package and its immediate parent to generate a debdiff.
 #
-# Copyright 2007-2008, Kees Cook <kees@ubuntu.com>,
+# Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
-#                2010, Stefano Rivera <stefanor@ubuntu.com>
+# Inspired by a tool of the same name by Kees Cook.
 #
-# ##################################################################
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
 #
-# This program is free software; you can redistribute it and/or
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-# modify it under the terms of the GNU General Public License
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# as published by the Free Software Foundation; either version 3
+# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-# of the License, or (at your option) any later version.
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-#
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-# This program is distributed in the hope that it will be useful,
+# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# PERFORMANCE OF THIS SOFTWARE.
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# See file /usr/share/common-licenses/GPL for more details.
-#
-# ##################################################################
-#
-# This script attempts to find and download a specific version of a Debian
-# package and its immediate parent to generate a debdiff.
-#
-# Requirements: devscripts diffstat dpkg-dev
 
-use strict;
+import optparse
-use warnings;
+import os.path
-use File::Basename;
+import subprocess
-use Getopt::Long;
+import sys
-use LWP::Simple;
+import urllib2
 
-die("Please install 'devscripts'\n") if(! grep -x "$_/dget", split(':',$ENV{'PATH'}));
+import debian.changelog
+try:
+    import json
+except ImportError:
+    import simplejson as json
 
-my($debmirror, $debsecmirror, $no_fallback);
+from ubuntutools.config import UDTConfig
+from ubuntutools.logger import Logger
 
-sub geturls
+DEFAULT_DEBMIRROR = 'http://ftp.debian.org/debian'
-{
+DEFAULT_DEBSECMIRROR = 'http://security.debian.org'
-    my ($urlbase,$pkg,$version)=@_;
-    my $file;
 
-    $file = "${pkg}_${version}.dsc";
+opts = None
-    print "Want '$file'\n";
-    system("dget -du $urlbase/$file");
-    return 0 if ($? != 0);
-    return 1;
-}
 
-sub generate_base
+def dsc_name(package, version):
-{
+    "Return the source package dsc filename for the given package"
-    my ($pkg)=@_;
+    if ':' in version:
+        version = version.split(':', 1)[1]
+    return '%s_%s.dsc' % (package, version)
 
-    my @path;
+def build_url(mirror, package, version):
-    push(@path,"main");
+    "Build a source package URL"
-    if ($pkg =~ /^(lib.)/) {
+    group = package[:4] if package.startswith('lib') else package[0]
-        push(@path,$1);
+    fn = dsc_name(package, version)
-    }
+    # TODO: Not all packages are main :)
-    else {
+    # Practically this is fine, as it'll be found on snapshot, but still ugly.
-        push(@path,substr($pkg,0,1));
+    return os.path.join(mirror, 'pool', 'main', group, package, fn)
-    }
-    push(@path,$pkg);
-    return join("/",@path);
-}
 
-sub download_source
+def pull(package, version, unpack=False):
-{
+    "Download Debian source package version version"
-    my ($pkg,$version)=@_;
+    urls = []
-    my $urlbase;
+    if opts.debsecmirror and opts.debsecmirror != DEFAULT_DEBSECMIRROR:
+        urls.append(build_url(opts.debsecmirror, package, version))
+    if opts.fallback:
+        urls.append(build_url(DEFAULT_DEBSECMIRROR, package, version))
+    if opts.debmirror and opts.debmirror != DEFAULT_DEBMIRROR:
+        urls.append(build_url(opts.debmirror, package, version))
+    if opts.fallback:
+        urls.append(build_url(DEFAULT_DEBMIRROR, package, version))
 
-    my $base = generate_base($pkg);
+    for url in urls:
+        cmd = ('dget', '-u' + 'x' if unpack else 'd', url)
+        Logger.command(cmd)
+        p = subprocess.call(cmd)
+        if p == 0:
+            return True
 
-    my $defdebmirror = 'http://ftp.debian.org/debian';
+    Logger.normal('Trying snapshot.debian.org')
-    my $defdebsecmirror = 'http://security.debian.org';
+    return pull_from_snapshot(package, version, unpack)
 
-    my @mirrors;
+def pull_from_snapshot(package, version, unpack=False):
-    # Attempt to pull from security updates first
+    "Download Debian source package version version from snapshot.debian.org"
-    push @mirrors, "$debsecmirror/pool/updates/$base" if $debsecmirror;
+    try:
-    push @mirrors, "$defdebsecmirror/pool/updates/$base"
+        srcfiles = json.load(urllib2.urlopen(
-            if $debsecmirror ne $defdebsecmirror and !$no_fallback;
+                'http://snapshot.debian.org/mr/package/%s/%s/srcfiles'
-    # Try regular pool:
+                % (package, version)))
-    push @mirrors, "$debmirror/pool/$base" if $debmirror;
+    except urllib2.HTTPError:
-    push @mirrors, "$defdebmirror/pool/$base"
+        Logger.error('Version %s of %s not found on snapshot.debian.org',
-            if $debmirror ne $defdebmirror and !$no_fallback;
+                     version, package)
+        return False
+    for hash_ in srcfiles['result']:
+        hash_ = hash_['hash']
+        try:
+            info = json.load(urllib2.urlopen(
+                'http://snapshot.debian.org/mr/file/%s/info' % hash_))
+        except urllib2.URLError:
+            Logger.error('Unable to dowload info for hash.')
+            return False
+        fn = info['result'][0]['name']
+        if '/' in fn:
+            Logger.error('Unacceptable file name: %s', fn)
+            return False
+        Logger.normal('Downloading: %s (%0.3f MiB)', fn,
+                      info['result'][0]['size'] / 1024.0 / 1024)
+        try:
+            in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_)
+            out = open(fn, 'w')
+            while True:
+                b = in_.read(10240)
+                if b == '':
+                    break
+                out.write(b)
+                sys.stdout.write('.')
+                sys.stdout.flush()
+            sys.stdout.write('\n')
+            sys.stdout.flush()
+            out.close()
+        except urllib2.URLError:
+            Logger.error('Error downloading %s', fn)
+            return False
 
-    foreach $urlbase (@mirrors) {
+    if unpack:
-        return 1 if geturls($urlbase, $pkg, $version);
+        cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version))
-    }
+        Logger.command(cmd)
+        subprocess.check_call(cmd)
 
-    # Try snapshot:
+    return True
-    $urlbase="http://snapshot.debian.net/package/$pkg/$version";
-    warn "Fetching snapshot url via '$urlbase' ...\n";
-    my $scrape=get('http://snapshot.debian.net/package/$pkg/$version/');
-    $scrape =~ /a href=\"()
-    $urlbase =~ /
-    $urlbase =~ s/[\r\n]//g;
-    warn "Trying snapshot location '$urlbase' ...\n";
 
-    if ($urlbase ne "" && !geturls($urlbase,$pkg,$version)) {
+def previous_version(package, version, distance):
-        return 0;
+    "Given an (extracted) package, determine the version distance versions ago"
-    }
+    upver = version
+    if ':' in upver:
+        upver = upver.split(':', 1)[1]
+    upver = upver.split('-')[0]
+    fn = '%s-%s/debian/changelog' % (package, upver)
+    f = open(fn, 'r')
+    c = debian.changelog.Changelog(f.read())
+    f.close()
+    seen = 0
+    for entry in c:
+        if entry.distributions == 'UNRELEASED':
+            continue
+        if seen == distance:
+            return entry.version.full_version
+        seen += 1
+    return False
 
-    return 1;
+def main():
-}
+    global opts
+    p = optparse.OptionParser('%prog [options] <package> <version> [distance]')
+    p.add_option('-f', '--fetch',
+                 dest='fetch_only', default=False, action='store_true',
+                 help="Only fetch the source packages, don't diff.")
+    p.add_option('-d', '--debmirror', metavar='DEBMIRROR',
+                 dest='debmirror',
+                 help='Preferred Debian mirror '
+                      '(default: http://ftp.debian.org/debian)')
+    p.add_option('-s', '--debsecmirror', metavar='DEBSECMIRROR',
+                 dest='debsecmirror',
+                 help='Preferred Debian Security mirror '
+                      '(default: http://security.debian.org)')
+    p.add_option('-n', '--no-fallback',
+                 dest='fallback', default=None, action='store_false',
+                 help="If a custom mirror is provided and an error occurs "
+                      "while downloading, don't fall back to the default")
+    p.add_option('--no-conf',
+                 dest='no_conf', default=False, action='store_true',
+                 help="Don't read config files or environment variables")
 
-sub usage
+    opts, args = p.parse_args()
-{
+    if len(args) < 2:
-    my ($exit) = @_;
+        p.error('Must specify package and version')
-    my ($name) = basename($0);
+    elif len(args) > 3:
-    print <<"EOF";
+        p.error('Too many arguments')
-Usage: $name [options] PKG VERSION [DISTANCE]
+    package = args[0]
+    version = args[1]
+    distance = args[2] if len(args) > 2 else 1
 
-Attempts to find and download version VERSION of Debian package PKG and its
+    config = UDTConfig(opts.no_conf)
-immediate parent to generate a debdiff.
+    if opts.debmirror is None:
-If DISTANCE is specified, the debdiff is against DISTANCE versions before
+        opts.debmirror = config.get_value('DEBMIRROR')
-VERSION.
+    if opts.debsecmirror is None:
+        opts.debsecmirror = config.get_value('DEBSECMIRROR')
+    if opts.fallback is None:
+        opts.fallback = config.get_value('MIRROR_FALLBACK', boolean=True)
 
-Options:
+    Logger.normal('Downloading %s %s', package, version)
-  -h, --help            Show this help message and exit
+    if not pull(package, version, unpack=not opts.fetch_only):
-  -f, --fetch           Only fetch the source packages, don't diff.
+        Logger.error("Couldn't locate version %s of %s.", version, package)
-  -d DEBMIRROR, --debmirror=DEBMIRROR
+        sys.exit(1)
-                        Preferred Debian mirror
-                        (default: http://ftp.debian.org/debian)
-  -s DEBSECMIRROR, --debsecmirror=DEBSECMIRROR
-                        Preferred Debian Security mirror
-                        (default: http://security.debian.org)
-  -n, --no-fallback     If a custom mirror is provided and an error occurs
-                        while downloading, don't fall back to the default
-  --no-conf             Don't read config files or environment variables
-EOF
-    exit $exit;
-}
 
+    if opts.fetch_only:
+        sys.exit(0)
 
-my($help, $just_fetch, $no_conf);
+    oldversion = previous_version(package, version, distance)
-GetOptions('h|help' => \$help,
+    if not oldversion:
-           'f|fetch' => \$just_fetch,
+        Logger.error('No previous version could be found')
-           'd|debmirror=s' => \$debmirror,
+        sys.exit(1)
-           's|debsecmirror=s' => \$debsecmirror,
+    Logger.normal('Downloading %s %s', package, oldversion)
-           'n|no-fallback' => \$no_fallback,
+    if not pull(package, oldversion, unpack=True):
-           'no-conf' => \$no_conf,
+        Logger.error("Couldn't locate version %s of %s.", oldversion, package)
-          );
+        sys.exit(1)
 
-my $pkg = $ARGV[0];
+    cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version))
-my $version = $ARGV[1];
+    Logger.command(cmd)
-my $skip = $ARGV[2] || 1;
+    difffn = dsc_name(package, version)[:-3] + 'debdiff'
-$skip += 0;
+    f = open(difffn, 'w')
+    if subprocess.call(cmd, stdout=f) > 2:
+        Logger.error('Debdiff failed.')
+        sys.exit(1)
+    f.close()
+    cmd = ('diffstat', '-p0', difffn)
+    Logger.command(cmd)
+    subprocess.check_call(cmd)
+    print difffn
 
-if ($help) {
+if __name__ == '__main__':
-    usage(0);
+    main()
-} elsif (!defined($pkg) || !defined($version)) {
-    usage(2);
-}
-
-# Read configuration files
-if (! $no_conf) {
-    my($shell_cmd);
-    $shell_cmd .= "[ -f /etc/devscripts.conf ] && . /etc/devscripts.conf\n";
-    $shell_cmd .= "[ -f ~/.devscripts ] && . ~/.devscripts\n";
-    foreach my $var qw(PULL_DEBIAN_DEBDIFF_DEBMIRROR UBUNTUTOOLS_DEBMIRROR
-                       PULL_DEBIAN_DEBDIFF_DEBSECMIRROR
-                       UBUNTUTOOLS_DEBSECMIRROR
-                       PULL_DEBIAN_DEBDIFF_MIRROR_FALLBACK
-                       UBUNTUTOOLS_MIRROR_FALLBACK) {
-        $shell_cmd .= "echo $var=\$$var\n";
-    }
-    my $shell_out = `/bin/bash -c '$shell_cmd'`;
-    my %config_values;
-    foreach my $line (split /\n/, $shell_out) {
-        my($k, $v) = split /=/, $line, 2;
-        $config_values{$k} = $v;
-    }
-    $debmirror = $config_values{'PULL_DEBIAN_DEBDIFF_DEBMIRROR'}
-                 || $config_values{'UBUNTUTOOLS_DEBMIRROR'}
-                 if (! $debmirror);
-    $debsecmirror = $config_values{'PULL_DEBIAN_DEBDIFF_DEBSECMIRROR'}
-                    || $config_values{'UBUNTUTOOLS_DEBSECMIRROR'}
-                    if (! $debsecmirror);
-    if (! $no_fallback) {
-        my($v) = $config_values{'PULL_DEBIAN_DEBDIFF_MIRROR_FALLBACK'}
-                 || $config_values{'UBUNTUTOOLS_MIRROR_FALLBACK'};
-        $no_fallback = 1 if $v eq "no";
-    }
-}
-
-# Extract latest source
-die "Cannot locate $pkg $version\n" unless download_source($pkg,$version);
-exit(0) if ($just_fetch);
-system("dpkg-source -x ${pkg}_${version}.dsc");
-die "Unpack of $pkg $version failed\n" unless ($? == 0);
-
-# Locate prior changelog entry
-my $prev_ver;
-my $upstream_version = $version;
-if ($upstream_version =~ /^([^-]+)-/) {
-    $upstream_version = $1;
-}
-my $srcdir="$pkg-$upstream_version";
-if (! -d "$srcdir") {
-    undef $srcdir;
-    my $dir;
-    opendir(DIR,".");
-    while ($dir = readdir(DIR)) {
-        if ($dir =~ /^${pkg}-/ && -d $dir) {
-            $srcdir = $dir;
-            last;
-        }
-    }
-    closedir(DIR);
-}
-die "Cannot locate source tree\n" if (!defined($srcdir));
-my $log = "$srcdir/debian/changelog";
-open(LOG,"<$log") || die "$log: $!\n";
-while (my $line=<LOG>) {
-    if ($line =~ /^$pkg \((?:\d+:)?([^\)]+)\)/) {
-        my $seen = $1;
-        if ($seen ne $version) {
-            $skip--;
-
-            if ($skip==0) {
-                $prev_ver=$seen;
-                last;
-            }
-        }
-    }
-}
-close(LOG);
-die "Cannot find earlier source version\n" if (!defined($prev_ver));
-
-die "Cannot locate $pkg $prev_ver\n" unless download_source($pkg,$prev_ver);
-#system("dpkg-source -x ${pkg}_${prev_ver}.dsc");
-#die "Unpack of $pkg $prev_ver failed\n" unless ($? == 0);
-
-system("debdiff ${pkg}_${prev_ver}.dsc ${pkg}_${version}.dsc > ${pkg}_${version}.debdiff");
-die "Cannot debdiff\n" unless ($? == 0);
-
-system("diffstat -p0 ${pkg}_${version}.debdiff");
-print "${pkg}_${version}.debdiff\n";