* pull-debian-debdiff: Rewrite in Python, and use snapshot.debian.org.

* pull-lp-source: Support -d (LP: #681699)
This commit is contained in:
Stefano Rivera 2010-12-25 16:26:43 +01:00 committed by Benjamin Drung
commit 9f340a7562
15 changed files with 536 additions and 236 deletions

View File

@ -34,6 +34,7 @@ from ubuntutools.config import UDTConfig, ubu_email
from ubuntutools.builder import get_builder
from ubuntutools.logger import Logger
from ubuntutools.question import YesNoQuestion
from ubuntutools.misc import dsc_url
def error(msg):
Logger.error(msg)
@ -98,6 +99,11 @@ def parse(args):
default=None,
help='Specify a working directory (default: temporary dir)',
metavar='WORKDIR')
p.add_option('-m', '--mirror',
dest='ubuntu_mirror',
default=None,
help='Preferred Ubuntu mirror (default: Launchpad)',
metavar='INSTANCE')
p.add_option('-l', '--lpinstance',
dest='lpinstance',
default=None,
@ -121,6 +127,8 @@ def parse(args):
opts.workdir = config.get_value('WORKDIR')
if opts.lpinstance is None:
opts.lpinstance = config.get_value('LPINSTANCE')
if opts.ubuntu_mirror is None:
opts.ubuntu_mirror = config.get_value('UBUNTU_MIRROR')
if not opts.upload and not opts.workdir:
p.error('Please specify either a working dir or an upload target!')
@ -163,7 +171,7 @@ def find_version_package(lp, package, version):
error('Version %s of package %s was never published in Ubuntu.' %
(version, package))
def dscurl_from_package(lp, workdir, package, version, source_release):
def dscurls_from_package(lp, mirror, workdir, package, version, source_release):
if not source_release and not version:
source_release = lp.distributions['ubuntu'].current_series.name
@ -173,9 +181,15 @@ def dscurl_from_package(lp, workdir, package, version, source_release):
else:
srcpkg = find_version_package(lp, package, version)
urls = []
if mirror:
urls.append(dsc_url(mirror, srcpkg.component_name, package,
srcpkg.source_package_version))
for f in srcpkg.sourceFileUrls():
if f.endswith('.dsc'):
return urllib.unquote(f)
urls.append(urllib.unquote(f))
return urls
else:
error('Package %s contains no .dsc file.' % package)
@ -187,17 +201,20 @@ def dscurl_from_dsc(package):
# Can't resolve it as a local path? Let's just hope it's good as-is
return package
def fetch_package(lp, workdir, package, version, source_release):
def fetch_package(lp, mirror, workdir, package, version, source_release):
# Returns the path to the .dsc file that was fetched
if package.endswith('.dsc'):
dsc = dscurl_from_dsc(package)
dscs = [dscurl_from_dsc(package)]
else:
dsc = dscurl_from_package(lp, workdir, package, version, source_release)
dscs = dscurls_from_package(lp, mirror, workdir, package, version,
source_release)
check_call(['dget', '--download-only', '--allow-unauthenticated', dsc],
cwd=workdir)
return os.path.join(workdir, os.path.basename(dsc))
for dsc in dscs:
cmd = ('dget', '--download-only', '--allow-unauthenticated', dsc)
Logger.command(cmd)
ret = subprocess.call(cmd, cwd=workdir)
if ret == 0:
return os.path.join(workdir, os.path.basename(dsc))
def get_backport_version(version, suffix, upload, release):
v = version + ('~%s1' % release)
@ -295,6 +312,7 @@ def main(args):
try:
dscfile = fetch_package(lp,
opts.ubuntu_mirror,
workdir,
package_or_dsc,
opts.version,

8
debian/changelog vendored
View File

@ -11,6 +11,8 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
* Support reading configuration variables from devscripts configuration
files. (LP: #681693)
- Added ubuntu-dev-tools.5
- Support this in many u-d-t scripts, and update manpages.
- Deprecate old configuration environment variables.
* Support the combined "Name <email>" format in UBUMAIL, DEBFULLNAME, and
DEBEMAIL. (LP: #665202)
* Add the beginnings of a test suite. (LP: #690386)
@ -19,12 +21,12 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
- 404main, merge-changelog, pull-debian-debdiff, pull-debian-source,
pull-revu-source:
+ Return 0 after showing help.
- Support this in many u-d-t scripts, and update manpages.
- Deprecate old configuration environment variables.
* ubuntutools/common.py: Remove https_proxy unsetting code, working around
LP: #94130.
* edit-patch: Don't let cat error through if debian/source/format doesn't
exist.
* pull-debian-debdiff: Rewrite in Python, and use snapshot.debian.org.
* pull-lp-source: Support -d (LP: #681699)
[ Michael Bienia ]
* ubuntutools/lp/lpapicache.py: Allow easier selection of 'staging' as LP
@ -46,7 +48,7 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
* add "add-patch" that provides the non-interactive version of
edit-patch
-- Stefano Rivera <stefanor@ubuntu.com> Thu, 23 Dec 2010 17:37:53 +0200
-- Stefano Rivera <stefanor@dvorak.kardiogramm.lan> Fri, 24 Dec 2010 12:16:10 +0200
ubuntu-dev-tools (0.108) experimental; urgency=low

1
debian/control vendored
View File

@ -16,6 +16,7 @@ Build-Depends: dctrl-tools,
python-apt (>= 0.7.93~),
python-debian (>= 0.1.15),
python-gnupginterface,
python-simplejson,
python-launchpadlib (>= 1.5.7),
python-magic,
python-setuptools,

7
debian/copyright vendored
View File

@ -132,7 +132,6 @@ Files: dch-repeat,
doc/lp-list-bugs.1,
doc/manage-credentials.1,
doc/mk-sbuild.1,
doc/pull-debian-debdiff.1,
doc/pull-debian-source.1,
doc/pull-lp-source.1,
doc/pull-revu-source.1,
@ -144,7 +143,6 @@ Files: dch-repeat,
lp-list-bugs,
manage-credentials,
mk-sbuild,
pull-debian-debdiff,
pull-debian-source,
pull-lp-source,
pull-revu-source,
@ -165,6 +163,7 @@ Copyright: 2007, Albin Tonnerre (Lutin) <lut1n.tne@gmail.com>
2009-2010, Michael Bienia <geser@ubuntu.com>
2009, Nathan Handler <nhandler@ubuntu.com>
2007-2008, Siegfried-Angel Gevatter Pujals <rainct@ubuntu.com>
2010, Stefano Rivera <stefanor@ubuntu.com>
License: GPL-3+
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -179,10 +178,12 @@ License: GPL-3+
On Debian systems, the complete text of the GNU General Public License
version 3 can be found in the /usr/share/common-licenses/GPL-3 file.
Files: doc/sponsor-patch.1,
Files: doc/pull-debian-debdiff.1,
doc/sponsor-patch.1,
doc/suspicious-source.1,
doc/ubuntu-dev-tools.5,
doc/wrap-and-sort.1,
pull-debian-debdiff,
sponsor-patch,
suspicious-source,
ubuntutools/builder.py,

View File

@ -80,6 +80,12 @@ unpacked, built into, and otherwise manipulated in
\fIWORKDIR\fR. Otherwise, a temporary directory is created, which is
deleted before \fIbackportpackage\fR exits.
.TP
.B \-m \fIUBUNTU_MIRROR\fR, \fB\-\-mirror\fR=\fIUBUNTU_MIRROR\fR
Use the specified mirror.
Should be in the form \fBhttp://archive.ubuntu.com/ubuntu\fR.
If the package isn't found on this mirror, \fBbackportpackage\fR
will use Launchpad.
.TP
.B \-l \fIINSTANCE\fR, \fB\-\-lpinstance\fR=\fIINSTANCE\fR
Use the specified instance of Launchpad (e.g. "staging"), instead of
the default of "production".
@ -108,6 +114,9 @@ The default value for \fB--update\fR.
.BR BACKPORTPACKAGE_WORKDIR ", " UBUNTUTOOLS_WORKDIR
The default value for \fB--workdir\fR.
.TP
.BR BACKPORTPACKAGE_UBUNTU_MIRROR ", " UBUNTUTOOLS_UBUNTU_MIRROR
The default value for \fB\-\-mirror\fR.
.TP
.BR BACKPORTPACKAGE_LPINSTANCE ", " UBUNTUTOOLS_LPINSTANCE
The default value for \fB--lpinstance\fR.
.SH EXAMPLES

View File

@ -1,16 +1,91 @@
.TH PULL-DEBIAN-DEBDIFF "1" "June 2010" "ubuntu-dev-tools"
.\" Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
.\"
.\" Permission to use, copy, modify, and/or distribute this software for any
.\" purpose with or without fee is hereby granted, provided that the above
.\" copyright notice and this permission notice appear in all copies.
.\"
.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
.\" REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
.\" AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
.\" INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
.\" LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
.\" OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
.\" PERFORMANCE OF THIS SOFTWARE.
.TH PULL-DEBIAN-DEBDIFF "1" "December 2010" "ubuntu-dev-tools"
.SH NAME
\fBpull-debian-debdiff\fR \- find, download, and generate a debdiff
.SH SYNOPSIS
\fBpull-debian-debdiff\fR <package> <version>
\fBpull-debian-debdiff\fR [\fIoptions\fR] <\fIpackage\fR>
<\fIversion\fR> [\fIdistance\fR]
.SH DESCRIPTION
\fBpull-debian-debdiff\fR attempts to find and download a specific version of a Debian package and its immediate parent to generate a debdiff.
\fBpull-debian-debdiff\fR attempts to find and download a specific
version of a Debian package and its immediate parent to generate a
debdiff.
.SH OPTIONS
.TP
.I package
The source package to download and diff.
.TP
.I version
The most recent of the two versions you want to diff.
.TP
.I distance
If specified (default \fB1\fR), the debdiff is against that many
versions previous.
.TP
.BR \-h ", " \-\-help
Display the usage instructions and exit.
.TP
.BR \-f ", " \-\-fetch
Simply download the specified version and exit.
.TP
.B \-d \fIDEBIAN_MIRROR\fR, \fB\-\-debian\-mirror\fR=\fIDEBIAN_MIRROR\fR
Use the specified mirror.
Should be in the form \fBhttp://ftp.debian.org/debian\fR.
If the package isn't found on this mirror, \fBpull\-debian\-source\fR
will fall back to the default mirror.
.TP
.B \-s \fIDEBSEC_MIRROR\fR, \fB\-\-debsec\-mirror\fR=\fIDEBSEC_MIRROR\fR
Use the specified Debian security mirror.
Should be in the form \fBhttp://security.debian.org\fR.
If the package isn't found on this mirror, \fBpull\-debian\-source\fR
will fall back to the default mirror.
.TP
.B \-\-no\-conf
Do not read any configuration files, or configuration from environment
variables.
.SH ENVIRONMENT
All of the \fBCONFIGURATION VARIABLES\fR below are also supported as
environment variables.
Variables in the environment take precedence to those in configuration
files.
.SH CONFIGURATION VARIABLES
The following variables can be set in the environment or in
.BR ubuntu\-dev\-tools (5)
configuration files.
In each case, the script\-specific variable takes precedence over the
package\-wide variable.
.TP
.BR PULL_DEBIAN_DEBDIFF_DEBIAN_MIRROR ", " UBUNTUTOOLS_DEBIAN_MIRROR
The default value for \fB\-\-debian\-mirror\fR.
.TP
.BR PULL_DEBIAN_DEBDIFF_DEBSEC_MIRROR ", " UBUNTUTOOLS_DEBSEC_MIRROR
The default value for \fB\-\-debsec\-mirror\fR.
.SH SEE ALSO
.BR debdiff (1),
.BR dget (1),
.BR pull\-debian\-source (1),
.BR ubuntu\-dev\-tools (5)
.SH AUTHORS
\fBpull-debian-debdiff\fR was written by Kees Cook <kees@ubuntu.com>.
\fBpull-debian-debdiff\fR was written by Stefano Rivera
<stefanor@ubuntu.com>, a clone of a tool by Kees Cook <kees@ubuntu.com>.
This manual page was written by Andrew Starr-Bochicchio <a.starr.b@gmail.com>.
.PP
Both are released under the terms of the GNU General Public License, version 3, or (at your option) any later version.
This manual page was written by Stefano Rivera, based on the original by
Andrew Starr\-Bochicchio <a.starr.b@gmail.com>.

View File

@ -4,7 +4,7 @@
pull\-debian\-source \- download a source package from Debian
.SH SYNOPSIS
.B pull\-debian\-source \fR[\fB\-h\fR]\fB <\fBsource package\fR> [\fItarget release\fR]
.B pull\-debian\-source \fR[\fIoptions\fR] <\fIsource package\fR> [\fItarget release\fR]
.SH DESCRIPTION
\fBpull\-debian\-source\fR downloads and extracts the latest version of
@ -15,15 +15,47 @@ version in that release will be downloaded instead.
.SH OPTIONS
Listed below are the command line options for pull\-debian\-source:
.TP
.B \-h, \-\-help
Display the usage instructions and exit.
.TP
.B <source package>
.I source package
This is the source package that you would like to be downloaded from Debian.
.TP
.B [target release]
.I target release
This is the release that you would like the source package to be downloaded from.
This value defaults to 'unstable'.
.TP
.BR \-h ", " \-\-help
Display the usage instructions and exit.
.TP
.B \-m \fIDEBIAN_MIRROR\fR, \fB\-\-mirror\fR=\fIDEBIAN_MIRROR\fR
Use the specified mirror.
Should be in the form \fBhttp://ftp.debian.org/debian\fR.
If the package isn't found on this mirror, \fBpull\-debian\-source\fR
will fall back to the default mirror.
.TP
.B \-\-no\-conf
Do not read any configuration files, or configuration from environment
variables.
.SH ENVIRONMENT
All of the \fBCONFIGURATION VARIABLES\fR below are also supported as
environment variables.
Variables in the environment take precedence to those in configuration
files.
.SH CONFIGURATION VARIABLES
The following variables can be set in the environment or in
.BR ubuntu\-dev\-tools (5)
configuration files.
In each case, the script\-specific variable takes precedence over the
package\-wide variable.
.TP
.BR PULL_DEBIAN_SOURCE_DEBIAN_MIRROR ", " UBUNTUTOOLS_DEBIAN_MIRROR
The default value for \fB\-\-mirror\fR.
.SH SEE ALSO
.BR dget (1),
.BR pull\-debian\-debdiff (1),
.BR pull\-lp\-source (1),
.BR ubuntu\-dev\-tools (5)
.SH AUTHOR
.PP

View File

@ -4,7 +4,7 @@
pull\-lp\-source \- download a source package from Launchpad
.SH SYNOPSIS
.B pull\-lp\-source \fR[\fB\-h\fR]\fB <\fBsource package\fR> [\fItarget release\fR]
.B pull\-lp\-source \fR[\fIoptions\fR]\fB <\fBsource package\fR> [\fItarget release\fR]
.SH DESCRIPTION
\fBpull\-lp\-source\fR downloads and extracts the latest version of
@ -15,21 +15,52 @@ version in that release will be downloaded instead.
.SH OPTIONS
Listed below are the command line options for pull\-lp\-source:
.TP
.B \-h, \-\-help
Display a help message and exit.
.TP
.B <source package>
This is the source package that you would like to be downloaded from Launchpad.
.TP
.B [target release]
This is the release that you would like the source package to be downloaded from.
This value defaults to the current development release.
.SH ENVIRONMENT VARIABLES
.TP
.BR \-h ", " \-\-help
Display a help message and exit.
.TP
.B \-m \fIUBUNTU_MIRROR\fR, \fB\-\-mirror\fR=\fIUBUNTU_MIRROR\fR
Use the specified Ubuntu mirror.
Should be in the form \fBhttp://archive.ubuntu.com/ubuntu\fR.
If the package isn't found on this mirror, \fBpull\-lp\-source\fR will
fall back to Launchpad, as its name implies.
.TP
.B \-\-no\-conf
Do not read any configuration files, or configuration from environment
variables.
.SH ENVIRONMENT
All of the \fBCONFIGURATION VARIABLES\fR below are also supported as
environment variables.
Variables in the environment take precedence to those in configuration
files.
.TP
.B
DIST
Specifies the default target.
.SH CONFIGURATION VARIABLES
The following variables can be set in the environment or in
.BR ubuntu\-dev\-tools (5)
configuration files.
In each case, the script\-specific variable takes precedence over the
package\-wide variable.
.TP
.BR PULL_LP_SOURCE_UBUNTU_MIRROR ", " UBUNTUTOOLS_UBUNTU_MIRROR
The default value for \fB\-\-mirror\fR.
.SH SEE ALSO
.BR dget (1),
.BR pull\-debian\-source (1),
.BR pull\-debian\-debdiff (1),
.BR ubuntu\-dev\-tools (5)
.SH AUTHOR
.PP
\fBpull\-lp\-source\fR and this manual page were written by Iain Lane

View File

@ -56,10 +56,35 @@ The currently recognised package\-wide variables are:
This specifies the preferred test\-builder, one of
.BR pbuilder " (default), " sbuild ", " pbuilder\-dist .
.TP
.B UBUNTUTOOLS_DEBIAN_MIRROR
The preferred Debian archive mirror.
Should be of the form \fBhttp://ftp.debian.org/debian\fR (no trailing
slash).
If not specified, the master will be used.
.TP
.B UBUNTUTOOLS_DEBSEC_MIRROR
The preferred Debian security archive mirror.
Should be of the form \fBhttp://security.debian.org\fR (no trailing
slash).
If not specified, the master will be used.
.TP
.B UBUNTUTOOLS_UBUNTU_MIRROR
The preferred Ubuntu archive mirror.
Should be of the form \fBhttp://archive.ubuntu.com/ubuntu\fR (no
trailing slash).
If not specified, the master will be used.
.TP
.B UBUNTUTOOLS_LPINSTANCE
The launchpad instance to communicate with. e.g. \fBproduction\fR
(default) or \fBstaging\fR.
.TP
.B UBUNTUTOOLS_MIRROR_FALLBACK
Whether or not to fall\-back to the master archive mirror.
This is usually the desired behaviour, as mirrors can lag the masters.
If on a private network with only a local mirror, you may want to set
this to \fBno\fR.
.RB "One of " yes " (default) or " no .
.TP
.B UBUNTUTOOLS_UPDATE_BUILDER
Whether or not to update the test\-builder before each test build.
.RB "One of " yes " or " no " (default).

View File

@ -1,192 +1,207 @@
#!/usr/bin/perl
#!/usr/bin/python
# pull-debian-debdiff - find and download a specific version of a Debian
# package and its immediate parent to generate a debdiff.
#
# Copyright 2007-2008 Kees Cook <kees@ubuntu.com>
# Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
# Inspired by a tool of the same name by Kees Cook.
#
# ##################################################################
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# See file /usr/share/common-licenses/GPL for more details.
#
# ##################################################################
#
# This script attempts to find and download a specific version of a Debian
# package and its immediate parent to generate a debdiff.
#
# Requirements: devscripts diffstat dpkg-dev
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
use strict;
use warnings;
import hashlib
import optparse
import os.path
import subprocess
import sys
import urllib2
sub geturls
{
my ($urlbase,$pkg,$version)=@_;
my $file;
import debian.changelog
try:
import json
except ImportError:
import simplejson as json
$file = "${pkg}_${version}.dsc";
print "Want '$file'\n";
if (! -r "$file") {
warn "Trying $urlbase/$file ...\n";
system("wget $urlbase/$file");
return 0 if ($? != 0);
}
from ubuntutools.config import UDTConfig
from ubuntutools.logger import Logger
from ubuntutools.misc import dsc_name, dsc_url
# Parse the .dsc file for list of required files...
my @needed;
open(DSC,"$file") || return 0;
while (my $line=<DSC>) {
if ($line =~ /^Files:/) {
while (my $file=<DSC>) {
chomp($file);
last if ($file !~ /^ /);
my @parts = split(/\s+/,$file);
my $want = pop(@parts);
print "Want '$want'\n";
push(@needed,$want);
}
}
}
close(DSC);
DEFAULT_DEBIAN_MIRROR = 'http://ftp.debian.org/debian'
DEFAULT_DEBSEC_MIRROR = 'http://security.debian.org'
foreach my $file (@needed) {
if (! -r "$file") {
warn "Pulling $urlbase/$file ...\n";
system("wget $urlbase/$file");
return 0 if ($? != 0);
}
}
opts = None
return 1;
}
def pull(package, version, unpack=False):
"Download Debian source package version version"
urls = []
# TODO: Not all packages are main :)
# Practically this is fine, as it'll be found on snapshot, but still ugly.
if opts.debsec_mirror and opts.debsec_mirror != DEFAULT_DEBSEC_MIRROR:
urls.append(dsc_url(opts.debsec_mirror, 'main', package, version))
urls.append(dsc_url(DEFAULT_DEBSEC_MIRROR, 'main', package, version))
if opts.debian_mirror and opts.debian_mirror != DEFAULT_DEBIAN_MIRROR:
urls.append(dsc_url(opts.debian_mirror, 'main', package, version))
urls.append(dsc_url(DEFAULT_DEBIAN_MIRROR, 'main', package, version))
sub generate_base
{
my ($pkg)=@_;
for url in urls:
cmd = ('dget', '-u' + ('x' if unpack else 'd'), url)
Logger.command(cmd)
p = subprocess.call(cmd)
if p == 0:
return True
my @path;
push(@path,"main");
if ($pkg =~ /^(lib.)/) {
push(@path,$1);
}
else {
push(@path,substr($pkg,0,1));
}
push(@path,$pkg);
return join("/",@path);
}
Logger.normal('Trying snapshot.debian.org')
return pull_from_snapshot(package, version, unpack)
sub download_source
{
my ($pkg,$version)=@_;
my $urlbase;
def pull_from_snapshot(package, version, unpack=False):
"Download Debian source package version version from snapshot.debian.org"
try:
srcfiles = json.load(urllib2.urlopen(
'http://snapshot.debian.org/mr/package/%s/%s/srcfiles'
% (package, version)))
except urllib2.HTTPError:
Logger.error('Version %s of %s not found on snapshot.debian.org',
version, package)
return False
for hash_ in srcfiles['result']:
hash_ = hash_['hash']
my $base = generate_base($pkg);
try:
info = json.load(urllib2.urlopen(
'http://snapshot.debian.org/mr/file/%s/info' % hash_))
except urllib2.URLError:
Logger.error('Unable to dowload info for hash.')
return False
# Attempt to pull from security updates first
$urlbase = "http://security.debian.org/pool/updates/$base";
fn = info['result'][0]['name']
if '/' in fn:
Logger.error('Unacceptable file name: %s', fn)
return False
if (!geturls($urlbase,$pkg,$version)) {
# Try regular pool
if os.path.exists(fn):
f = open(fn, 'r')
s = hashlib.sha1()
s.update(f.read())
f.close()
if s.hexdigest() == hash_:
Logger.normal('Using existing %s', fn)
continue
$urlbase = "http://ftp.debian.org/debian/pool/$base";
if (!geturls($urlbase,$pkg,$version)) {
# Try snapshot
Logger.normal('Downloading: %s (%0.3f MiB)', fn,
info['result'][0]['size'] / 1024.0 / 1024)
try:
in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_)
out = open(fn, 'w')
while True:
b = in_.read(10240)
if b == '':
break
out.write(b)
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
sys.stdout.flush()
out.close()
except urllib2.URLError:
Logger.error('Error downloading %s', fn)
return False
$urlbase="http://snapshot.debian.net/package/$pkg/$version";
warn "Fetching snapshot url via '$urlbase' ...\n";
$urlbase=`curl -sI 'http://snapshot.debian.net/package/$pkg/$version' | grep ^[lL]ocation | cut -d' ' -f2 | head -1`;
$urlbase =~ s/[\r\n]//g;
warn "Trying snapshot location '$urlbase' ...\n";
if unpack:
cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version))
Logger.command(cmd)
subprocess.check_call(cmd)
if ($urlbase ne "" && !geturls($urlbase,$pkg,$version)) {
return 0;
}
}
}
return True
return 1;
}
def previous_version(package, version, distance):
"Given an (extracted) package, determine the version distance versions ago"
upver = version
if ':' in upver:
upver = upver.split(':', 1)[1]
upver = upver.split('-')[0]
fn = '%s-%s/debian/changelog' % (package, upver)
f = open(fn, 'r')
c = debian.changelog.Changelog(f.read())
f.close()
seen = 0
for entry in c:
if entry.distributions == 'UNRELEASED':
continue
if seen == distance:
return entry.version.full_version
seen += 1
return False
sub usage
{
my ($exit) = @_;
print "Usage: $0 PKG VERSION\n";
exit $exit;
}
def main():
global opts
p = optparse.OptionParser('%prog [options] <package> <version> [distance]')
p.add_option('-f', '--fetch',
dest='fetch_only', default=False, action='store_true',
help="Only fetch the source packages, don't diff.")
p.add_option('-d', '--debian-mirror', metavar='DEBIAN_MIRROR',
dest='debian_mirror',
help='Preferred Debian mirror '
'(default: http://ftp.debian.org/debian)')
p.add_option('-s', '--debsec-mirror', metavar='DEBSEC_MIRROR',
dest='debsec_mirror',
help='Preferred Debian Security mirror '
'(default: http://security.debian.org)')
p.add_option('--no-conf',
dest='no_conf', default=False, action='store_true',
help="Don't read config files or environment variables")
opts, args = p.parse_args()
if len(args) < 2:
p.error('Must specify package and version')
elif len(args) > 3:
p.error('Too many arguments')
package = args[0]
version = args[1]
distance = args[2] if len(args) > 2 else 1
config = UDTConfig(opts.no_conf)
if opts.debian_mirror is None:
opts.debian_mirror = config.get_value('DEBIAN_MIRROR')
if opts.debsec_mirror is None:
opts.debsec_mirror = config.get_value('DEBSEC_MIRROR')
my $pkg = $ARGV[0];
my $version = $ARGV[1];
my $just_fetch = ($ARGV[2] && $ARGV[2] eq "--fetch");
my $skip = $ARGV[2] || 1;
$skip+=0;
Logger.normal('Downloading %s %s', package, version)
if not pull(package, version, unpack=not opts.fetch_only):
Logger.error("Couldn't locate version %s of %s.", version, package)
sys.exit(1)
if (defined($pkg) && ($pkg eq '--help' || $pkg eq '-h')) {
usage(0);
} elsif (!defined($pkg) || !defined($version)) {
usage(2);
}
if opts.fetch_only:
sys.exit(0)
oldversion = previous_version(package, version, distance)
if not oldversion:
Logger.error('No previous version could be found')
sys.exit(1)
Logger.normal('Downloading %s %s', package, oldversion)
if not pull(package, oldversion, unpack=True):
Logger.error("Couldn't locate version %s of %s.", oldversion, package)
sys.exit(1)
# Extract latest source
die "Cannot locate $pkg $version\n" unless download_source($pkg,$version);
exit(0) if ($just_fetch);
system("dpkg-source -x ${pkg}_${version}.dsc");
die "Unpack of $pkg $version failed\n" unless ($? == 0);
cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version))
Logger.command(cmd)
difffn = dsc_name(package, version)[:-3] + 'debdiff'
f = open(difffn, 'w')
if subprocess.call(cmd, stdout=f) > 2:
Logger.error('Debdiff failed.')
sys.exit(1)
f.close()
cmd = ('diffstat', '-p0', difffn)
Logger.command(cmd)
subprocess.check_call(cmd)
print difffn
# Locate prior changelog entry
my $prev_ver;
my $upstream_version = $version;
if ($upstream_version =~ /^([^-]+)-/) {
$upstream_version = $1;
}
my $srcdir="$pkg-$upstream_version";
if (! -d "$srcdir") {
undef $srcdir;
my $dir;
opendir(DIR,".");
while ($dir = readdir(DIR)) {
if ($dir =~ /^${pkg}-/ && -d $dir) {
$srcdir = $dir;
last;
}
}
closedir(DIR);
}
die "Cannot locate source tree\n" if (!defined($srcdir));
my $log = "$srcdir/debian/changelog";
open(LOG,"<$log") || die "$log: $!\n";
while (my $line=<LOG>) {
if ($line =~ /^$pkg \((?:\d+:)?([^\)]+)\)/) {
my $seen = $1;
if ($seen ne $version) {
$skip--;
if ($skip==0) {
$prev_ver=$seen;
last;
}
}
}
}
close(LOG);
die "Cannot find earlier source version\n" if (!defined($prev_ver));
die "Cannot locate $pkg $prev_ver\n" unless download_source($pkg,$prev_ver);
#system("dpkg-source -x ${pkg}_${prev_ver}.dsc");
#die "Unpack of $pkg $prev_ver failed\n" unless ($? == 0);
system("debdiff ${pkg}_${prev_ver}.dsc ${pkg}_${version}.dsc > ${pkg}_${version}.debdiff");
die "Cannot debdiff\n" unless ($? == 0);
system("diffstat -p0 ${pkg}_${version}.debdiff");
print "${pkg}_${version}.debdiff\n";
if __name__ == '__main__':
main()

View File

@ -2,7 +2,8 @@
# Script Name: pull-debian-source
# Author: Nathan Handler <nhandler@ubuntu.com>
# Usage: pull-debian-source <source package> [release]
# Copyright (C) 2008, 2009 Nathan Handler <nhandler@ubuntu.com>
# Copyright (C) 2008-2009 Nathan Handler <nhandler@ubuntu.com>,
# 2010 Stefano Rivera <stefanor@ubuntu.com>
# License: GNU General Public License
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -26,17 +27,53 @@ use AptPkg::Version;
die("Please install 'devscripts'\n") if(! grep -x "$_/dget", split(':',$ENV{'PATH'}));
my($name)=basename($0);
my($package)=$ARGV[0] || &usage(2);
my($help)=0;
GetOptions('help' => \$help);
my($name) = basename($0);
my($help, $mirror, $no_conf);
GetOptions('h|help' => \$help,
'm|mirror=s' => \$mirror,
'no-conf' => \$no_conf,
);
my($package) = $ARGV[0] || &usage(2);
&usage(0) if($help);
if (! $no_conf) {
my($shell_cmd);
$shell_cmd .= "[ -f /etc/devscripts.conf ] && . /etc/devscripts.conf\n";
$shell_cmd .= "[ -f ~/.devscripts ] && . ~/.devscripts\n";
foreach my $var qw(PULL_DEBIAN_SOURCE_DEBIAN_MIRROR
UBUNTUTOOLS_DEBIAN_MIRROR
PULL_DEBIAN_SOURCE_MIRROR_FALLBACK
UBUNTUTOOLS_MIRROR_FALLBACK) {
$shell_cmd .= "echo $var=\$$var\n";
}
my $shell_out = `/bin/bash -c '$shell_cmd'`;
my %config_values;
foreach my $line (split /\n/, $shell_out) {
my($k, $v) = split /=/, $line, 2;
$config_values{$k} = $v;
}
$mirror = $config_values{'PULL_DEBIAN_SOURCE_DEBIAN_MIRROR'}
|| $config_values{'UBUNTUTOOLS_DEBIAN_MIRROR'}
if (! $mirror);
}
my($default_mirror) = 'http://ftp.debian.org/debian';
my(@mirrors);
push @mirrors, $mirror if $mirror && $mirror ne $default_mirror;
push @mirrors, $default_mirror;
my($release)=$ARGV[1] || 'unstable';
$release=&convertCodeName($release);
&checkRelease($release);
my($dsc)=&getDSC(&getMadison(&getURL($package,$release)));
print "$dsc\n";
exec("dget -xu $dsc");
my($madison) = &getMadison(&getURL($package,$release));
foreach my $mirror (@mirrors) {
my($dsc)=&getDSC($madison, $mirror);
print "$dsc\n";
system("dget -xu $dsc");
exit(0) if ($? == 0);
}
exit(1);
sub convertCodeName {
my($release)=shift || die("No Release Passed To convertCodeName!\n");
chomp $release;
@ -90,12 +127,12 @@ sub getDSC {
if($madison=~m/^[WE]:/i) {
die("$madison");
}
my($baseURL)='http://ftp.debian.org/debian/pool/';
my($baseURL)=shift || die ("No baseURL Passed to getDSC: $!\n");
my(@madison)=split(/\n/,$madison);
my %urls;
my $url;
foreach my $line (@madison) {
$url = $baseURL;
$url = $baseURL . '/pool/';
my($package,$version,$release,$archs)=split(/\|/,$line,4);
$package=~s/\s*//g;
$version=~s/\s*//g;
@ -133,7 +170,16 @@ sub getDSC {
}
sub usage {
my($exit)=shift;
print("USAGE: $name [-h] <source package> [target release]\n");
print <<"EOF";
USAGE: $name [options] <source package> [target release]
Options:
-h, --help Show this help message and exit
-m DEBIAN_MIRROR, --mirror=DEBIAN_MIRROR
Preferred Debian mirror
(default: http://ftp.debian.org/debian)
--no-conf Don't read config files or environment variables
EOF
exit($exit);
}
sub invalidRelease {

View File

@ -3,10 +3,8 @@
# pull-lp-source -- pull a source package from Launchpad
# Basic usage: pull-lp-source <source package> [<release>]
#
# Copyright (C) 2008 Iain Lane <iain@orangesquash.org.uk>
#
# BackportFromLP class taken from prevu tool, which is:
# Copyright (C) 2006 John Dong <jdong@ubuntu.com>
# Copyright (C) 2008, Iain Lane <iain@orangesquash.org.uk>,
# 2010, Stefano Rivera <stefanor@ubuntu.com>
#
# ##################################################################
#
@ -31,25 +29,40 @@ import subprocess
import urllib
from optparse import OptionParser
# ubuntu-dev-tools modules.
from ubuntutools.config import UDTConfig
from ubuntutools.logger import Logger
from ubuntutools.lp.lpapicache import Distribution, Launchpad
from ubuntutools.lp.udtexceptions import (SeriesNotFoundException,
PackageNotFoundException, PocketDoesNotExistError)
from ubuntutools.misc import splitReleasePocket
from ubuntutools.misc import splitReleasePocket, dsc_name, dsc_url
if not os.path.exists("/usr/bin/dget"):
print "E: dget is not installed - please install the 'devscripts' package" \
" and rerun this script again."
print ("E: dget is not installed - please install the 'devscripts' package"
" and rerun this script again.")
sys.exit(1)
if __name__ == '__main__':
usage = "Usage: %prog <package> [release]"
optParser = OptionParser(usage)
optParser.add_option('-d', '--download-only',
dest='download_only', default=False,
action='store_true',
help="Do not extract the source package")
optParser.add_option('-m', '--mirror', metavar='UBUNTU_MIRROR',
dest='ubuntu_mirror',
help='Preferred Ubuntu mirror '
'(default: Launchpad)')
optParser.add_option('--no-conf',
dest='no_conf', default=False, action='store_true',
help="Don't read config files or environment "
"variables")
(options, args) = optParser.parse_args()
if not args:
optParser.print_help()
sys.exit(1)
optParser.error("Must specify package name")
config = UDTConfig(options.no_conf)
if options.ubuntu_mirror is None:
options.ubuntu_mirror = config.get_value('UBUNTU_MIRROR')
# Login anonymously to LP
Launchpad.login_anonymously()
@ -59,28 +72,41 @@ if __name__ == '__main__':
if len(args) >= 2: # Custom distribution specified.
release = str(args[1]).lower()
else:
release = os.getenv('DIST') or Distribution('ubuntu').getDevelopmentSeries().name
release = os.getenv('DIST') or (Distribution('ubuntu')
.getDevelopmentSeries().name)
try:
(release, pocket) = splitReleasePocket(release)
except PocketDoesNotExistError, e:
print 'E: %s' % e
Logger.error(e)
sys.exit(1)
try:
spph = Distribution('ubuntu').getArchive().getSourcePackage(package, release, pocket)
spph = Distribution('ubuntu').getArchive().getSourcePackage(package,
release,
pocket)
except (SeriesNotFoundException, PackageNotFoundException), e:
print 'E: %s' % e
Logger.error(e)
sys.exit(1)
urls = []
if options.ubuntu_mirror:
urls.append(dsc_url(options.ubuntu_mirror, spph.getComponent(),
package, spph.getVersion()))
dsc_url = [url for url in spph.sourceFileUrls() if url.endswith('.dsc')]
assert dsc_url, 'No .dsc file found'
urls.append(urllib.unquote(dsc_url[0]))
# All good - start downloading...
print 'Fetching the source for %s from %s (%s)...' % (
package, release.capitalize(), pocket)
if subprocess.call(['/usr/bin/dget', '-xu', urllib.unquote(dsc_url[0])]) == 0:
print 'Success!'
else:
print 'Failed to fetch and extrace the source.', \
'Please check the output for the error.'
Logger.normal('Fetching the source for %s from %s (%s)...',
package, release.capitalize(), pocket)
for url in urls:
cmd = ('dget', '-u' + ('d' if options.download_only else 'x'), url)
Logger.command(cmd)
r = subprocess.call(cmd)
if r == 0:
Logger.normal("Success!")
sys.exit(0)
Logger.error('Failed to fetch and extrace the source. '
'Please check the output for the error.')
sys.exit(1)

View File

@ -34,7 +34,11 @@ class UDTConfig(object):
# These are reqired to be used by at least two scripts.
defaults = {
'BUILDER': 'pbuilder',
'DEBIAN_MIRROR': None,
'DEBSEC_MIRROR': None,
'LPINSTANCE': 'production',
'MIRROR_FALLBACK': True,
'UBUNTU_MIRROR': None,
'UPDATE_BUILDER': False,
'WORKDIR': None,
}

View File

@ -226,7 +226,8 @@ class Distribution(BaseWrapper):
'''
if name_or_version not in self._series:
try:
series = DistroSeries(self().getSeries(name_or_version))
series = DistroSeries(
self().getSeries(name_or_version=name_or_version))
# Cache with name and version
self._series[series.name] = series
self._series[series.version] = series

View File

@ -1,8 +1,9 @@
#
# misc.py - misc functions for the Ubuntu Developer Tools scripts.
#
# Copyright (C) 2008 Jonathan Davies <jpds@ubuntu.com>
# Copyright (C) 2008-2009 Siegfried-Angel Gevatter Pujals <rainct@ubuntu.com>
# Copyright (C) 2008, Jonathan Davies <jpds@ubuntu.com>,
# 2008-2009, Siegfried-Angel Gevatter Pujals <rainct@ubuntu.com>,
# 2010, Stefano Rivera <stefanor@ubuntu.com>
#
# ##################################################################
#
@ -22,6 +23,7 @@
# Modules.
import os
import os.path
from subprocess import Popen, PIPE
from ubuntutools.lp.udtexceptions import PocketDoesNotExistError
@ -116,3 +118,15 @@ def splitReleasePocket(release):
pocket)
return (release, pocket)
def dsc_name(package, version):
"Return the source package dsc filename for the given package"
if ':' in version:
version = version.split(':', 1)[1]
return '%s_%s.dsc' % (package, version)
def dsc_url(mirror, component, package, version):
"Build a source package URL"
group = package[:4] if package.startswith('lib') else package[0]
fn = dsc_name(package, version)
return os.path.join(mirror, 'pool', component, group, package, fn)