diff --git a/debian/changelog b/debian/changelog index 8201081..d8688da 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ ubuntu-dev-tools (0.61) UNRELEASED; urgency=low - * Changes go here. + [ Terence Simpson ] + * dgetlp: Replaced Bash version with a new Python script. -- Jonathan Davies Sun, 01 Feb 2009 10:57:51 +0000 diff --git a/dgetlp b/dgetlp index 95b02ee..f8382a4 100755 --- a/dgetlp +++ b/dgetlp @@ -1,22 +1,20 @@ -#!/bin/bash +#!/usr/bin/python +# -*- coding: UTF-8 -*- # Copyright (C) 2008 Terence Simpson -# Modified by Siegfried-A. Gevatter +# License: +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. # -# ################################################################## +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# See file /usr/share/common-licenses/GPL for more details. -# -# ################################################################## +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # This script simulates «dget»'s behaviour for files hosted at # launchpadlibrarian.net. @@ -26,16 +24,19 @@ # way as dget does, but from launchpadlibrarian.net, which doesn't # store all the files in the same directory. It (the script) assumes # that the files are stored in sequential directories on Launchpad -# Librarian and attemps to download and then unpack them. +# Librarian and attempts to download and then unpack them. +# This is a Python rewrite of the original bash script -GET="wget" -UNPACK="dpkg-source -x {dsc-file}" -DIRECT_TO_NULL="/dev/null" +import sys, os +from optparse import OptionParser +import urllib2 +import md5 +import subprocess +import GnuPGInterface +from cStringIO import StringIO +from email import FeedParser -usage() -{ -cat << EOF -Usage: $0 [-d] +Usage = u"""Usage: %prog [-d|(-v|-q)] This scripts simulates «dget»'s behaviour for files hosted at launchpadlibrarian.net. @@ -44,128 +45,274 @@ If you specify the -d option then it won't do anything, except download the .dsc file, but just print the commands it would run otherwise. Example: - $(basename $0) http://launchpadlibrarian.net/10348157/coreutils_5.97-5.4ubuntu1.dsc -EOF -} + %prog http://launchpadlibrarian.net/10348157/coreutils_5.97-5.4ubuntu1.dsc +""" -while [ $# -ne 1 ]; do -case "$1" in - -d|--debug) - # Debug Mode - OUTPUT="/tmp/" - GET="echo ${GET}" - UNPACK="echo ${UNPACK}" - shift - ;; - -v|--verbose) - DIRECT_TO_NULL="$(tty)" - shift - ;; - -h|--help) - usage - exit 0 - ;; - -*) - echo "Unknown option: \`$1'" - usage >&2 - exit 1 - ;; - *) - break - ;; -esac -done +unpack_cmd = "dpkg-source -x " +base_url = "http://launchpadlibrarian.net/" -if [ $# -ne 1 ] -then - usage - exit 1 -fi +Debug = Verbose = Quiet = False -## internal functions -getBase() { -echo "Getting ${BASE}/${NUMBER}/${FILE}" -if [ -f "$FILE" ]; then rm -f $FILE; fi # Remove .dsc incase the last run was with debug -if ! wget ${BASE}/${NUMBER}/${FILE} -O ${OUTPUT}${FILE} 2>${DIRECT_TO_NULL} -then - echo "Failed to fetch «.dsc» file, aborting." - exit 1 -fi -} +def Unsign(data): + if data.splitlines()[0] != "-----BEGIN PGP SIGNED MESSAGE-----": + return data + oldstdout = sys.stdout + oldstderr = sys.stderr + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + gpg = GnuPGInterface.GnuPG() + proc = gpg.run(["--decrypt"], create_fhs=['stdin', 'stdout']) + proc.handles['stdin'].write(data) + proc.handles['stdin'].close() + plain = proc.handles['stdout'].read() + proc.handles['stdout'].close() + try: + proc.wait() + except: + pass + sys.stdout = oldstdout + sys.stderr = oldstderr + return plain -getUrl() { -if [ -f "$2" ] -then -##Todo: Check the md5sum in the .dsc to see if we should redownload or not - echo "Skipping already downloaded ${2}" - return 0 -fi +def getEntries(data): + parser = FeedParser.FeedParser() + parser.feed(data) + return parser.close() -echo "Getting ${1}${2}" -if ! $GET ${1}${2} 2>$DIRECT_TO_NULL -then - echo "Failed to fetch «${3}»" - exit 1 -fi -} +class DscParse(object): + """Attempt to get the file list from the .dsc file""" + def __init__(self, data): + """ + __init__(data) + Given the contents of a .dsc, parse it and extract it's content + """ + self.entries = getEntries(Unsign(data)) + self.files = [x.strip().split() for x in self.entries['Files'].splitlines()] -unpack() { -if ! $UNPACK -then - echo "Failed to unpack source, aborting." - exit 1 -fi -} + def verify_all(self): + """ + verify_all() + Verifies all the files, first checking the size, then the md5 sum. + Currently not used in this utility. + """ + assert self.files, "I have no files" + ret = [] + for f in self.files: + ret.append(self.verify(f)) + return ret -## begin #!/bin/bash + def verify(self, name): + """ + verify(name) + Verify the file 'name', first checking the size, then the md5 sum. + """ + assert self.files, "I have no files" + f = None + if isinstance(name, list): + f = name + else: + for i in self.files: + if i[2] == name: + f = i + if not f: + raise ValueError, "%s is not in the .dsc" % name + (sum, size, name) = tuple(f) + stat = os.stat(name) + if str(stat.st_size) != size: + return (False, name, "Expected a size of %s, got %s" % \ + (size, stat.st_size)) + return self.getsum(name, sum) -# Store download URL into a local variable to be able to modify it -URL=$1 + def getsum(self, name, sum=None): + """ + getsum(name[, sum]) + Read the file 'name' (in 1MB chunks) and generate an md5 sum, + then compares that to the md5 sum in the .dsc file. + """ + chunk_size = 1073741824 + fd = open(name, 'rb') + res = md5.new() + if not sum: + assert self.files, "I have no files" + sum = [x[0] for x in self.files if x[2] == name][0] + data = fd.read(chunk_size) + while data: + res.update(data) + data = fd.read(chunk_size) + if res.hexdigest() != sum: + return (False, name, "Expected md5sum of %r, got %r" % \ + (sum, res.hexdigest()) ) + return (True, name, None) -if [ ${URL:0:4} == 'www.' ] -then - URL="http://"${URL:4} -fi + def isNative(self): + """ + isNative() + Returns True if this .dsc describes a native debian package; + else false. + """ + return len(self.files) == 1 -if [ ${URL:0:7} != 'http://' ] -then - URL="http://"$URL -fi + # Access to fields in the .dsc via a dict-like interface + def __getitem__(self, item): + """ + x.__getitem(item) -> x[item] + """ + return self.entries.__getitem__(item) -if [ ${URL:0:30} != 'http://launchpadlibrarian.net/' ] -then - echo "Error: This utility only works for files on http://launchpadlibrarian.net." - exit 1 -fi + def __contains__(self, item): + """ + x.__contains__(item) -> item in x + """ + return self.entries.__contains__(item) -if [ ${URL##*.} != 'dsc' ] -then - echo "You have to provide the URL for the .dsc file." - exit 1 -fi + def __getattr__(self, attr): + """ + x.__getattr__(attr) -> item.attr + """ + return getattr(self.entries, attr) -#BASE="http://$(echo $URL|cut -d '/' -f 3)" #Not needed as we know the base URL -BASE="http://launchpadlibrarian.net" -NUMBER="$(echo $URL|cut -d '/' -f 4)" -FILE="$(echo $URL|cut -d '/' -f 5)" +def error(ret, msg, *args): + """Prints an error message, unless quiet is set, and exits with ret""" + if not Quiet: + print >> sys.stderr, msg % args + sys.exit(ret) -UNPACK=$(echo $UNPACK | sed s/{dsc-file}/${FILE}/g) +def debug(msg, *args): + """If debugging is enabled, print a message""" + if Debug: + print >> sys.stderr, msg % args -if [ -f "$FILE" ]; then rm -f $FILE; fi -getBase; -PkgVersion="$(grep -B100 "^Files:" ${OUTPUT}${FILE}|grep "^Version:"|cut -d' ' -f2)" -PkgName="$(grep -B100 "^Files:" ${OUTPUT}${FILE}|grep "^Source:"|cut -d' ' -f2)" +def info(msg, *args): + """If verbose is enabled, print a message""" + if Verbose: + print msg % tuple(args) -if $(echo ${PkgVersion} | grep '-' >/dev/null) -then - getUrl ${BASE}/$((${NUMBER}-1))/ "$(echo $FILE|sed 's,\.dsc,.diff.gz,')" "diff.gz" - getUrl ${BASE}/$((${NUMBER}-2))/ "${PkgName}_$(echo ${PkgVersion}|sed 's,-[0-9]*[a-z]*[0-9]*,.orig.tar.gz,')" "orig.tar.gz" -else - getUrl ${BASE}/$((${NUMBER}-1))/ "${PkgName}_${PkgVersion}.tar.gz" "tar.gz" -fi +def status(msg, *args): + """Prints a message, unless quiet is enabled""" + if not Quiet: + print msg % tuple(args) -if [ "x${OUTPUT}" != "x" ] -then rm -f ${OUTPUT}${FILE} -fi +def Download(dscinfo, number, filename, verify=True): + """Download filename""" + ftype = filename.endswith(".diff.gz") and "diff.gz" or \ + filename.endswith(".orig.tar.gz") and "orig.tar.gz" or \ + filename.endswith(".dsc") and "dsc" or "tar.gz" + if verify and os.path.exists(filename): + info('Verifying "%s"', filename) + res = dscinfo.verify(filename) + if not res[0]: + error(104, "Verification of %s failed: %s", filename, res[2]) + status("Getting %s", filename) + debug("%s%s/%s", base_url,number,filename) + try: + fd = urllib2.urlopen("%s%s/%s" % (base_url, number, filename)) + outfd = open(filename, 'wb') + outfd.write(fd.read()) + fd.close() + outfd.close() + except urllib2.URLError, e: + status("Failed to fetch «%s» file, aborting.", ftype) + error(105, "Error: %s", e) + except urllib2.HTTPError, e: + status("Failed to fetch «%s» file, aborting.", ftype) + error(106, "Error: (%d %s)", e.code, e.msg) + except IOError, e: + status('Could not create "%s"', filename) + error(107, "Error: %s", e) -unpack; +def unpack(): + out = open('/dev/null', 'w') + err = open('/dev/null', 'w') + ret = subprocess.call(unpack_cmd.split(), stdout=out, stderr=err) + out.close() + err.close() + if ret: + status("Failed to unpack source, aborting.") + sys.exit(108) + +def getHost(url): + return urllib2.splithost(urllib2.splittype(url)[1])[0] + +if __name__ == "__main__": + parser = OptionParser(usage=Usage) + parser.add_option("-d", "--debug", action="store_true", dest="debug", + default=False, help="Enable debugging") + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + default=False, help="Enable verbose output") + parser.add_option("-q", "--quiet", action="store_true", dest="quiet", + default=False, help="Never print any output") + + (options, args) = parser.parse_args() + Debug = options.debug + Verbose = options.verbose + Quiet = options.quiet + if Verbose and Quiet: + error(4, "Specifying both --verbose and --quiet does not make sense") + if Quiet: + sys.stderr = StringIO() + sys.stdout = StringIO() + + url = args[0] + + if url.startswith("https://"): + url = url.replace("https://", "http://", 1) + + if not url.startswith("http://"): + url = "http://" + url + + if getHost(url).startswith("www."): + url = url.replace("www.", "", 1) + + if getHost(url) != getHost(base_url): + error(1, "Error: This utility only works for files on %s.\nMaybe you want to try dget?", base_url) + + (number, filename) = url.split('/')[3:] + + if not filename.endswith('.dsc'): + error(2, "You have to provide the URL for the .dsc file.") + + try: + number = int(number) + except: + error(3, "Bad URL format") + + unpack_cmd += filename + + if os.path.exists(filename): + os.remove(filename) + + Download(None, number, filename, False) + try: + fd = open(filename) + dsc_data = fd.read() + fd.close() + except Exception, e: + status("Error: Please report this bug, providing the URL and attach"\ + " the following backtrace") + raise + + dscinfo = DscParse(dsc_data) + +# launchpadlibrarian.net seems to store in this order: +# For native packages: +# /.changes +# +1/.tar.gz +# +2/.dsc +# For non-native packages: +# /.changes +# +1/.orig.tar.gz +# +2/.diff.gz +# +3/.dsc +## +# *Assuming* this does not change, we can figure out where the files are on +# launchpadlibrarian.net relative to the .dsc file we're given. + +# Only one file listed in the .dsc means it's native package + if len(dscinfo.files) == 1: + Download(dscinfo, number-1, dscinfo.files[0][-1]) # .tar.gz + else: + Download(dscinfo, number-1, dscinfo.files[1][-1]) # .diff.gz + Download(dscinfo, number-2, dscinfo.files[0][-1]) # .orig.tar.gz + + status("Unpacking") + unpack()