Added Terence's new dgetlp script.

debian/changelog

@@ -1,6 +1,7 @@
 ubuntu-dev-tools (0.61) UNRELEASED; urgency=low
 
-  * Changes go here.
+  [ Terence Simpson ]
+  * dgetlp: Replaced Bash version with a new Python script.
 
  -- Jonathan Davies <jpds@ubuntu.com>  Sun, 01 Feb 2009 10:57:51 +0000
 

dgetlp

@@ -1,22 +1,20 @@
-#!/bin/bash
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
 # Copyright (C) 2008 Terence Simpson <tsimpson@ubuntu.com>
-# Modified by Siegfried-A. Gevatter <rainct@ubuntu.com>
+# License:
-#
+#  This program is free software; you can redistribute it and/or modify
-# ##################################################################
+#  it under the terms of the GNU General Public License as published by
-#
+#  the Free Software Foundation; either version 2 of the License, or
-# This program is free software; you can redistribute it and/or
+#  (at your option) any later version.
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
 #
 #  This program is distributed in the hope that it will be useful,
 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #  GNU General Public License for more details.
 #
-# See file /usr/share/common-licenses/GPL for more details.
+#  You should have received a copy of the GNU General Public License along
-#
+#  with this program; if not, write to the Free Software Foundation, Inc.,
-# ##################################################################
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # This script simulates «dget»'s behaviour for files hosted at
 # launchpadlibrarian.net.
@@ -26,16 +24,19 @@
 # way as dget does, but from launchpadlibrarian.net, which doesn't
 # store all the files in the same directory. It (the script) assumes
 # that the files are stored in sequential directories on Launchpad
-# Librarian and attemps to download and then unpack them.
+# Librarian and attempts to download and then unpack them.
+# This is a Python rewrite of the original bash script
 
-GET="wget"
+import sys, os
-UNPACK="dpkg-source -x {dsc-file}"
+from optparse import OptionParser
-DIRECT_TO_NULL="/dev/null"
+import urllib2
+import md5
+import subprocess
+import GnuPGInterface
+from cStringIO import StringIO
+from email import FeedParser
 
-usage()
+Usage = u"""Usage: %prog [-d|(-v|-q)] <Launchpad URL>
-{
-cat << EOF
-Usage: $0 [-d] <Launchpad URL>
 
 This scripts simulates «dget»'s behaviour for files hosted at
 launchpadlibrarian.net.
@@ -44,128 +45,274 @@ If you specify the -d option then it won't do anything, except download the
 .dsc file, but just print the commands it would run otherwise.
 
 Example:
-	$(basename $0) http://launchpadlibrarian.net/10348157/coreutils_5.97-5.4ubuntu1.dsc
+	%prog http://launchpadlibrarian.net/10348157/coreutils_5.97-5.4ubuntu1.dsc
-EOF
+"""
-}
 
-while [ $# -ne 1 ]; do
+unpack_cmd = "dpkg-source -x "
-case "$1" in
+base_url = "http://launchpadlibrarian.net/"
-	-d|--debug)
-	  # Debug Mode
-	  OUTPUT="/tmp/"
-	  GET="echo ${GET}"
-	  UNPACK="echo ${UNPACK}"
-	  shift
-	;;
-	-v|--verbose)
-	  DIRECT_TO_NULL="$(tty)"
-	  shift
-	;;
-	-h|--help)
-	  usage
-	  exit 0
-	;;
-	-*)
-	  echo "Unknown option: \`$1'"
-	  usage >&2
-	  exit 1
-	;;
-	*)
-	  break
-	;;
-esac
-done
 
-if [ $# -ne 1 ]
+Debug = Verbose = Quiet = False
-then
-	usage
-	exit 1
-fi
 
-## internal functions
+def Unsign(data):
-getBase() {
+    if data.splitlines()[0] != "-----BEGIN PGP SIGNED MESSAGE-----":
-echo "Getting ${BASE}/${NUMBER}/${FILE}"
+        return data
-if [ -f "$FILE" ]; then rm -f $FILE; fi # Remove .dsc incase the last run was with debug
+    oldstdout = sys.stdout
-if ! wget ${BASE}/${NUMBER}/${FILE} -O ${OUTPUT}${FILE} 2>${DIRECT_TO_NULL}
+    oldstderr = sys.stderr
-then
+    sys.stdout = sys.__stdout__
-	echo "Failed to fetch «.dsc» file, aborting."
+    sys.stderr = sys.__stderr__
-	exit 1
+    gpg = GnuPGInterface.GnuPG()
-fi
+    proc = gpg.run(["--decrypt"], create_fhs=['stdin', 'stdout'])
-}
+    proc.handles['stdin'].write(data)
+    proc.handles['stdin'].close()
+    plain = proc.handles['stdout'].read()
+    proc.handles['stdout'].close()
+    try:
+        proc.wait()
+    except:
+        pass
+    sys.stdout = oldstdout
+    sys.stderr = oldstderr
+    return plain
 
-getUrl() {
+def getEntries(data):
-if [ -f "$2" ]
+    parser = FeedParser.FeedParser()
-then
+    parser.feed(data)
-##Todo: Check the md5sum in the .dsc to see if we should redownload or not
+    return parser.close()
-	echo "Skipping already downloaded ${2}"
-	return 0
-fi
 
-echo "Getting ${1}${2}"
+class DscParse(object):
-if ! $GET ${1}${2} 2>$DIRECT_TO_NULL
+    """Attempt to get the file list from the .dsc file"""
-then
+    def __init__(self, data):
-	echo "Failed to fetch «${3}»"
+        """
-	exit 1
+        __init__(data)
-fi
+        Given the contents of a .dsc, parse it and extract it's content
-}
+        """
+        self.entries = getEntries(Unsign(data))
+        self.files = [x.strip().split() for x in self.entries['Files'].splitlines()]
 
-unpack() {
+    def verify_all(self):
-if ! $UNPACK
+        """
-then
+        verify_all()
-	echo "Failed to unpack source, aborting."
+        Verifies all the files, first checking the size, then the md5 sum.
-	exit 1
+        Currently not used in this utility.
-fi
+        """
-}
+        assert self.files, "I have no files"
+        ret = []
+        for f in self.files:
+            ret.append(self.verify(f))
+        return ret
 
-## begin #!/bin/bash
+    def verify(self, name):
+        """
+        verify(name)
+        Verify the file 'name', first checking the size, then the md5 sum.
+        """
+        assert self.files, "I have no files"
+        f = None
+        if isinstance(name, list):
+            f = name
+        else:
+            for i in self.files:
+                if i[2] == name:
+                    f = i
+        if not f:
+            raise ValueError, "%s is not in the .dsc" % name
+        (sum, size, name) = tuple(f)
+        stat = os.stat(name)
+        if str(stat.st_size) != size:
+            return (False, name, "Expected a size of %s, got %s" % \
+                (size, stat.st_size))
+        return self.getsum(name, sum)
 
-# Store download URL into a local variable to be able to modify it
+    def getsum(self, name, sum=None):
-URL=$1
+        """
+        getsum(name[, sum])
+        Read the file 'name' (in 1MB chunks) and generate an md5 sum,
+        then compares that to the md5 sum in the .dsc file.
+        """
+        chunk_size = 1073741824
+        fd = open(name, 'rb')
+        res = md5.new()
+        if not sum:
+            assert self.files, "I have no files"
+            sum = [x[0] for x in self.files if x[2] == name][0]
+        data = fd.read(chunk_size)
+        while data:
+            res.update(data)
+            data = fd.read(chunk_size)
+        if res.hexdigest() != sum:
+            return (False, name, "Expected md5sum of %r, got %r" % \
+                (sum, res.hexdigest()) )
+        return (True, name, None)
 
-if [ ${URL:0:4} == 'www.' ]
+    def isNative(self):
-then
+        """
-	URL="http://"${URL:4}
+        isNative()
-fi
+        Returns True if this .dsc describes a native debian package;
+        else false.
+        """
+        return len(self.files) == 1
 
-if [ ${URL:0:7} != 'http://' ]
+    # Access to fields in the .dsc via a dict-like interface
-then
+    def __getitem__(self, item):
-	URL="http://"$URL
+        """
-fi
+        x.__getitem(item) -> x[item]
+        """
+        return self.entries.__getitem__(item)
 
-if [ ${URL:0:30} != 'http://launchpadlibrarian.net/' ]
+    def __contains__(self, item):
-then
+        """
-	echo "Error: This utility only works for files on http://launchpadlibrarian.net."
+        x.__contains__(item) -> item in x
-	exit 1
+        """
-fi
+        return self.entries.__contains__(item)
 
-if [ ${URL##*.} !=  'dsc' ]
+    def __getattr__(self, attr):
-then
+        """
-	echo "You have to provide the URL for the .dsc file."
+        x.__getattr__(attr) -> item.attr
-	exit 1
+        """
-fi
+        return getattr(self.entries, attr)
 
-#BASE="http://$(echo $URL|cut -d '/' -f 3)" #Not needed as we know the base URL
+def error(ret, msg, *args):
-BASE="http://launchpadlibrarian.net"
+    """Prints an error message, unless quiet is set, and exits with ret"""
-NUMBER="$(echo $URL|cut -d '/' -f 4)"
+    if not Quiet:
-FILE="$(echo $URL|cut -d '/' -f 5)"
+        print >> sys.stderr, msg % args
+    sys.exit(ret)
 
-UNPACK=$(echo $UNPACK | sed s/{dsc-file}/${FILE}/g)
+def debug(msg, *args):
+    """If debugging is enabled, print a message"""
+    if Debug:
+        print >> sys.stderr, msg % args
 
-if [ -f "$FILE" ]; then rm -f $FILE; fi
+def info(msg, *args):
-getBase;
+    """If verbose is enabled, print a message"""
-PkgVersion="$(grep -B100 "^Files:" ${OUTPUT}${FILE}|grep "^Version:"|cut -d' ' -f2)"
+    if Verbose:
-PkgName="$(grep -B100 "^Files:" ${OUTPUT}${FILE}|grep "^Source:"|cut -d' ' -f2)"
+        print msg % tuple(args)
 
-if $(echo ${PkgVersion} | grep '-' >/dev/null)
+def status(msg, *args):
-then
+    """Prints a message, unless quiet is enabled"""
-	getUrl ${BASE}/$((${NUMBER}-1))/ "$(echo $FILE|sed 's,\.dsc,.diff.gz,')" "diff.gz"
+    if not Quiet:
-	getUrl ${BASE}/$((${NUMBER}-2))/ "${PkgName}_$(echo ${PkgVersion}|sed 's,-[0-9]*[a-z]*[0-9]*,.orig.tar.gz,')" "orig.tar.gz"
+        print msg % tuple(args)
-else
-	getUrl ${BASE}/$((${NUMBER}-1))/ "${PkgName}_${PkgVersion}.tar.gz" "tar.gz"
-fi
 
-if [ "x${OUTPUT}" != "x" ]
+def Download(dscinfo, number, filename, verify=True):
-then rm -f ${OUTPUT}${FILE}
+    """Download filename"""
-fi
+    ftype = filename.endswith(".diff.gz") and "diff.gz" or \
+        filename.endswith(".orig.tar.gz") and "orig.tar.gz" or \
+        filename.endswith(".dsc") and "dsc" or "tar.gz"
+    if verify and os.path.exists(filename):
+        info('Verifying "%s"', filename)
+        res = dscinfo.verify(filename)
+        if not res[0]:
+            error(104, "Verification of %s failed: %s", filename, res[2])
+    status("Getting %s", filename)
+    debug("%s%s/%s", base_url,number,filename)
+    try:
+        fd = urllib2.urlopen("%s%s/%s" % (base_url, number, filename))
+        outfd = open(filename, 'wb')
+        outfd.write(fd.read())
+        fd.close()
+        outfd.close()
+    except urllib2.URLError, e:
+        status("Failed to fetch «%s» file, aborting.", ftype)
+        error(105, "Error: %s", e)
+    except urllib2.HTTPError, e:
+        status("Failed to fetch «%s» file, aborting.", ftype)
+        error(106, "Error: (%d %s)", e.code, e.msg)
+    except IOError, e:
+        status('Could not create "%s"', filename)
+        error(107, "Error: %s", e)
 
-unpack;
+def unpack():
+    out = open('/dev/null', 'w')
+    err = open('/dev/null', 'w')
+    ret = subprocess.call(unpack_cmd.split(), stdout=out, stderr=err)
+    out.close()
+    err.close()
+    if ret:
+        status("Failed to unpack source, aborting.")
+        sys.exit(108)
+
+def getHost(url):
+    return urllib2.splithost(urllib2.splittype(url)[1])[0]
+
+if __name__ == "__main__":
+    parser = OptionParser(usage=Usage)
+    parser.add_option("-d", "--debug", action="store_true", dest="debug",
+        default=False, help="Enable debugging")
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
+        default=False, help="Enable verbose output")
+    parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
+        default=False, help="Never print any output")
+
+    (options, args) = parser.parse_args()
+    Debug = options.debug
+    Verbose = options.verbose
+    Quiet = options.quiet
+    if Verbose and Quiet:
+        error(4, "Specifying both --verbose and --quiet does not make sense")
+    if Quiet:
+        sys.stderr = StringIO()
+        sys.stdout = StringIO()
+
+    url = args[0]
+
+    if url.startswith("https://"):
+        url = url.replace("https://", "http://", 1)
+
+    if not url.startswith("http://"):
+        url = "http://" + url
+
+    if getHost(url).startswith("www."):
+        url = url.replace("www.", "", 1)
+
+    if getHost(url) != getHost(base_url):
+        error(1, "Error: This utility only works for files on %s.\nMaybe you want to try dget?", base_url)
+
+    (number, filename) = url.split('/')[3:]
+
+    if not filename.endswith('.dsc'):
+        error(2, "You have to provide the URL for the .dsc file.")
+
+    try:
+        number = int(number)
+    except:
+        error(3, "Bad URL format")
+
+    unpack_cmd += filename
+
+    if os.path.exists(filename):
+        os.remove(filename)
+
+    Download(None, number, filename, False)
+    try:
+        fd = open(filename)
+        dsc_data = fd.read()
+        fd.close()
+    except Exception, e:
+        status("Error: Please report this bug, providing the URL and attach"\
+               " the following backtrace")
+        raise
+
+    dscinfo = DscParse(dsc_data)
+
+# launchpadlibrarian.net seems to store in this order:
+# For native packages:
+# <number>/.changes
+# <number>+1/.tar.gz
+# <number>+2/.dsc
+# For non-native packages:
+# <number>/.changes
+# <number>+1/.orig.tar.gz
+# <number>+2/.diff.gz
+# <number>+3/.dsc
+##
+# *Assuming* this does not change, we can figure out where the files are on
+# launchpadlibrarian.net relative to the .dsc file we're given.
+
+# Only one file listed in the .dsc means it's native package
+    if len(dscinfo.files) == 1:
+        Download(dscinfo, number-1, dscinfo.files[0][-1]) # .tar.gz
+    else:
+        Download(dscinfo, number-1, dscinfo.files[1][-1]) # .diff.gz
+        Download(dscinfo, number-2, dscinfo.files[0][-1]) # .orig.tar.gz
+
+    status("Unpacking")
+    unpack()