mirror of
https://git.launchpad.net/ubuntu-dev-tools
synced 2025-03-13 08:01:09 +00:00
Added Terence's new dgetlp script.
This commit is contained in:
Jonathan Davies
parent
ab7001a5a6
commit
14ebb357cc
3
debian/changelog
vendored
3
debian/changelog
vendored
@ -1,6 +1,7 @@
|
||||
ubuntu-dev-tools (0.61) UNRELEASED; urgency=low
|
||||
|
||||
* Changes go here.
|
||||
[ Terence Simpson ]
|
||||
* dgetlp: Replaced Bash version with a new Python script.
|
||||
|
||||
-- Jonathan Davies <jpds@ubuntu.com> Sun, 01 Feb 2009 10:57:51 +0000
|
||||
|
||||
|
||||
409
dgetlp
409
dgetlp
@ -1,22 +1,20 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/python
|
||||
# -*- coding: UTF-8 -*-
|
||||
# Copyright (C) 2008 Terence Simpson <tsimpson@ubuntu.com>
|
||||
# Modified by Siegfried-A. Gevatter <rainct@ubuntu.com>
|
||||
# License:
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# ##################################################################
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# See file /usr/share/common-licenses/GPL for more details.
|
||||
#
|
||||
# ##################################################################
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# This script simulates «dget»'s behaviour for files hosted at
|
||||
# launchpadlibrarian.net.
|
||||
@ -26,16 +24,19 @@
|
||||
# way as dget does, but from launchpadlibrarian.net, which doesn't
|
||||
# store all the files in the same directory. It (the script) assumes
|
||||
# that the files are stored in sequential directories on Launchpad
|
||||
# Librarian and attemps to download and then unpack them.
|
||||
# Librarian and attempts to download and then unpack them.
|
||||
# This is a Python rewrite of the original bash script
|
||||
|
||||
GET="wget"
|
||||
UNPACK="dpkg-source -x {dsc-file}"
|
||||
DIRECT_TO_NULL="/dev/null"
|
||||
import sys, os
|
||||
from optparse import OptionParser
|
||||
import urllib2
|
||||
import md5
|
||||
import subprocess
|
||||
import GnuPGInterface
|
||||
from cStringIO import StringIO
|
||||
from email import FeedParser
|
||||
|
||||
usage()
|
||||
{
|
||||
cat << EOF
|
||||
Usage: $0 [-d] <Launchpad URL>
|
||||
Usage = u"""Usage: %prog [-d|(-v|-q)] <Launchpad URL>
|
||||
|
||||
This scripts simulates «dget»'s behaviour for files hosted at
|
||||
launchpadlibrarian.net.
|
||||
@ -44,128 +45,274 @@ If you specify the -d option then it won't do anything, except download the
|
||||
.dsc file, but just print the commands it would run otherwise.
|
||||
|
||||
Example:
|
||||
$(basename $0) http://launchpadlibrarian.net/10348157/coreutils_5.97-5.4ubuntu1.dsc
|
||||
EOF
|
||||
}
|
||||
%prog http://launchpadlibrarian.net/10348157/coreutils_5.97-5.4ubuntu1.dsc
|
||||
"""
|
||||
|
||||
while [ $# -ne 1 ]; do
|
||||
case "$1" in
|
||||
-d|--debug)
|
||||
# Debug Mode
|
||||
OUTPUT="/tmp/"
|
||||
GET="echo ${GET}"
|
||||
UNPACK="echo ${UNPACK}"
|
||||
shift
|
||||
;;
|
||||
-v|--verbose)
|
||||
DIRECT_TO_NULL="$(tty)"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "Unknown option: \`$1'"
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
unpack_cmd = "dpkg-source -x "
|
||||
base_url = "http://launchpadlibrarian.net/"
|
||||
|
||||
if [ $# -ne 1 ]
|
||||
then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
Debug = Verbose = Quiet = False
|
||||
|
||||
## internal functions
|
||||
getBase() {
|
||||
echo "Getting ${BASE}/${NUMBER}/${FILE}"
|
||||
if [ -f "$FILE" ]; then rm -f $FILE; fi # Remove .dsc incase the last run was with debug
|
||||
if ! wget ${BASE}/${NUMBER}/${FILE} -O ${OUTPUT}${FILE} 2>${DIRECT_TO_NULL}
|
||||
then
|
||||
echo "Failed to fetch «.dsc» file, aborting."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
def Unsign(data):
|
||||
if data.splitlines()[0] != "-----BEGIN PGP SIGNED MESSAGE-----":
|
||||
return data
|
||||
oldstdout = sys.stdout
|
||||
oldstderr = sys.stderr
|
||||
sys.stdout = sys.__stdout__
|
||||
sys.stderr = sys.__stderr__
|
||||
gpg = GnuPGInterface.GnuPG()
|
||||
proc = gpg.run(["--decrypt"], create_fhs=['stdin', 'stdout'])
|
||||
proc.handles['stdin'].write(data)
|
||||
proc.handles['stdin'].close()
|
||||
plain = proc.handles['stdout'].read()
|
||||
proc.handles['stdout'].close()
|
||||
try:
|
||||
proc.wait()
|
||||
except:
|
||||
pass
|
||||
sys.stdout = oldstdout
|
||||
sys.stderr = oldstderr
|
||||
return plain
|
||||
|
||||
getUrl() {
|
||||
if [ -f "$2" ]
|
||||
then
|
||||
##Todo: Check the md5sum in the .dsc to see if we should redownload or not
|
||||
echo "Skipping already downloaded ${2}"
|
||||
return 0
|
||||
fi
|
||||
def getEntries(data):
|
||||
parser = FeedParser.FeedParser()
|
||||
parser.feed(data)
|
||||
return parser.close()
|
||||
|
||||
echo "Getting ${1}${2}"
|
||||
if ! $GET ${1}${2} 2>$DIRECT_TO_NULL
|
||||
then
|
||||
echo "Failed to fetch «${3}»"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
class DscParse(object):
|
||||
"""Attempt to get the file list from the .dsc file"""
|
||||
def __init__(self, data):
|
||||
"""
|
||||
__init__(data)
|
||||
Given the contents of a .dsc, parse it and extract it's content
|
||||
"""
|
||||
self.entries = getEntries(Unsign(data))
|
||||
self.files = [x.strip().split() for x in self.entries['Files'].splitlines()]
|
||||
|
||||
unpack() {
|
||||
if ! $UNPACK
|
||||
then
|
||||
echo "Failed to unpack source, aborting."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
def verify_all(self):
|
||||
"""
|
||||
verify_all()
|
||||
Verifies all the files, first checking the size, then the md5 sum.
|
||||
Currently not used in this utility.
|
||||
"""
|
||||
assert self.files, "I have no files"
|
||||
ret = []
|
||||
for f in self.files:
|
||||
ret.append(self.verify(f))
|
||||
return ret
|
||||
|
||||
## begin #!/bin/bash
|
||||
def verify(self, name):
|
||||
"""
|
||||
verify(name)
|
||||
Verify the file 'name', first checking the size, then the md5 sum.
|
||||
"""
|
||||
assert self.files, "I have no files"
|
||||
f = None
|
||||
if isinstance(name, list):
|
||||
f = name
|
||||
else:
|
||||
for i in self.files:
|
||||
if i[2] == name:
|
||||
f = i
|
||||
if not f:
|
||||
raise ValueError, "%s is not in the .dsc" % name
|
||||
(sum, size, name) = tuple(f)
|
||||
stat = os.stat(name)
|
||||
if str(stat.st_size) != size:
|
||||
return (False, name, "Expected a size of %s, got %s" % \
|
||||
(size, stat.st_size))
|
||||
return self.getsum(name, sum)
|
||||
|
||||
# Store download URL into a local variable to be able to modify it
|
||||
URL=$1
|
||||
def getsum(self, name, sum=None):
|
||||
"""
|
||||
getsum(name[, sum])
|
||||
Read the file 'name' (in 1MB chunks) and generate an md5 sum,
|
||||
then compares that to the md5 sum in the .dsc file.
|
||||
"""
|
||||
chunk_size = 1073741824
|
||||
fd = open(name, 'rb')
|
||||
res = md5.new()
|
||||
if not sum:
|
||||
assert self.files, "I have no files"
|
||||
sum = [x[0] for x in self.files if x[2] == name][0]
|
||||
data = fd.read(chunk_size)
|
||||
while data:
|
||||
res.update(data)
|
||||
data = fd.read(chunk_size)
|
||||
if res.hexdigest() != sum:
|
||||
return (False, name, "Expected md5sum of %r, got %r" % \
|
||||
(sum, res.hexdigest()) )
|
||||
return (True, name, None)
|
||||
|
||||
if [ ${URL:0:4} == 'www.' ]
|
||||
then
|
||||
URL="http://"${URL:4}
|
||||
fi
|
||||
def isNative(self):
|
||||
"""
|
||||
isNative()
|
||||
Returns True if this .dsc describes a native debian package;
|
||||
else false.
|
||||
"""
|
||||
return len(self.files) == 1
|
||||
|
||||
if [ ${URL:0:7} != 'http://' ]
|
||||
then
|
||||
URL="http://"$URL
|
||||
fi
|
||||
# Access to fields in the .dsc via a dict-like interface
|
||||
def __getitem__(self, item):
|
||||
"""
|
||||
x.__getitem(item) -> x[item]
|
||||
"""
|
||||
return self.entries.__getitem__(item)
|
||||
|
||||
if [ ${URL:0:30} != 'http://launchpadlibrarian.net/' ]
|
||||
then
|
||||
echo "Error: This utility only works for files on http://launchpadlibrarian.net."
|
||||
exit 1
|
||||
fi
|
||||
def __contains__(self, item):
|
||||
"""
|
||||
x.__contains__(item) -> item in x
|
||||
"""
|
||||
return self.entries.__contains__(item)
|
||||
|
||||
if [ ${URL##*.} != 'dsc' ]
|
||||
then
|
||||
echo "You have to provide the URL for the .dsc file."
|
||||
exit 1
|
||||
fi
|
||||
def __getattr__(self, attr):
|
||||
"""
|
||||
x.__getattr__(attr) -> item.attr
|
||||
"""
|
||||
return getattr(self.entries, attr)
|
||||
|
||||
#BASE="http://$(echo $URL|cut -d '/' -f 3)" #Not needed as we know the base URL
|
||||
BASE="http://launchpadlibrarian.net"
|
||||
NUMBER="$(echo $URL|cut -d '/' -f 4)"
|
||||
FILE="$(echo $URL|cut -d '/' -f 5)"
|
||||
def error(ret, msg, *args):
|
||||
"""Prints an error message, unless quiet is set, and exits with ret"""
|
||||
if not Quiet:
|
||||
print >> sys.stderr, msg % args
|
||||
sys.exit(ret)
|
||||
|
||||
UNPACK=$(echo $UNPACK | sed s/{dsc-file}/${FILE}/g)
|
||||
def debug(msg, *args):
|
||||
"""If debugging is enabled, print a message"""
|
||||
if Debug:
|
||||
print >> sys.stderr, msg % args
|
||||
|
||||
if [ -f "$FILE" ]; then rm -f $FILE; fi
|
||||
getBase;
|
||||
PkgVersion="$(grep -B100 "^Files:" ${OUTPUT}${FILE}|grep "^Version:"|cut -d' ' -f2)"
|
||||
PkgName="$(grep -B100 "^Files:" ${OUTPUT}${FILE}|grep "^Source:"|cut -d' ' -f2)"
|
||||
def info(msg, *args):
|
||||
"""If verbose is enabled, print a message"""
|
||||
if Verbose:
|
||||
print msg % tuple(args)
|
||||
|
||||
if $(echo ${PkgVersion} | grep '-' >/dev/null)
|
||||
then
|
||||
getUrl ${BASE}/$((${NUMBER}-1))/ "$(echo $FILE|sed 's,\.dsc,.diff.gz,')" "diff.gz"
|
||||
getUrl ${BASE}/$((${NUMBER}-2))/ "${PkgName}_$(echo ${PkgVersion}|sed 's,-[0-9]*[a-z]*[0-9]*,.orig.tar.gz,')" "orig.tar.gz"
|
||||
else
|
||||
getUrl ${BASE}/$((${NUMBER}-1))/ "${PkgName}_${PkgVersion}.tar.gz" "tar.gz"
|
||||
fi
|
||||
def status(msg, *args):
|
||||
"""Prints a message, unless quiet is enabled"""
|
||||
if not Quiet:
|
||||
print msg % tuple(args)
|
||||
|
||||
if [ "x${OUTPUT}" != "x" ]
|
||||
then rm -f ${OUTPUT}${FILE}
|
||||
fi
|
||||
def Download(dscinfo, number, filename, verify=True):
|
||||
"""Download filename"""
|
||||
ftype = filename.endswith(".diff.gz") and "diff.gz" or \
|
||||
filename.endswith(".orig.tar.gz") and "orig.tar.gz" or \
|
||||
filename.endswith(".dsc") and "dsc" or "tar.gz"
|
||||
if verify and os.path.exists(filename):
|
||||
info('Verifying "%s"', filename)
|
||||
res = dscinfo.verify(filename)
|
||||
if not res[0]:
|
||||
error(104, "Verification of %s failed: %s", filename, res[2])
|
||||
status("Getting %s", filename)
|
||||
debug("%s%s/%s", base_url,number,filename)
|
||||
try:
|
||||
fd = urllib2.urlopen("%s%s/%s" % (base_url, number, filename))
|
||||
outfd = open(filename, 'wb')
|
||||
outfd.write(fd.read())
|
||||
fd.close()
|
||||
outfd.close()
|
||||
except urllib2.URLError, e:
|
||||
status("Failed to fetch «%s» file, aborting.", ftype)
|
||||
error(105, "Error: %s", e)
|
||||
except urllib2.HTTPError, e:
|
||||
status("Failed to fetch «%s» file, aborting.", ftype)
|
||||
error(106, "Error: (%d %s)", e.code, e.msg)
|
||||
except IOError, e:
|
||||
status('Could not create "%s"', filename)
|
||||
error(107, "Error: %s", e)
|
||||
|
||||
unpack;
|
||||
def unpack():
|
||||
out = open('/dev/null', 'w')
|
||||
err = open('/dev/null', 'w')
|
||||
ret = subprocess.call(unpack_cmd.split(), stdout=out, stderr=err)
|
||||
out.close()
|
||||
err.close()
|
||||
if ret:
|
||||
status("Failed to unpack source, aborting.")
|
||||
sys.exit(108)
|
||||
|
||||
def getHost(url):
|
||||
return urllib2.splithost(urllib2.splittype(url)[1])[0]
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = OptionParser(usage=Usage)
|
||||
parser.add_option("-d", "--debug", action="store_true", dest="debug",
|
||||
default=False, help="Enable debugging")
|
||||
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
|
||||
default=False, help="Enable verbose output")
|
||||
parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
|
||||
default=False, help="Never print any output")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
Debug = options.debug
|
||||
Verbose = options.verbose
|
||||
Quiet = options.quiet
|
||||
if Verbose and Quiet:
|
||||
error(4, "Specifying both --verbose and --quiet does not make sense")
|
||||
if Quiet:
|
||||
sys.stderr = StringIO()
|
||||
sys.stdout = StringIO()
|
||||
|
||||
url = args[0]
|
||||
|
||||
if url.startswith("https://"):
|
||||
url = url.replace("https://", "http://", 1)
|
||||
|
||||
if not url.startswith("http://"):
|
||||
url = "http://" + url
|
||||
|
||||
if getHost(url).startswith("www."):
|
||||
url = url.replace("www.", "", 1)
|
||||
|
||||
if getHost(url) != getHost(base_url):
|
||||
error(1, "Error: This utility only works for files on %s.\nMaybe you want to try dget?", base_url)
|
||||
|
||||
(number, filename) = url.split('/')[3:]
|
||||
|
||||
if not filename.endswith('.dsc'):
|
||||
error(2, "You have to provide the URL for the .dsc file.")
|
||||
|
||||
try:
|
||||
number = int(number)
|
||||
except:
|
||||
error(3, "Bad URL format")
|
||||
|
||||
unpack_cmd += filename
|
||||
|
||||
if os.path.exists(filename):
|
||||
os.remove(filename)
|
||||
|
||||
Download(None, number, filename, False)
|
||||
try:
|
||||
fd = open(filename)
|
||||
dsc_data = fd.read()
|
||||
fd.close()
|
||||
except Exception, e:
|
||||
status("Error: Please report this bug, providing the URL and attach"\
|
||||
" the following backtrace")
|
||||
raise
|
||||
|
||||
dscinfo = DscParse(dsc_data)
|
||||
|
||||
# launchpadlibrarian.net seems to store in this order:
|
||||
# For native packages:
|
||||
# <number>/.changes
|
||||
# <number>+1/.tar.gz
|
||||
# <number>+2/.dsc
|
||||
# For non-native packages:
|
||||
# <number>/.changes
|
||||
# <number>+1/.orig.tar.gz
|
||||
# <number>+2/.diff.gz
|
||||
# <number>+3/.dsc
|
||||
##
|
||||
# *Assuming* this does not change, we can figure out where the files are on
|
||||
# launchpadlibrarian.net relative to the .dsc file we're given.
|
||||
|
||||
# Only one file listed in the .dsc means it's native package
|
||||
if len(dscinfo.files) == 1:
|
||||
Download(dscinfo, number-1, dscinfo.files[0][-1]) # .tar.gz
|
||||
else:
|
||||
Download(dscinfo, number-1, dscinfo.files[1][-1]) # .diff.gz
|
||||
Download(dscinfo, number-2, dscinfo.files[0][-1]) # .orig.tar.gz
|
||||
|
||||
status("Unpacking")
|
||||
unpack()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user