mirror of
https://git.launchpad.net/ubuntu-dev-tools
synced 2025-03-12 15:41:09 +00:00
Move common downloading code into new ubuntutools.mirrors. Use in backportpackage, pull-debian-debdiff, pull-lp-source
This commit is contained in:
Stefano Rivera
parent
f7525d6dd5
commit
2c27cf68a6
@ -24,7 +24,6 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib
|
||||
|
||||
from debian.deb822 import Dsc
|
||||
from launchpadlib.launchpad import Launchpad
|
||||
@ -33,8 +32,8 @@ import lsb_release
|
||||
from ubuntutools.config import UDTConfig, ubu_email
|
||||
from ubuntutools.builder import get_builder
|
||||
from ubuntutools.logger import Logger
|
||||
from ubuntutools.mirrors import dsc_name, pull_source_pkg
|
||||
from ubuntutools.question import YesNoQuestion
|
||||
from ubuntutools.misc import dsc_url
|
||||
|
||||
def error(msg):
|
||||
Logger.error(msg)
|
||||
@ -179,7 +178,17 @@ def find_version_package(launchpad, package, version):
|
||||
error('Version %s of package %s was never published in Ubuntu.' %
|
||||
(version, package))
|
||||
|
||||
def dscurls_from_package(launchpad, mirror, package, version, source_release):
|
||||
def fetch_package(launchpad, mirror, workdir, package, version, source_release):
|
||||
"Returns the path to the .dsc file that was fetched"
|
||||
|
||||
if package.endswith('.dsc'):
|
||||
cmd = ('dget', '--download-only', '--allow-unauthenticated', package)
|
||||
Logger.command(cmd)
|
||||
ret = subprocess.call(cmd, cwd=workdir)
|
||||
if ret == 0:
|
||||
return os.path.join(workdir, os.path.basename(package))
|
||||
sys.exit(1)
|
||||
|
||||
if not source_release and not version:
|
||||
source_release = launchpad.distributions['ubuntu'].current_series.name
|
||||
|
||||
@ -190,40 +199,10 @@ def dscurls_from_package(launchpad, mirror, package, version, source_release):
|
||||
else:
|
||||
srcpkg = find_version_package(launchpad, package, version)
|
||||
|
||||
urls = []
|
||||
if mirror:
|
||||
urls.append(dsc_url(mirror, srcpkg.component_name, package,
|
||||
srcpkg.source_package_version))
|
||||
|
||||
for source_file in srcpkg.sourceFileUrls():
|
||||
if source_file.endswith('.dsc'):
|
||||
urls.append(urllib.unquote(source_file))
|
||||
return urls
|
||||
else:
|
||||
error('Package %s contains no .dsc file.' % package)
|
||||
|
||||
def dscurl_from_dsc(package):
|
||||
path = os.path.abspath(os.path.expanduser(package))
|
||||
if os.path.exists(path):
|
||||
return 'file://%s' % path
|
||||
else:
|
||||
# Can't resolve it as a local path? Let's just hope it's good as-is
|
||||
return package
|
||||
|
||||
def fetch_package(launchpad, mirror, workdir, package, version, source_release):
|
||||
# Returns the path to the .dsc file that was fetched
|
||||
if package.endswith('.dsc'):
|
||||
dscs = [dscurl_from_dsc(package)]
|
||||
else:
|
||||
dscs = dscurls_from_package(launchpad, mirror, package, version,
|
||||
source_release)
|
||||
|
||||
for dsc in dscs:
|
||||
cmd = ('dget', '--download-only', '--allow-unauthenticated', dsc)
|
||||
Logger.command(cmd)
|
||||
ret = subprocess.call(cmd, cwd=workdir)
|
||||
if ret == 0:
|
||||
return os.path.join(workdir, os.path.basename(dsc))
|
||||
version = srcpkg.source_package_version
|
||||
pull_source_pkg('UBUNTU', mirror, srcpkg.component_name, package, version,
|
||||
workdir=workdir, unpack=False)
|
||||
return dsc_name(package, version)
|
||||
|
||||
def get_backport_version(version, suffix, upload, release):
|
||||
backport_version = version + ('~%s1' % release)
|
||||
|
||||
1
debian/copyright
vendored
1
debian/copyright
vendored
@ -187,6 +187,7 @@ Files: doc/pull-debian-debdiff.1,
|
||||
ubuntutools/config.py,
|
||||
ubuntutools/control.py,
|
||||
ubuntutools/logger.py,
|
||||
ubuntutools/mirrors.py,
|
||||
ubuntutools/question.py,
|
||||
ubuntutools/sponsor_patch/*,
|
||||
ubuntutools/test/*,
|
||||
|
||||
@ -17,112 +17,15 @@
|
||||
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
# PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
import hashlib
|
||||
import optparse
|
||||
import os.path
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib2
|
||||
|
||||
import debian.changelog
|
||||
|
||||
from ubuntutools.config import UDTConfig
|
||||
from ubuntutools.logger import Logger
|
||||
from ubuntutools.misc import dsc_name, dsc_url
|
||||
|
||||
DEFAULT_DEBIAN_MIRROR = 'http://ftp.debian.org/debian'
|
||||
DEFAULT_DEBSEC_MIRROR = 'http://security.debian.org'
|
||||
|
||||
def pull(package, version, opts, unpack=False):
|
||||
"Download Debian source package version version"
|
||||
urls = []
|
||||
# TODO: Not all packages are main :)
|
||||
# Practically this is fine, as it'll be found on snapshot, but still ugly.
|
||||
if opts.debsec_mirror and opts.debsec_mirror != DEFAULT_DEBSEC_MIRROR:
|
||||
urls.append(dsc_url(opts.debsec_mirror, 'main', package, version))
|
||||
urls.append(dsc_url(DEFAULT_DEBSEC_MIRROR, 'main', package, version))
|
||||
if opts.debian_mirror and opts.debian_mirror != DEFAULT_DEBIAN_MIRROR:
|
||||
urls.append(dsc_url(opts.debian_mirror, 'main', package, version))
|
||||
urls.append(dsc_url(DEFAULT_DEBIAN_MIRROR, 'main', package, version))
|
||||
|
||||
for url in urls:
|
||||
cmd = ('dget', '-u' + ('x' if unpack else 'd'), url)
|
||||
Logger.command(cmd)
|
||||
return_code = subprocess.call(cmd)
|
||||
if return_code == 0:
|
||||
return True
|
||||
|
||||
Logger.normal('Trying snapshot.debian.org')
|
||||
return pull_from_snapshot(package, version, unpack)
|
||||
|
||||
def pull_from_snapshot(package, version, unpack=False):
|
||||
"Download Debian source package version version from snapshot.debian.org"
|
||||
try:
|
||||
import json
|
||||
except ImportError:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
Logger.error("Please install python-simplejson.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
srcfiles = json.load(urllib2.urlopen(
|
||||
'http://snapshot.debian.org/mr/package/%s/%s/srcfiles'
|
||||
% (package, version)))
|
||||
except urllib2.HTTPError:
|
||||
Logger.error('Version %s of %s not found on snapshot.debian.org',
|
||||
version, package)
|
||||
return False
|
||||
|
||||
for hash_ in srcfiles['result']:
|
||||
hash_ = hash_['hash']
|
||||
|
||||
try:
|
||||
info = json.load(urllib2.urlopen(
|
||||
'http://snapshot.debian.org/mr/file/%s/info' % hash_))
|
||||
except urllib2.URLError:
|
||||
Logger.error('Unable to dowload info for hash.')
|
||||
return False
|
||||
|
||||
filename = info['result'][0]['name']
|
||||
if '/' in filename:
|
||||
Logger.error('Unacceptable file name: %s', filename)
|
||||
return False
|
||||
|
||||
if os.path.exists(filename):
|
||||
source_file = open(filename, 'r')
|
||||
sha1 = hashlib.sha1()
|
||||
sha1.update(source_file.read())
|
||||
source_file.close()
|
||||
if sha1.hexdigest() == hash_:
|
||||
Logger.normal('Using existing %s', filename)
|
||||
continue
|
||||
|
||||
Logger.normal('Downloading: %s (%0.3f MiB)', filename,
|
||||
info['result'][0]['size'] / 1024.0 / 1024)
|
||||
try:
|
||||
in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_)
|
||||
out = open(filename, 'w')
|
||||
while True:
|
||||
block = in_.read(10240)
|
||||
if block == '':
|
||||
break
|
||||
out.write(block)
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flush()
|
||||
sys.stdout.write('\n')
|
||||
sys.stdout.flush()
|
||||
out.close()
|
||||
except urllib2.URLError:
|
||||
Logger.error('Error downloading %s', filename)
|
||||
return False
|
||||
|
||||
if unpack:
|
||||
cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version))
|
||||
Logger.command(cmd)
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
return True
|
||||
from ubuntutools.mirrors import dsc_name, pull_source_pkg
|
||||
|
||||
def previous_version(package, version, distance):
|
||||
"Given an (extracted) package, determine the version distance versions ago"
|
||||
@ -177,9 +80,13 @@ def main():
|
||||
opts.debsec_mirror = config.get_value('DEBSEC_MIRROR')
|
||||
|
||||
Logger.normal('Downloading %s %s', package, version)
|
||||
if not pull(package, version, opts, unpack=not opts.fetch_only):
|
||||
Logger.error("Couldn't locate version %s of %s.", version, package)
|
||||
sys.exit(1)
|
||||
|
||||
# TODO: Not all packages are main, but snapshot.debian.org should save
|
||||
# the day, as it doesn't care about component.
|
||||
pull_source_pkg(('DEBSEC', 'DEBIAN'),
|
||||
{'DEBSEC': opts.debsec_mirror,
|
||||
'DEBIAN': opts.debian_mirror},
|
||||
'main', package, version, unpack=True)
|
||||
|
||||
if opts.fetch_only:
|
||||
sys.exit(0)
|
||||
@ -189,9 +96,10 @@ def main():
|
||||
Logger.error('No previous version could be found')
|
||||
sys.exit(1)
|
||||
Logger.normal('Downloading %s %s', package, oldversion)
|
||||
if not pull(package, oldversion, opts, unpack=True):
|
||||
Logger.error("Couldn't locate version %s of %s.", oldversion, package)
|
||||
sys.exit(1)
|
||||
pull_source_pkg(('DEBSEC', 'DEBIAN'),
|
||||
{'DEBSEC': opts.debsec_mirror,
|
||||
'DEBIAN': opts.debian_mirror},
|
||||
'main', package, oldversion, unpack=True)
|
||||
|
||||
cmd = ('debdiff', dsc_name(package, oldversion), dsc_name(package, version))
|
||||
Logger.command(cmd)
|
||||
|
||||
@ -25,16 +25,16 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import urllib
|
||||
from optparse import OptionParser
|
||||
|
||||
from ubuntutools.config import UDTConfig
|
||||
from ubuntutools.logger import Logger
|
||||
from ubuntutools.lp.lpapicache import Distribution, Launchpad
|
||||
from ubuntutools.lp.udtexceptions import (SeriesNotFoundException,
|
||||
PackageNotFoundException, PocketDoesNotExistError)
|
||||
from ubuntutools.misc import split_release_pocket, dsc_url
|
||||
PackageNotFoundException,
|
||||
PocketDoesNotExistError)
|
||||
from ubuntutools.mirrors import pull_source_pkg
|
||||
from ubuntutools.misc import split_release_pocket
|
||||
|
||||
def main():
|
||||
usage = "Usage: %prog <package> [release]"
|
||||
@ -83,27 +83,9 @@ def main():
|
||||
Logger.error(error)
|
||||
sys.exit(1)
|
||||
|
||||
urls = []
|
||||
if options.ubuntu_mirror:
|
||||
urls.append(dsc_url(options.ubuntu_mirror, spph.getComponent(),
|
||||
package, spph.getVersion()))
|
||||
dsc_url = [url for url in spph.sourceFileUrls() if url.endswith('.dsc')]
|
||||
assert dsc_url, 'No .dsc file found'
|
||||
urls.append(urllib.unquote(dsc_url[0]))
|
||||
|
||||
Logger.normal('Fetching the source for %s from %s (%s)...',
|
||||
package, release.capitalize(), pocket)
|
||||
for url in urls:
|
||||
cmd = ('dget', '-u' + ('d' if options.download_only else 'x'), url)
|
||||
Logger.command(cmd)
|
||||
return_code = subprocess.call(cmd)
|
||||
if return_code == 0:
|
||||
Logger.normal("Success!")
|
||||
sys.exit(0)
|
||||
|
||||
Logger.error('Failed to fetch and extrace the source. '
|
||||
'Please check the output for the error.')
|
||||
sys.exit(1)
|
||||
pull_source_pkg('UBUNTU', options.ubuntu_mirror, spph.getComponent(),
|
||||
package, spph.getVersion(),
|
||||
unpack=not options.download_only)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@ -34,11 +34,11 @@ class UDTConfig(object):
|
||||
# These are reqired to be used by at least two scripts.
|
||||
defaults = {
|
||||
'BUILDER': 'pbuilder',
|
||||
'DEBIAN_MIRROR': None,
|
||||
'DEBSEC_MIRROR': None,
|
||||
'DEBIAN_MIRROR': 'http://ftp.debian.org/debian',
|
||||
'DEBSEC_MIRROR': 'http://security.debian.org',
|
||||
'LPINSTANCE': 'production',
|
||||
'MIRROR_FALLBACK': True,
|
||||
'UBUNTU_MIRROR': None,
|
||||
'UBUNTU_MIRROR': 'http://archive.ubuntu.com/ubuntu',
|
||||
'UPDATE_BUILDER': False,
|
||||
'WORKDIR': None,
|
||||
}
|
||||
@ -74,7 +74,7 @@ class UDTConfig(object):
|
||||
f.close()
|
||||
return config
|
||||
|
||||
def get_value(self, key, default=None, boolean=False, compat_keys=[]):
|
||||
def get_value(self, key, default=None, boolean=False, compat_keys=()):
|
||||
"""Retrieve a value from the environment or configuration files.
|
||||
keys are prefixed with the script name, falling back to UBUNTUTOOLS for
|
||||
package-wide keys.
|
||||
|
||||
170
ubuntutools/mirrors.py
Normal file
170
ubuntutools/mirrors.py
Normal file
@ -0,0 +1,170 @@
|
||||
# mirrors.py - Functions for dealing with Debian source packages and mirrors.
|
||||
#
|
||||
# Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
|
||||
#
|
||||
# Permission to use, copy, modify, and/or distribute this software for any
|
||||
# purpose with or without fee is hereby granted, provided that the above
|
||||
# copyright notice and this permission notice appear in all copies.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
# PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
import hashlib
|
||||
import os.path
|
||||
import subprocess
|
||||
import urllib2
|
||||
import sys
|
||||
|
||||
from ubuntutools.config import UDTConfig
|
||||
from ubuntutools.logger import Logger
|
||||
|
||||
def dsc_name(package, version):
|
||||
"Return the source package dsc filename for the given package"
|
||||
if ':' in version:
|
||||
version = version.split(':', 1)[1]
|
||||
return '%s_%s.dsc' % (package, version)
|
||||
|
||||
def dsc_url(mirror, component, package, version):
|
||||
"Build a source package URL"
|
||||
group = package[:4] if package.startswith('lib') else package[0]
|
||||
filename = dsc_name(package, version)
|
||||
return os.path.join(mirror, 'pool', component, group, package, filename)
|
||||
|
||||
def pull_source_pkg(archives, mirrors, component, package, version, workdir='.',
|
||||
unpack=False):
|
||||
"""Download a source package or die.
|
||||
archives may be a list or single item (in which case mirrors can be too)
|
||||
mirrors should be a dict (keyed on archive) unless archives is single"""
|
||||
|
||||
if not isinstance(archives, (tuple, list)):
|
||||
if not isinstance(mirrors, dict):
|
||||
mirrors = {archives: mirrors}
|
||||
archives = [archives]
|
||||
assert all(x in ('DEBIAN', 'DEBSEC', 'UBUNTU') for x in archives)
|
||||
|
||||
for archive in archives:
|
||||
if try_pull_from_archive(archive, mirrors.get(archive), component,
|
||||
package, version, workdir, unpack):
|
||||
return
|
||||
|
||||
if 'DEBIAN' in archives or 'DEBSEC' in archives:
|
||||
Logger.info('Trying snapshot.debian.org')
|
||||
if try_pull_from_snapshot(package, version, workdir, unpack):
|
||||
return
|
||||
|
||||
if 'UBUNTU' in archives:
|
||||
Logger.info('Trying Launchpad')
|
||||
if try_pull_from_lp(package, 'ubuntu', version, workdir, unpack):
|
||||
return
|
||||
|
||||
raise Exception('Unable to locate %s/%s %s' % (package, component, version))
|
||||
|
||||
def try_pull_from_archive(archive, mirror, component, package, version,
|
||||
workdir='.', unpack=False):
|
||||
"""Download a source package from the specified source or return False.
|
||||
Try mirror first, then master.
|
||||
"""
|
||||
assert archive in ('DEBIAN', 'DEBSEC', 'UBUNTU')
|
||||
urls = []
|
||||
if mirror and mirror != UDTConfig.defaults[archive + '_MIRROR']:
|
||||
urls.append(dsc_url(mirror, component, package, version))
|
||||
urls.append(dsc_url(UDTConfig.defaults[archive + '_MIRROR'], component,
|
||||
package, version))
|
||||
|
||||
for url in urls:
|
||||
cmd = ('dget', '-u' + ('x' if unpack else 'd'), url)
|
||||
Logger.command(cmd)
|
||||
return_code = subprocess.call(cmd, cwd=workdir)
|
||||
if return_code == 0:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def try_pull_from_snapshot(package, version, workdir='.', unpack=False):
|
||||
"""Download Debian source package version version from snapshot.debian.org
|
||||
or return False
|
||||
"""
|
||||
try:
|
||||
import json
|
||||
except ImportError:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
Logger.error("Please install python-simplejson.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
srcfiles = json.load(urllib2.urlopen(
|
||||
'http://snapshot.debian.org/mr/package/%s/%s/srcfiles'
|
||||
% (package, version)))
|
||||
except urllib2.HTTPError:
|
||||
Logger.error('Version %s of %s not found on snapshot.debian.org',
|
||||
version, package)
|
||||
return False
|
||||
|
||||
for hash_ in srcfiles['result']:
|
||||
hash_ = hash_['hash']
|
||||
|
||||
try:
|
||||
info = json.load(urllib2.urlopen(
|
||||
'http://snapshot.debian.org/mr/file/%s/info' % hash_))
|
||||
except urllib2.URLError:
|
||||
Logger.error('Unable to dowload info for hash.')
|
||||
return False
|
||||
|
||||
filename = info['result'][0]['name']
|
||||
if '/' in filename:
|
||||
Logger.error('Unacceptable file name: %s', filename)
|
||||
return False
|
||||
pathname = os.path.join(workdir, filename)
|
||||
|
||||
if os.path.exists(pathname):
|
||||
source_file = open(pathname, 'r')
|
||||
sha1 = hashlib.sha1()
|
||||
sha1.update(source_file.read())
|
||||
source_file.close()
|
||||
if sha1.hexdigest() == hash_:
|
||||
Logger.normal('Using existing %s', filename)
|
||||
continue
|
||||
|
||||
Logger.normal('Downloading: %s (%0.3f MiB)', filename,
|
||||
info['result'][0]['size'] / 1024.0 / 1024)
|
||||
try:
|
||||
in_ = urllib2.urlopen('http://snapshot.debian.org/file/%s' % hash_)
|
||||
out = open(pathname, 'w')
|
||||
while True:
|
||||
block = in_.read(10240)
|
||||
if block == '':
|
||||
break
|
||||
out.write(block)
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flush()
|
||||
sys.stdout.write('\n')
|
||||
sys.stdout.flush()
|
||||
out.close()
|
||||
except urllib2.URLError:
|
||||
Logger.error('Error downloading %s', filename)
|
||||
return False
|
||||
|
||||
if unpack:
|
||||
cmd = ('dpkg-source', '--no-check', '-x', dsc_name(package, version))
|
||||
Logger.command(cmd)
|
||||
subprocess.check_call(cmd)
|
||||
return True
|
||||
|
||||
def try_pull_from_lp(package, distro, version, workdir='.', unpack=False):
|
||||
"""Try to download the specified version of a source package from Launchpad
|
||||
or return False
|
||||
"""
|
||||
url = ('https://launchpad.net/%s/+archive/primary/+files/%s'
|
||||
% (distro, dsc_name(package, version)))
|
||||
cmd = ('dget', '-u' + ('x' if unpack else 'd'), url)
|
||||
Logger.command(cmd)
|
||||
return_code = subprocess.call(cmd, cwd=workdir)
|
||||
if return_code == 0:
|
||||
return True
|
||||
return False
|
||||
@ -119,15 +119,3 @@ def split_release_pocket(release):
|
||||
pocket)
|
||||
|
||||
return (release, pocket)
|
||||
|
||||
def dsc_name(package, version):
|
||||
"Return the source package dsc filename for the given package"
|
||||
if ':' in version:
|
||||
version = version.split(':', 1)[1]
|
||||
return '%s_%s.dsc' % (package, version)
|
||||
|
||||
def dsc_url(mirror, component, package, version):
|
||||
"Build a source package URL"
|
||||
group = package[:4] if package.startswith('lib') else package[0]
|
||||
filename = dsc_name(package, version)
|
||||
return os.path.join(mirror, 'pool', component, group, package, filename)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user