From b5ae0bdca36cd24637cba756663a78a38bc97cfd Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Mon, 11 Feb 2019 14:30:01 -0500 Subject: [PATCH 01/30] simplify subprocess usage --- submittodebian | 19 ++++++++----------- ubuntu-iso | 10 +++++----- ubuntutools/builder.py | 8 +++----- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/submittodebian b/submittodebian index 86fa45f..2599ff4 100755 --- a/submittodebian +++ b/submittodebian @@ -27,7 +27,9 @@ import os import re import shutil import sys -from subprocess import call, check_call, Popen, PIPE + +from subprocess import call, check_call, run, Popen, PIPE, DEVNULL + from tempfile import mkdtemp from debian.changelog import Changelog @@ -85,9 +87,8 @@ def gen_debdiff(tmpdir, changelog): debdiff = os.path.join(tmpdir, '%s_%s.debdiff' % (pkg, newver)) - devnull = open('/dev/null', 'w') diff_cmd = ['bzr', 'diff', '-r', 'tag:' + str(oldver)] - if call(diff_cmd, stdout=devnull, stderr=devnull) == 1: + if call(diff_cmd, stdout=DEVNULL, stderr=DEVNULL) == 1: print("Extracting bzr diff between %s and %s" % (oldver, newver)) else: if oldver.epoch is not None: @@ -104,14 +105,10 @@ def gen_debdiff(tmpdir, changelog): print("Generating debdiff between %s and %s" % (oldver, newver)) diff_cmd = ['debdiff', olddsc, newdsc] - diff = Popen(diff_cmd, stdout=PIPE) - debdiff_f = open(debdiff, 'w') - filterdiff = Popen(['filterdiff', '-x', '*changelog*'], - stdin=diff.stdout, stdout=debdiff_f) - diff.stdout.close() - filterdiff.wait() - debdiff_f.close() - devnull.close() + with Popen(diff_cmd, stdout=PIPE, encoding='utf-8') as diff: + with open(debdiff, 'w', encoding='utf-8') as debdiff_f: + run(['filterdiff', '-x', '*changelog*'], + stdin=diff.stdout, stdout=debdiff_f, encoding='utf-8') return debdiff diff --git a/ubuntu-iso b/ubuntu-iso index dc52bac..fd56a66 100755 --- a/ubuntu-iso +++ b/ubuntu-iso @@ -27,15 +27,15 @@ import sys def extract(iso, path): command = ['isoinfo', '-R', '-i', iso, '-x', path] - pipe = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, encoding='utf-8') - stdout, stderr = pipe.communicate() + pipe = subprocess.run(command, encoding='utf-8', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) if pipe.returncode != 0: - sys.stderr.write(stderr) + sys.stderr.write(pipe.stderr) sys.exit(pipe.returncode) - return stdout + return pipe.stdout def main(): diff --git a/ubuntutools/builder.py b/ubuntutools/builder.py index 4c201d9..8ab478f 100644 --- a/ubuntutools/builder.py +++ b/ubuntutools/builder.py @@ -34,8 +34,7 @@ class Builder(object): def __init__(self, name): self.name = name cmd = ["dpkg-architecture", "-qDEB_BUILD_ARCH_CPU"] - self.architecture = subprocess.check_output( - cmd, encoding='utf-8').strip() + self.architecture = subprocess.check_output(cmd, encoding='utf-8').strip() def _build_failure(self, returncode, dsc_file): if returncode != 0: @@ -124,9 +123,8 @@ class Sbuild(Builder): def update(self, dist): cmd = ["schroot", "--list"] Logger.command(cmd) - process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, encoding='utf-8') - chroots, _ = process.communicate()[0].strip().split() + process = subprocess.run(cmd, stdout=subprocess.PIPE, encoding='utf-8') + chroots, _ = process.stdout.strip().split() if process.returncode != 0: return process.returncode From d0aa64a51bf784db38310e7dea321bb9b3db7c85 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Sep 2019 16:24:49 -0400 Subject: [PATCH 02/30] ubuntutools/misc: define POCKETS and DEFAULT_POCKETS POCKETS is all valid pockets (capitalized), DEFAULT_POCKETS is all in POCKETS except 'Backports'. --- ubuntutools/misc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ubuntutools/misc.py b/ubuntutools/misc.py index 1d01d35..e322422 100644 --- a/ubuntutools/misc.py +++ b/ubuntutools/misc.py @@ -32,6 +32,9 @@ import distro_info from ubuntutools.lp.udtexceptions import PocketDoesNotExistError +DEFAULT_POCKETS = ('Release', 'Security', 'Updates', 'Proposed') +POCKETS = DEFAULT_POCKETS + ('Backports',) + _system_distribution_chain = [] @@ -141,7 +144,7 @@ def split_release_pocket(release, default='Release'): (release, pocket) = release.rsplit('-', 1) pocket = pocket.capitalize() - if pocket not in ('Release', 'Security', 'Updates', 'Proposed', 'Backports'): + if pocket not in POCKETS: raise PocketDoesNotExistError("Pocket '%s' does not exist." % pocket) return (release, pocket) From 51231f116c37d8dd60f13d28c7a7eda11c67d328 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 12 Feb 2019 16:57:57 -0500 Subject: [PATCH 03/30] debian/control: add python3-lazr.restfulclient dep to python3-ubuntutools --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 125a3cf..1d39a8b 100644 --- a/debian/control +++ b/debian/control @@ -132,6 +132,7 @@ Depends: python3-distro-info, python3-httplib2, python3-launchpadlib, + python3-lazr.restfulclient, sensible-utils, ${misc:Depends}, ${python3:Depends}, From d3b8d7a1b7c56d5058a0077cf4661b45902c5dba Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 23 Mar 2017 07:24:17 -0400 Subject: [PATCH 04/30] ubuntutools/lp/lpapicache: expand coverage for LP api --- ubuntutools/lp/lpapicache.py | 233 +++++++++++++++++++++++++++++------ 1 file changed, 193 insertions(+), 40 deletions(-) diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 8fb8399..b5ca2af 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -36,6 +36,7 @@ from lazr.restfulclient.resource import Entry from ubuntutools.version import Version from ubuntutools.lp import (service, api_version) +from ubuntutools.misc import host_architecture from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError, ArchiveNotFoundException, ArchSeriesNotFoundException, @@ -51,6 +52,7 @@ __all__ = [ 'Distribution', 'DistributionSourcePackage', 'DistroSeries', + 'DistroArchSeries', 'Launchpad', 'PersonTeam', 'SourcePackagePublishingHistory', @@ -273,6 +275,12 @@ class DistroArchSeries(BaseWrapper): ''' resource_type = 'distro_arch_series' + def getSeries(self): + ''' + Get DistroSeries for this. + ''' + return DistroSeries(self._lpobject.distroseries_link) + class DistroSeries(BaseWrapper): ''' @@ -284,12 +292,16 @@ class DistroSeries(BaseWrapper): if "_architectures" not in self.__dict__: self._architectures = dict() - def getArchSeries(self, archtag): + def getArchSeries(self, archtag=None): ''' Returns a DistroArchSeries object for an architecture passed by name (e.g. 'amd64'). + If arch is not specified, get the DistroArchSeries for the system arch. + The special archtag 'all' will get the system arch. If the architecture is not found: raise ArchSeriesNotFoundException. ''' + if not archtag or archtag == 'all': + archtag = host_architecture() if archtag not in self._architectures: try: architecture = DistroArchSeries( @@ -315,16 +327,22 @@ class Archive(BaseWrapper): self._pkgset_uploaders = {} self._component_uploaders = {} - def getSourcePackage(self, name, series=None, pocket=None): + def getSourcePackage(self, name, series=None, pocket=None, version=None, + status=None): ''' Returns a SourcePackagePublishingHistory object for the most recent source package in the distribution 'dist', series and pocket. series defaults to the current development series if not specified. + series must be either a series name string, or DistroSeries object. - pocket may be a list, if so, the highest version will be returned. - It defaults to all pockets except backports. + pocket may be a string or a list. If a list, the highest version + will be returned. It defaults to all pockets except backports. + + version may be specified to get only the exact version requested. + + status is optional, to restrict search to a given status only. If the requested source package doesn't exist a PackageNotFoundException is raised. @@ -332,33 +350,48 @@ class Archive(BaseWrapper): return self._getPublishedItem(name, series, pocket, cache=self._srcpkgs, function='getPublishedSources', name_key='source_name', - wrapper=SourcePackagePublishingHistory) + wrapper=SourcePackagePublishingHistory, + version=version, + status=status, + binary=False) - def getBinaryPackage(self, name, archtag=None, series=None, pocket=None): + def getBinaryPackage(self, name, archtag=None, series=None, pocket=None, + version=None, status=None): ''' Returns a BinaryPackagePublishingHistory object for the most recent source package in the distribution 'dist', architecture 'archtag', series and pocket. series defaults to the current development series if not specified. + series must be either a series name string, or DistroArchSeries object. - pocket may be a list, if so, the highest version will be returned. - It defaults to all pockets except backports. + pocket may be a string or a list. If a list, the highest version + will be returned. It defaults to all pockets except backports. + + version may be specified to get only the exact version requested. + + status is optional, to restrict search to a given status only. If the requested binary package doesn't exist a PackageNotFoundException is raised. ''' - if archtag is None: - archtag = [] return self._getPublishedItem(name, series, pocket, archtag=archtag, cache=self._binpkgs, function='getPublishedBinaries', name_key='binary_name', - wrapper=BinaryPackagePublishingHistory) + wrapper=BinaryPackagePublishingHistory, + version=version, + status=status, + binary=True) def _getPublishedItem(self, name, series, pocket, cache, - function, name_key, wrapper, archtag=None): - '''Common code between getSourcePackage and getBinaryPackage + function, name_key, wrapper, archtag=None, + version=None, status=None, + binary=False): + ''' + Common code between getSourcePackage and getBinaryPackage. + + Don't use this directly. ''' if pocket is None: pockets = frozenset(('Proposed', 'Updates', 'Security', 'Release')) @@ -373,34 +406,44 @@ class Archive(BaseWrapper): pocket) dist = Distribution(self.distribution_link) - # Check if series is already a DistroSeries object or not - if not isinstance(series, DistroSeries): - if series: - series = dist.getSeries(series) - else: - series = dist.getDevelopmentSeries() - # getPublishedSources requires a distro_series, while - # getPublishedBinaries requires a distro_arch_series. - # If archtag is not None, I'll assume it's getPublishedBinaries. - if archtag is not None and archtag != []: - if not isinstance(archtag, DistroArchSeries): - arch_series = series.getArchSeries(archtag=archtag) - else: - arch_series = archtag + arch_series = None - if archtag is not None and archtag != []: - index = (name, series.name, archtag, pockets) + # please don't pass DistroArchSeries as archtag! + # but, the code was like that before so keep + # backwards compatibility. + if isinstance(archtag, DistroArchSeries): + series = archtag + archtag = None + + if isinstance(series, DistroArchSeries): + arch_series = series + series = series.getSeries() + elif isinstance(series, DistroSeries): + pass + elif series: + series = dist.getSeries(series) else: - index = (name, series.name, pockets) + series = dist.getDevelopmentSeries() + + if binary: + if arch_series is None: + arch_series = series.getArchSeries(archtag=archtag) + if archtag is None: + archtag = arch_series.architecture_tag + + index = (name, series.name, archtag, pockets, status, version) if index not in cache: params = { name_key: name, - 'status': 'Published', 'exact_match': True, } - if archtag is not None and archtag != []: + + if status: + params['status'] = status + + if binary: params['distro_arch_series'] = arch_series() else: params['distro_series'] = series() @@ -408,6 +451,9 @@ class Archive(BaseWrapper): if len(pockets) == 1: params['pocket'] = list(pockets)[0] + if version: + params['version'] = version + records = getattr(self, function)(**params) latest = None @@ -427,7 +473,7 @@ class Archive(BaseWrapper): package_type = "package" msg = ("The %s '%s' does not exist in the %s %s archive" % (package_type, name, dist.display_name, self.name)) - if archtag is not None and archtag != []: + if binary: msg += " for architecture %s" % archtag pockets = [series.name if pocket == 'Release' else '%s-%s' % (series.name, pocket.lower()) @@ -509,13 +555,23 @@ class SourcePackagePublishingHistory(BaseWrapper): resource_type = 'source_package_publishing_history' def __init__(self, *args): + self._archive = None self._changelog = None self._binaries = None + self._distro_series = None # Don't share _builds between different # SourcePackagePublishingHistory objects if '_builds' not in self.__dict__: self._builds = dict() + def getDistroSeries(self): + ''' + Return the DistroSeries. + ''' + if not self._distro_series: + self._distro_series = DistroSeries(self._lpobject.distro_series_link) + return self._distro_series + def getPackageName(self): ''' Returns the source package name. @@ -534,16 +590,33 @@ class SourcePackagePublishingHistory(BaseWrapper): ''' return self._lpobject.component_name + def getSeriesName(self): + ''' + Returns the series + + Named getSeriesName() to avoid confusion with + getDistroSeries() + ''' + return self.getDistroSeries().name + def getSeriesAndPocket(self): ''' Returns a human-readable release-pocket ''' - series = DistroSeries(self._lpobject.distro_series_link) - release = series.name - if self._lpobject.pocket != 'Release': - release += '-' + self._lpobject.pocket.lower() + release = self.getSeriesName() + if self.pocket != 'Release': + release += '-' + self.pocket.lower() return release + def getArchive(self): + ''' + Get this SPPH's archive. + ''' + if not self._archive: + self._archive = Archive(self._lpobject.archive_link) + + return self._archive + def getChangelog(self, since_version=None): ''' Return the changelog, optionally since a particular version @@ -580,15 +653,19 @@ class SourcePackagePublishingHistory(BaseWrapper): new_entries.append(str(block)) return ''.join(new_entries) - def getBinaries(self): + def getBinaries(self, arch=None): ''' - Returns the resulting BinaryPackagePublishingHistorys + Returns the resulting BinaryPackagePublishingHistorys. + If arch is specified, only returns BPPH for that arch. ''' if self._binaries is None: self._binaries = [BinaryPackagePublishingHistory(bpph) for bpph in self._lpobject.getPublishedBinaries()] - return self._binaries + if not arch: + return list(self._binaries) + + return [b for b in self._binaries if b.arch == arch] def _fetch_builds(self): '''Populate self._builds with the build records.''' @@ -648,6 +725,22 @@ class BinaryPackagePublishingHistory(BaseWrapper): ''' resource_type = 'binary_package_publishing_history' + def __init__(self, *args): + self._arch = None + + @property + def arch(self): + if not self._arch: + das = DistroArchSeries(self._lpobject.distro_arch_series_link) + self._arch = das.architecture_tag + return self._arch + + def getSourcePackageName(self): + ''' + Returns the source package name. + ''' + return self.getBuild().source_package_name + def getPackageName(self): ''' Returns the binary package name. @@ -677,6 +770,54 @@ class BinaryPackagePublishingHistory(BaseWrapper): raise AttributeError("binaryFileUrls can only be found in lpapi " "devel, not 1.0. Login using devel to have it.") + def getBuild(self): + ''' + Returns the original build of the binary package. + ''' + return Build(self._lpobject.build_link) + + def getUrl(self): + ''' + Returns the original build URL of the binary package. + ''' + return "{build}/+files/{filename}".format(build=self.getBuild().getUrl(), + filename=self.getFileName()) + + def getFileVersion(self): + ''' + Returns the file version, which is the package version without the epoch + ''' + return Version(self.getVersion()).strip_epoch() + + def getFileArch(self): + ''' + Returns the file arch, which is 'all' if not arch-specific + ''' + if bool(self._lpobject.architecture_specific): + return self.arch + else: + return 'all' + + def getFileExt(self): + ''' + Returns the file extension; "deb", "ddeb", or "udeb". + ''' + if self.getPackageName().endswith("-dbgsym"): + return "ddeb" + elif self.getPackageName().endswith("-di"): + return "udeb" + else: + return "deb" + + def getFileName(self): + ''' + Returns the filename for this binary package. + ''' + return "{name}_{version}_{arch}.{ext}".format(name=self.getPackageName(), + version=self.getFileVersion(), + arch=self.getFileArch(), + ext=self.getFileExt()) + class MetaPersonTeam(MetaWrapper): @property @@ -792,6 +933,18 @@ class Build(BaseWrapper): def __str__(self): return u'%s: %s' % (self.arch_tag, self.buildstate) + def getSourcePackagePublishingHistory(self): + link = self._lpobject.current_source_publication_link + if link: + if re.search('redacted', link): + # Too old - the link has been 'redacted' + return None + return SourcePackagePublishingHistory(link) + return None + + def getUrl(self): + return self() + def rescore(self, score): if self.can_be_rescored: self().rescore(score=score) From 506e3db60156c9934101d1517e38f612b9a5ae4e Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 23 Mar 2017 07:51:15 -0400 Subject: [PATCH 05/30] pull-pkg: add SourcePackage.pull_binaries() function --- ubuntutools/archive.py | 83 +++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 3bc54b3..8dd7a7b 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -44,7 +44,8 @@ import httplib2 from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Launchpad, Distribution, - SourcePackagePublishingHistory) + SourcePackagePublishingHistory, + BinaryPackagePublishingHistory) from ubuntutools.logger import Logger from ubuntutools.version import Version @@ -224,6 +225,16 @@ class SourcePackage(object): yield self._mirror_url(mirror, name) yield self._lp_url(name) + def _binary_urls(self, name, default_url): + "Generator of URLs for name" + for mirror in self.mirrors: + yield self._mirror_url(mirror, name) + for mirror in self.masters: + if mirror not in self.mirrors: + yield self._mirror_url(mirror, name) + yield self._lp_url(name) + yield default_url + def pull_dsc(self, verify_signature=False): "Retrieve dscfile and parse" if self._dsc_source: @@ -301,28 +312,42 @@ class SourcePackage(object): with open(self.dsc_pathname, 'wb') as f: f.write(self.dsc.raw_text) - def _download_file(self, url, filename): + def _download_file(self, url, filename, verify=True): "Download url to filename in workdir." pathname = os.path.join(self.workdir, filename) - if self.dsc.verify_file(pathname): - Logger.debug('Using existing %s', filename) - return True - size = [entry['size'] for entry in self.dsc['Files'] - if entry['name'] == filename] - assert len(size) == 1 - size = int(size[0]) + size = 0 + if verify: + if self.dsc.verify_file(pathname): + Logger.debug('Using existing %s', filename) + return True + size = [entry['size'] for entry in self.dsc['Files'] + if entry['name'] == filename] + assert len(size) == 1 + size = int(size[0]) + parsed = urlparse(url) - if not self.quiet: - Logger.normal('Downloading %s from %s (%0.3f MiB)', - filename, parsed.hostname, size / 1024.0 / 1024) if parsed.scheme == 'file': in_ = open(parsed.path, 'rb') + if not size: + size = int(os.stat(parsed.path).st_size) else: try: in_ = self.url_opener.open(url) - except URLError: + Logger.debug("Using URL '%s'", url) + except URLError as e: + Logger.debug("URLError opening '%s': %s", url, e) return False + if not size: + contentlen = in_.info().get('Content-Length') + if not contentlen: + Logger.error("Invalid response, no Content-Length") + return False + size = int(contentlen) + + if not self.quiet: + Logger.normal('Downloading %s from %s (%0.3f MiB)', + filename, parsed.hostname, size / 1024.0 / 1024) downloaded = 0 bar_width = 60 @@ -345,7 +370,7 @@ class SourcePackage(object): if not self.quiet: Logger.stdout.write(' ' * (bar_width + 7) + '\r') Logger.stdout.flush() - if not self.dsc.verify_file(pathname): + if verify and not self.dsc.verify_file(pathname): Logger.error('Checksum for %s does not match.', filename) return False return True @@ -366,6 +391,36 @@ class SourcePackage(object): else: raise DownloadError('File %s could not be found' % name) + def pull_binaries(self, arch, name=None): + """Pull binary debs into workdir. + If name is specified, only binary packages matching the regex are included. + Must specify arch, or use 'all' to pull all archs. + Returns the number of files downloaded. + """ + total = 0 + + if not arch: + raise RuntimeError("Must specify arch") + + for bpph in self.lp_spph.getBinaries(arch): + if name and not re.match(name, bpph.binary_package_name): + continue + found = False + for url in self._binary_urls(bpph.getFileName(), bpph.getUrl()): + try: + if self._download_file(url, bpph.getFileName(), verify=False): + found = True + break + except HTTPError as e: + Logger.normal('HTTP Error %i: %s', e.code, str(e)) + except URLError as e: + Logger.normal('URL Error: %s', e.reason) + if found: + total += 1 + else: + Logger.normal("Could not download from any location: %s", bpph.getFileName()) + return total + def verify(self): """Verify that the source package in workdir matches the dsc. Return boolean From b0c22e1d572edadf59ab9bafeeac05b52efd4876 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 23 Mar 2017 11:58:51 -0400 Subject: [PATCH 06/30] pull-pkg: update archive.py SourcePackage verify it can parse/load DSC when created throw PackageNotFoundException from lp_spph if can't find package update test case expected order of url processing --- ubuntutools/archive.py | 183 +++++++++++++++++++------------ ubuntutools/test/test_archive.py | 2 - 2 files changed, 110 insertions(+), 75 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 8dd7a7b..242bc43 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -27,9 +27,9 @@ Approach: 3. Verify checksums. """ -from urllib.error import URLError, HTTPError +from urllib.error import (URLError, HTTPError) from urllib.parse import urlparse -from urllib.request import ProxyHandler, build_opener, urlopen +from urllib.request import urlopen import codecs import hashlib import json @@ -42,10 +42,13 @@ from debian.changelog import Changelog import debian.deb822 import httplib2 +from contextlib import closing + from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Launchpad, Distribution, SourcePackagePublishingHistory, BinaryPackagePublishingHistory) +from ubuntutools.lp.udtexceptions import PackageNotFoundException from ubuntutools.logger import Logger from ubuntutools.version import Version @@ -131,49 +134,56 @@ class SourcePackage(object): self._lp = lp self.workdir = workdir self.quiet = quiet + self._dsc_source = dscfile # Cached values: + self._distribution = None self._component = component self._dsc = None self._spph = None - - # State: - self._dsc_fetched = False + self._version = Version(version) if version else None # Mirrors - self._dsc_source = dscfile self.mirrors = list(mirrors) if self.distribution: self.masters = [UDTConfig.defaults['%s_MIRROR' % self.distribution.upper()]] - if dscfile is not None: - if self.source is None: - self.source = 'unknown' - if version is None: - version = 'unknown' - self.version = Version(version) - - # uses default proxies from the environment - proxy = ProxyHandler() - self.url_opener = build_opener(proxy) + # if a dsc was specified, pull it to get the source/version info + if self._dsc_source: + self.pull_dsc() @property def lp_spph(self): "Return the LP Source Package Publishing History entry" - if not self._spph: - if not Launchpad.logged_in: - if self._lp: - Launchpad.login_existing(self._lp) - else: - Launchpad.login_anonymously() - spph = Distribution(self.distribution).getArchive().getPublishedSources( - source_name=self.source, - version=self.version.full_version, - exact_match=True, - ) - self._spph = SourcePackagePublishingHistory(spph[0]) - return self._spph + if self._spph: + return self._spph + + if not Launchpad.logged_in: + if self._lp: + Launchpad.login_existing(self._lp) + else: + Launchpad.login_anonymously() + + distro = self.getDistribution() + archive = self.getArchive() + params = {'exact_match': True, 'order_by_date': True} + params['version'] = self._version.full_version + spphs = archive.getPublishedSources(source_name=self.source, **params) + if spphs: + self._spph = SourcePackagePublishingHistory(spphs[0]) + return self._spph + + msg = "No {} package found".format(self.source) + msg += " for version {}".format(self._version.full_version) + raise PackageNotFoundException(msg) + + @property + def version(self): + "Return Package version" + if not self._version: + self._version = Version(self.lp_spph.getVersion()) + return self._version @property def component(self): @@ -200,6 +210,15 @@ class SourcePackage(object): self.pull_dsc() return self._dsc + def getDistribution(self): + if not self._distribution: + self._distribution = Distribution(self.distribution) + + return self._distribution + + def getArchive(self): + return self.getDistribution().getArchive() + def _mirror_url(self, mirror, filename): "Build a source package URL on a mirror" if self.source.startswith('lib'): @@ -264,13 +283,12 @@ class SourcePackage(object): raise DownloadError("%s: %s %s" % (url, response.status, response.reason)) self._dsc = Dsc(body) - self._dsc_fetched = True def _check_dsc(self, verify_signature=False): "Check that the dsc matches what we are expecting" assert self._dsc is not None self.source = self.dsc['Source'] - self.version = Version(self.dsc['Version']) + self._version = Version(self.dsc['Version']) valid = False message = None @@ -312,6 +330,43 @@ class SourcePackage(object): with open(self.dsc_pathname, 'wb') as f: f.write(self.dsc.raw_text) + def _download_file_helper(self, f, pathname, size): + "Perform the dowload." + BLOCKSIZE = 16 * 1024 + + with open(pathname, 'wb') as out: + if not (Logger.isEnabledFor(logging.INFO) and + sys.stderr.isatty() and + size): + shutil.copyfileobj(f, out, BLOCKSIZE) + return + + XTRALEN = len('[] 99%') + downloaded = 0 + bar_width = 60 + term_width = os.get_terminal_size(sys.stderr.fileno())[0] + if term_width < bar_width + XTRALEN + 1: + bar_width = term_width - XTRALEN - 1 + + try: + while True: + block = f.read(BLOCKSIZE) + if not block: + break + out.write(block) + downloaded += len(block) + pct = float(downloaded) / size + bar = ('=' * int(pct * bar_width))[:-1] + '>' + fmt = '[{bar:<%d}]{pct:>3}%%\r' % bar_width + sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100))) + sys.stderr.flush() + finally: + sys.stderr.write(' ' * (bar_width + XTRALEN) + '\r') + if downloaded < size: + Logger.error('Partial download: %0.3f MiB of %0.3f MiB' % + (downloaded / 1024.0 / 1024, + size / 1024.0 / 1024)) + def _download_file(self, url, filename, verify=True): "Download url to filename in workdir." pathname = os.path.join(self.workdir, filename) @@ -325,51 +380,32 @@ class SourcePackage(object): assert len(size) == 1 size = int(size[0]) - parsed = urlparse(url) - - if parsed.scheme == 'file': - in_ = open(parsed.path, 'rb') - if not size: - size = int(os.stat(parsed.path).st_size) + if urlparse(url).scheme in ["", "file"]: + frompath = os.path.abspath(urlparse(url).path) + Logger.info("Copying %s from %s" % (filename, frompath)) + shutil.copyfile(frompath, pathname) else: try: - in_ = self.url_opener.open(url) - Logger.debug("Using URL '%s'", url) - except URLError as e: - Logger.debug("URLError opening '%s': %s", url, e) - return False - if not size: - contentlen = in_.info().get('Content-Length') - if not contentlen: - Logger.error("Invalid response, no Content-Length") + with closing(urlopen(url)) as f: + Logger.debug("Using URL '%s'", f.geturl()) + if not size: + try: + size = int(f.info().get('Content-Length')) + except (AttributeError, TypeError, ValueError): + pass + + Logger.info('Downloading %s from %s%s' % + (filename, urlparse(url).hostname, + ' (%0.3f MiB)' % (size / 1024.0 / 1024) + if size else '')) + + self._download_file_helper(f, pathname, size) + except HTTPError as e: + # It's ok if the file isn't found; we try multiple places to download + if e.code == 404: return False - size = int(contentlen) + raise e - if not self.quiet: - Logger.normal('Downloading %s from %s (%0.3f MiB)', - filename, parsed.hostname, size / 1024.0 / 1024) - - downloaded = 0 - bar_width = 60 - try: - with open(pathname, 'wb') as out: - while True: - block = in_.read(10240) - if block == b'': - break - downloaded += len(block) - out.write(block) - if not self.quiet: - percent = downloaded * 100 // size - bar = '=' * int(round(downloaded * bar_width / size)) - bar = (bar + '>' + ' ' * bar_width)[:bar_width] - Logger.stdout.write('[%s] %#3i%%\r' % (bar, percent)) - Logger.stdout.flush() - in_.close() - finally: - if not self.quiet: - Logger.stdout.write(' ' * (bar_width + 7) + '\r') - Logger.stdout.flush() if verify and not self.dsc.verify_file(pathname): Logger.error('Checksum for %s does not match.', filename) return False @@ -630,7 +666,8 @@ class FakeSPPH(object): self.name + '_' + pkgversion, 'changelog' + extension) try: - self._changelog = urlopen(url).read() + with closing(urlopen(url)) as f: + self._changelog = f.read() except HTTPError as error: print(('%s: %s' % (url, error)), file=sys.stderr) return None diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index aa8c372..f7f94f3 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -232,8 +232,6 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): debsec_mirror = 'http://mirror/debsec' sequence = [self.urlopen_null, - self.urlopen_404, - self.urlopen_404, self.urlopen_404, self.urlopen_404, lambda x: BytesIO( From 9f58f5cb80f3bfb9ce35929d1d93f1f86f0bde98 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 23 Mar 2017 12:01:51 -0400 Subject: [PATCH 07/30] pull-pkg: find latest pkg in series in SourcePackage Allow specifying series instead of version, to find the latest version in that series; or only specify package name, to get the latest version in the devel series --- ubuntutools/archive.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 242bc43..59a76d0 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -125,15 +125,23 @@ class SourcePackage(object): distribution = None def __init__(self, package=None, version=None, component=None, - dscfile=None, lp=None, mirrors=(), workdir='.', quiet=False): - "Can be initialised either using package, version or dscfile" - assert ((package is not None and version is not None) - or dscfile is not None) + dscfile=None, lp=None, mirrors=(), workdir='.', quiet=False, + series=None, pocket=None): + """Can be initialised using either package or dscfile. + If package is specified, either the version or series can also be + specified; using version will get the specific package version, + while using the series will get the latest version from that series. + Specifying only the package with no version or series will get the + latest version from the development series. + """ + assert (package is not None or dscfile is not None) self.source = package self._lp = lp self.workdir = workdir self.quiet = quiet + self._series = series + self._pocket = pocket self._dsc_source = dscfile # Cached values: @@ -167,15 +175,33 @@ class SourcePackage(object): distro = self.getDistribution() archive = self.getArchive() + series = None params = {'exact_match': True, 'order_by_date': True} - params['version'] = self._version.full_version + if self._version: + # if version was specified, use that + params['version'] = self._version.full_version + else: + if self._series: + # if version not specified, get the latest from this series + series = distro.getSeries(self._series) + else: + # if no version or series, get the latest from devel series + series = distro.getDevelopmentSeries() + params['distro_series'] = series() + if self._pocket: + params['pocket'] = self._pocket spphs = archive.getPublishedSources(source_name=self.source, **params) if spphs: self._spph = SourcePackagePublishingHistory(spphs[0]) return self._spph msg = "No {} package found".format(self.source) - msg += " for version {}".format(self._version.full_version) + if self._version: + msg += " for version {}".format(self._version.full_version) + elif series: + msg += " in series {}".format(series.name) + if self._pocket: + msg += " pocket {}".format(self._pocket) raise PackageNotFoundException(msg) @property From f944d3146a593fd0696e4325f46d59cc42c29d2d Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 23 Mar 2017 13:22:53 -0400 Subject: [PATCH 08/30] pull-pkg: find src pkg name from binary pkg name --- ubuntutools/archive.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 59a76d0..eb15560 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -138,6 +138,7 @@ class SourcePackage(object): self.source = package self._lp = lp + self.binary = None self.workdir = workdir self.quiet = quiet self._series = series @@ -195,6 +196,35 @@ class SourcePackage(object): self._spph = SourcePackagePublishingHistory(spphs[0]) return self._spph + if not self.binary: + if series: + arch_series = series.getArchSeries() + params['distro_arch_series'] = arch_series() + del params['distro_series'] + bpphs = archive.getPublishedBinaries(binary_name=self.source, **params) + if bpphs: + bpph = BinaryPackagePublishingHistory(bpphs[0]) + self.binary = self.source + self.source = bpph.getSourcePackageName() + Logger.normal("Using source package '{}' for binary package '{}'" + .format(self.source, self.binary)) + try: + spph = bpph.getBuild().getSourcePackagePublishingHistory() + except Exception: + spph = None + if spph: + self._spph = spph + return self._spph + else: + # binary build didn't include source link, unfortunately + # so try again with the updated self.source name + if not self._version: + # Get version first if user didn't specify it, as some + # binaries have their version hardcoded in their name, + # such as the kernel package + self._version = Version(bpph.getVersion()) + return self.lp_spph + msg = "No {} package found".format(self.source) if self._version: msg += " for version {}".format(self._version.full_version) From b96885f05e59aa24f17aed9d2437929108519e80 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 31 Mar 2017 11:26:26 -0400 Subject: [PATCH 09/30] ubuntutools/archive: convert external rmadison call to native implementation instead of a function that calls the system program rmadison, use a fully-functional class to interface with the madison api, as well as the debian snapshot api --- ubuntutools/archive.py | 628 ++++++++++++++++++++++++------- ubuntutools/lp/udtexceptions.py | 5 + ubuntutools/requestsync/mail.py | 36 +- ubuntutools/test/test_archive.py | 1 - 4 files changed, 512 insertions(+), 158 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index eb15560..06b0247 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -48,7 +48,9 @@ from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Launchpad, Distribution, SourcePackagePublishingHistory, BinaryPackagePublishingHistory) -from ubuntutools.lp.udtexceptions import PackageNotFoundException +from ubuntutools.lp.udtexceptions import (PackageNotFoundException, + SeriesNotFoundException, + InvalidDistroValueError) from ubuntutools.logger import Logger from ubuntutools.version import Version @@ -123,6 +125,7 @@ class SourcePackage(object): directly. """ distribution = None + spph_class = SourcePackagePublishingHistory def __init__(self, package=None, version=None, component=None, dscfile=None, lp=None, mirrors=(), workdir='.', quiet=False, @@ -139,6 +142,7 @@ class SourcePackage(object): self.source = package self._lp = lp self.binary = None + self.try_binary = True self.workdir = workdir self.quiet = quiet self._series = series @@ -193,10 +197,10 @@ class SourcePackage(object): params['pocket'] = self._pocket spphs = archive.getPublishedSources(source_name=self.source, **params) if spphs: - self._spph = SourcePackagePublishingHistory(spphs[0]) + self._spph = self.spph_class(spphs[0]) return self._spph - if not self.binary: + if self.try_binary and not self.binary: if series: arch_series = series.getArchSeries() params['distro_arch_series'] = arch_series() @@ -310,6 +314,12 @@ class SourcePackage(object): yield self._lp_url(name) yield default_url + def _binary_files_info(self, arch, name): + for bpph in self.lp_spph.getBinaries(arch): + if name and not re.match(name, bpph.binary_package_name): + continue + yield (bpph.getFileName(), bpph.getUrl(), 0) + def pull_dsc(self, verify_signature=False): "Retrieve dscfile and parse" if self._dsc_source: @@ -423,10 +433,9 @@ class SourcePackage(object): (downloaded / 1024.0 / 1024, size / 1024.0 / 1024)) - def _download_file(self, url, filename, verify=True): + def _download_file(self, url, filename, verify=True, size=0): "Download url to filename in workdir." pathname = os.path.join(self.workdir, filename) - size = 0 if verify: if self.dsc.verify_file(pathname): Logger.debug('Using existing %s', filename) @@ -494,13 +503,11 @@ class SourcePackage(object): if not arch: raise RuntimeError("Must specify arch") - for bpph in self.lp_spph.getBinaries(arch): - if name and not re.match(name, bpph.binary_package_name): - continue + for (fname, furl, fsize) in self._binary_files_info(arch, name): found = False - for url in self._binary_urls(bpph.getFileName(), bpph.getUrl()): + for url in self._binary_urls(fname, furl): try: - if self._download_file(url, bpph.getFileName(), verify=False): + if self._download_file(url, fname, False, fsize): found = True break except HTTPError as e: @@ -510,7 +517,7 @@ class SourcePackage(object): if found: total += 1 else: - Logger.normal("Could not download from any location: %s", bpph.getFileName()) + Logger.normal("Could not download from any location: %s", fname) return total def verify(self): @@ -562,42 +569,68 @@ class SourcePackage(object): return os.path.abspath(difffn) +class DebianSPPH(SourcePackagePublishingHistory): + """SPPH with getBinaries() overridden, + as LP doesn't have Debian binaries + """ + resource_type = 'source_package_publishing_history' + + def getBinaries(self, arch=None): + if not self._binaries: + Logger.normal('Using Snapshot to find binary packages') + srcpkg = Snapshot.getSourcePackage(self.getPackageName(), + version=self.getVersion()) + self._binaries = [b.getBPPH() for b in srcpkg.getBinaryFiles()] + return super(DebianSPPH, self).getBinaries(arch) + + class DebianSourcePackage(SourcePackage): "Download / unpack a Debian source package" distribution = 'debian' + spph_class = DebianSPPH def __init__(self, *args, **kwargs): super(DebianSourcePackage, self).__init__(*args, **kwargs) self.masters.append(UDTConfig.defaults['DEBSEC_MIRROR']) - # Cached values: - self._snapshot_list = None - # Overridden methods: + # Cached values: + self._snapshot_package = None + self._snapshot_files = None + # Don't bother searching in LP for debian binaries, they aren't there + self.try_binary = False + + # Debian doesn't have 'pockets' + if self._pocket: + if self._pocket.lower() != 'release': + Logger.error("Debian does not use 'pockets', ignoring pocket '%s'", + self._pocket) + self._pocket = None + + # Overridden properties/methods: @property def lp_spph(self): "Return the LP Source Package Publishing History entry" if not self._spph: try: + # superclass will set self._spph return super(DebianSourcePackage, self).lp_spph - except IndexError: + except PackageNotFoundException: + pass + except SeriesNotFoundException: pass - Logger.normal('Using rmadison for component determination') - comp = 'main' - for record in rmadison(self.distribution, self.source): - if record.get('source') != self.source: - continue - comp = record['component'] - if record['version'] == self.version.full_version: - self._spph = FakeSPPH(record['source'], record['version'], - comp, 'debian') - return self._spph - - Logger.normal('Guessing component from most recent upload') - self._spph = FakeSPPH(self.source, self.version.full_version, comp, - 'debian') + Logger.normal('Package not found in Launchpad, using Snapshot') + self._spph = self.snapshot_package.getSPPH() return self._spph + @property + def component(self): + "Cached archive component, in available" + if not self._component: + Logger.debug('Determining component from Snapshot') + self._component = Snapshot.getComponent(self.source, self.version) + return self._component + def _source_urls(self, name): "Generator of sources for name" wrapped_iterator = super(DebianSourcePackage, self)._source_urls(name) @@ -606,8 +639,13 @@ class DebianSourcePackage(SourcePackage): yield next(wrapped_iterator) except StopIteration: break - if self.snapshot_list: - yield self._snapshot_url(name) + yield self.snapshot_files[name] + + def _binary_files_info(self, arch, name): + for f in self.snapshot_package.getBinaryFiles(arch): + if name and not re.match(name, f.package_name): + continue + yield (f.name, f.getUrl(), f.size) def pull_dsc(self, verify_signature=True): "Retrieve dscfile and parse" @@ -631,32 +669,37 @@ class DebianSourcePackage(SourcePackage): # Local methods: @property - def snapshot_list(self): - "Return a filename -> hash dictionary from snapshot.debian.org" - if self._snapshot_list is None: + def snapshot_package(self): + if not self._snapshot_package: + if self._version or self._spph: + # as .version uses .lpph, and our .lpph might use us, + # only use .version if _version or _spph are set + version = self.version.full_version + srcpkg = Snapshot.getSourcePackage(self.source, version=version) + if not srcpkg: + msg = "Package {} {} not found".format(self.source, version) + raise PackageNotFoundException(msg) + self._snapshot_package = srcpkg + else: + # we have neither version nor spph, so look up our version using madison + Logger.normal('Using madison to find latest version number') + series = self._series + params = {'series': series} if series else {} + srcpkg = Madison(self.distribution).getSourcePackage(self.source, **params) + if not srcpkg: + raise PackageNotFoundException("Package {} not found".format(self.source)) + if self.source != srcpkg.name: + self.binary = self.source + self.source = srcpkg.name + self._snapshot_package = srcpkg + return self._snapshot_package - try: - data = self.url_opener.open( - 'http://snapshot.debian.org/mr/package/%s/%s/srcfiles?fileinfo=1' % - (self.source, self.version.full_version)) - reader = codecs.getreader('utf-8') - srcfiles = json.load(reader(data)) - - except HTTPError: - Logger.error('Version %s of %s not found on ' - 'snapshot.debian.org', - self.version.full_version, self.source) - self._snapshot_list = False - return False - self._snapshot_list = dict((info[0]['name'], hash_) - for hash_, info - in srcfiles['fileinfo'].items()) - return self._snapshot_list - - def _snapshot_url(self, name): - "Return the snapshot.debian.org URL for name" - return os.path.join('http://snapshot.debian.org/file', - self.snapshot_list[name]) + @property + def snapshot_files(self): + if not self._snapshot_files: + files = self.snapshot_package.getFiles() + self._snapshot_files = {f.name: f.getUrl() for f in files} + return self._snapshot_files class UbuntuSourcePackage(SourcePackage): @@ -678,25 +721,342 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): '+files', filename) -class FakeSPPH(object): - """Provide the same interface as - ubuntutools.lpapicache.SourcePackagePublishingHistory - """ - def __init__(self, name, version, component, distribution): +class _WebJSON(object): + def getHostUrl(self): + raise Exception("Not implemented") + + def load(self, path=''): + reader = codecs.getreader('utf-8') + url = self.getHostUrl() + path + Logger.debug("Loading %s" % url) + with closing(urlopen(url)) as data: + return json.load(reader(data)) + + +# DAKweb madison API +# https://github.com/Debian/dak/blob/master/dakweb/queries/madison.py +# This is really only useful to easily find the latest version of a +# package for a specific series (or unstable). This does not provide +# any details at all for older-than-latest package versions. +class Madison(_WebJSON): + urls = { + 'debian': 'https://api.ftp-master.debian.org/madison', + 'ubuntu': 'http://people.canonical.com/~ubuntu-archive/madison.cgi', + } + + def __init__(self, distro='debian'): + super(Madison, self).__init__() + self._distro = distro + # This currently will NOT work with ubuntu; it doesn't support f=json + if distro != 'debian': + raise InvalidDistroValueError("Madison currently only supports Debian") + + def getHostUrl(self): + return self.urls[self._distro] + + def getSourcePackage(self, name, series='unstable'): + url = "?f=json&package={name}&s={series}".format(name=name, series=series) + try: + result = self.load(url) + except HTTPError: + result = None + if not result: + msg = "Package {} not found in '{}'".format(name, series) + raise PackageNotFoundException(msg) + versions = list(result[0][name].values())[0] + latest = versions[sorted(versions.keys(), reverse=True)[0]] + return Snapshot.getSourcePackage(name=latest['source'], + version=latest['source_version']) + + +# Snapshot API +# https://anonscm.debian.org/cgit/mirror/snapshot.debian.org.git/plain/API +class _Snapshot(_WebJSON): + DEBIAN_COMPONENTS = ["main", "contrib", "non-free"] + + def getHostUrl(self): + return "http://snapshot.debian.org" + + def getComponent(self, name, version): + # unfortunately there is no (easy) way to find the component for older + # package versions (madison only lists the most recent versions). + # so we have to parse the file path to determine the component :( + url = "/mr/package/{}/{}/srcfiles".format(name, version) + try: + response = self.load("{}?fileinfo=1".format(url)) + except HTTPError: + msg = "Package {} version {} not found" + raise PackageNotFoundException(msg.format(name, version)) + result = response.get('result') + info = response.get('fileinfo') + if len(result) < 1: + msg = "No source files for package {} version {}" + raise PackageNotFoundException(msg.format(name, version)) + path = info[result[0]['hash']][0]['path'] + # this expects the 'component' to follow 'pool[-*]' in the path + found_pool = False + component = None + for s in path.split('/'): + if found_pool: + component = s + break + if s.startswith('pool'): + found_pool = True + if not component: + Logger.warn("could not determine component from path %s" % path) + return self.DEBIAN_COMPONENTS[0] + if component not in self.DEBIAN_COMPONENTS: + Logger.warn("unexpected component %s" % component) + return component + + def _get_package(self, name, url, pkginit, version, sort_key): + try: + results = self.load("/mr/{}/{}/".format(url, name))['result'] + except HTTPError: + raise PackageNotFoundException("Package {} not found.".format(name)) + + results = sorted(results, key=lambda r: r[sort_key], reverse=True) + results = [pkginit(r) for r in results if version == r['version']] + if not results: + msg = "Package {name} version {version} not found." + raise PackageNotFoundException(msg.format(name=name, version=version)) + return results + + def getSourcePackages(self, name, version): + return self._get_package(name, "package", + lambda obj: SnapshotSourcePackage(obj, name), + version, "version") + + def getSourcePackage(self, name, version): + return self.getSourcePackages(name, version)[0] + + def getBinaryPackages(self, name, version): + return self._get_package(name, "binary", + lambda obj: SnapshotBinaryPackage(obj), + version, "binary_version") + + def getBinaryPackage(self, name, version): + return self.getBinaryPackages(name, version)[0] + + +Snapshot = _Snapshot() + + +class SnapshotPackage(object): + def __init__(self, obj): + self._obj = obj + self._files = None + self._component = None + + @property + def version(self): + return self._obj['version'] + + @property + def component(self): + if not self._component: + self._component = Snapshot.getComponent(self.name, self.version) + return self._component + + +class SnapshotSourcePackage(SnapshotPackage): + def __init__(self, obj, name): + # obj required fields: 'version' + super(SnapshotSourcePackage, self).__init__(obj) self.name = name - self.version = version + self._binary_files = None + self._spph = None + + def getSPPH(self): + if not self._spph: + self._spph = SnapshotSPPH(self) + return self._spph + + def getAllFiles(self): + return self.getFiles() + self.getBinaryFiles() + + def getBinaryFiles(self, arch=None): + if not self._binary_files: + url = "/mr/package/{}/{}/allfiles".format(self.name, self.version) + response = Snapshot.load("{}?fileinfo=1".format(url)) + info = response['fileinfo'] + files = [SnapshotBinaryFile(b['name'], b['version'], self.component, + info[r['hash']][0], r['hash'], + r['architecture'], self.name) + for b in response['result']['binaries'] for r in b['files']] + self._binary_files = files + if not arch: + return list(self._binary_files) + return filter(lambda f: f.isArch(arch), self._binary_files) + + def getFiles(self): + if not self._files: + url = "/mr/package/{}/{}/srcfiles".format(self.name, self.version) + response = Snapshot.load("{}?fileinfo=1".format(url)) + info = response['fileinfo'] + self._files = [SnapshotSourceFile(self.name, self.version, self.component, + info[r['hash']][0], r['hash']) + for r in response['result']] + return list(self._files) + + +class SnapshotBinaryPackage(SnapshotPackage): + def __init__(self, obj): + # obj required fields: 'version', 'binary_version', 'name', 'source' + super(SnapshotBinaryPackage, self).__init__(obj) + + @property + def name(self): + return self._obj['name'] + + @property + def binary_version(self): + return self._obj['binary_version'] + + @property + def source(self): + return self._obj['source'] + + def getBPPH(self, arch): + f = self.getFiles(arch) + if not f: + return None + # Can only be 1 binary file for this pkg name/version/arch + return f[0].getBPPH() + + def getFiles(self, arch=None): + if not self._files: + url = "/mr/binary/{}/{}/binfiles".format(self.name, self.version) + response = Snapshot.load("{}?fileinfo=1".format(url)) + info = response['fileinfo'] + self._files = [SnapshotBinaryFile(self.name, self.version, self.component, + info[r['hash']][0], r['hash'], + r['architecture'], self.source) + for r in response['result']] + if not arch: + return list(self._files) + return filter(lambda f: f.isArch(arch), self._files) + + +class SnapshotFile(object): + def __init__(self, pkg_name, pkg_version, component, obj, h): + self.package_name = pkg_name + self.package_version = pkg_version self.component = component - self.distribution = distribution - self._changelog = None + self._obj = obj + self._hash = h + + @property + def getType(self): + return None + + @property + def archive_name(self): + return self._obj['archive_name'] + + @property + def name(self): + return self._obj['name'] + + @property + def path(self): + return self._obj['path'] + + @property + def size(self): + return self._obj['size'] + + @property + def date(self): + if 'run' in self._obj: + return self._obj['run'] + elif 'first_seen' in self._obj: + return self._obj['first_seen'] + else: + Logger.error('File {} has no date information', self.name) + return 'unknown' + + def getHash(self): + return self._hash + + def getUrl(self): + return "{}/file/{}".format(Snapshot.getHostUrl(), self.getHash()) + + def __repr__(self): + return "{}/{} {} bytes {}".format(self.path, self.name, self.size, self.date) + + +class SnapshotSourceFile(SnapshotFile): + def __init__(self, name, version, component, obj, h): + super(SnapshotSourceFile, self).__init__(name, version, component, obj, h) + + def getType(self): + return 'source' + + +class SnapshotBinaryFile(SnapshotFile): + def __init__(self, name, version, component, obj, h, arch, source): + super(SnapshotBinaryFile, self).__init__(name, version, component, obj, h) + self.source = source + self.arch = arch + self._bpph = None + + def isArch(self, arch): + if not arch: + return True + if self.arch == 'all': + return True + return arch == self.arch + + def getType(self): + return 'binary' + + def getBPPH(self): + if not self._bpph: + self._bpph = SnapshotBPPH(self) + return self._bpph + + +class SnapshotSPPH(object): + """Provide the same interface as SourcePackagePublishingHistory""" + def __init__(self, snapshot_pkg): + self._pkg = snapshot_pkg + + # LP API defined fields + + @property + def component_name(self): + return self.getComponent() + + @property + def display_name(self): + return ("{name} {version}" + .format(name=self.getPackageName(), + version=self.getVersion())) + + @property + def pocket(self): + # Debian does not use 'pockets' + return 'Release' + + @property + def source_package_name(self): + return self.getPackageName() + + @property + def source_package_version(self): + return self.getVersion() + + # SPPH functions def getPackageName(self): - return self.name + return self._pkg.name def getVersion(self): - return self.version + return self._pkg.version def getComponent(self): - return self.component + return self._pkg.component def getChangelog(self, since_version=None): ''' @@ -704,28 +1064,23 @@ class FakeSPPH(object): May return None if the changelog isn't available ''' if self._changelog is None: - if self.name.startswith('lib'): - subdir = 'lib%s' % self.name[3] + name = self.getPackageName() + if name.startswith('lib'): + subdir = 'lib%s' % name[3] else: - subdir = self.name[0] - # Strip epoch from version - pkgversion = self.version.split(':', 1)[-1] - extension = '' - if self.distribution == 'debian': - base = 'http://packages.debian.org/' - extension = '.txt' - elif self.distribution == 'ubuntu': - base = 'http://changelogs.ubuntu.com/' + subdir = name[0] + pkgversion = Version(self.getVersion()).strip_epoch() + base = 'http://packages.debian.org/' url = os.path.join(base, 'changelogs', 'pool', - self.component, subdir, self.name, - self.name + '_' + pkgversion, - 'changelog' + extension) + self.getComponent(), subdir, name, + name + '_' + pkgversion, + 'changelog.txt') try: with closing(urlopen(url)) as f: self._changelog = f.read() except HTTPError as error: - print(('%s: %s' % (url, error)), file=sys.stderr) + Logger.error('{}: {}'.format(url, error)) return None if since_version is None: @@ -741,59 +1096,66 @@ class FakeSPPH(object): new_entries.append(str(block)) return ''.join(new_entries) + def getBinaries(self, arch=None): + return [b.getBPPH() for b in self._pkg.getBinaryFiles(arch)] -def rmadison(url, package, suite=None, arch=None): - "Call rmadison and parse the result" - cmd = ['rmadison', '-u', url] - if suite: - cmd += ['-s', suite] - if arch: - cmd += ['-a', arch] - cmd.append(package) - process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') - output, error_output = process.communicate() - if process.wait() != 0: - if error_output: - Logger.error('rmadison failed with: %s', error_output) - else: - Logger.error('rmadison failed') - sys.exit(1) - # rmadison uses some shorthand - if suite: - suite = suite.replace('-proposed-updates', '-p-u') +class SnapshotBPPH(object): + """Provide the same interface as BinaryPackagePublishingHistory""" + def __init__(self, snapshot_binfile): + self._file = snapshot_binfile - for line in output.strip().splitlines(): - pkg, ver, dist, archs = [x.strip() for x in line.split('|')] - comp = 'main' - if '/' in dist: - dist, comp = dist.split('/') - archs = set(x.strip() for x in archs.split(',')) + # LP API defined fields + @property + def architecture_specific(self): + return self._file.arch != 'all' - # rmadison returns some results outside the requested set. - # It'll include backports, and when given an unknown suite, - # it ignores that argument - # - # some versions (2.14.1ubuntu0.1) of rmadison return 'sid' when - # asked about 'unstable'. Others return 'unstable'. Accept either. - if (suite and dist != suite and not - (suite == 'sid' and dist == 'unstable')): - continue + @property + def binary_package_name(self): + return self.getPackageName() - if 'source' in archs: - yield { - 'source': pkg, - 'version': ver, - 'suite': dist, - 'component': comp, - } - archs.discard('source') - if archs: - yield { - 'binary': pkg, - 'version': ver, - 'suite': dist, - 'component': comp, - 'architectures': archs, - } + @property + def binary_package_version(self): + return self.getVersion() + + @property + def component_name(self): + return self.getComponent() + + @property + def display_name(self): + return ("{name} {version}" + .format(name=self.getPackageName(), + version=self.getVersion())) + + @property + def pocket(self): + # Debian does not use 'pockets' + return 'Release' + + # BPPH functions + + @property + def arch(self): + return self._file.arch + + def getSourcePackageName(self): + return self._file.source + + def getPackageName(self): + return self._file.package_name + + def getVersion(self): + return self._file.package_version + + def getComponent(self): + return self._file.component + + def getBuild(self): + return None + + def getUrl(self): + return self._file.getUrl() + + def getFileName(self): + return self._file.name diff --git a/ubuntutools/lp/udtexceptions.py b/ubuntutools/lp/udtexceptions.py index 6932c61..077502e 100644 --- a/ubuntutools/lp/udtexceptions.py +++ b/ubuntutools/lp/udtexceptions.py @@ -26,3 +26,8 @@ class AlreadyLoggedInError(Exception): class ArchSeriesNotFoundException(BaseException): """Thrown when a distroarchseries is not found.""" pass + + +class InvalidDistroValueError(ValueError): + """ Thrown when distro value is invalid """ + pass diff --git a/ubuntutools/requestsync/mail.py b/ubuntutools/requestsync/mail.py index 083b386..00b72ac 100644 --- a/ubuntutools/requestsync/mail.py +++ b/ubuntutools/requestsync/mail.py @@ -31,11 +31,10 @@ import tempfile from debian.changelog import Changelog from distro_info import DebianDistroInfo, DistroDataOutdated -from ubuntutools.archive import rmadison, FakeSPPH +from ubuntutools.archive import DebianSourcePackage, UbuntuSourcePackage from ubuntutools.lp.udtexceptions import PackageNotFoundException from ubuntutools.logger import Logger from ubuntutools.question import confirmation_prompt, YesNoQuestion -from ubuntutools.version import Version __all__ = [ @@ -48,32 +47,21 @@ __all__ = [ ] -def _get_srcpkg(distro, name, release): - if distro == 'debian': - # Canonicalise release: - debian_info = DebianDistroInfo() - try: - codename = debian_info.codename(release, default=release) - except DistroDataOutdated as e: - Logger.warn(e) - - lines = list(rmadison(distro, name, suite=codename, arch='source')) - if not lines: - lines = list(rmadison(distro, name, suite=release, arch='source')) - if not lines: - raise PackageNotFoundException("'%s' doesn't appear to exist in %s '%s'" % - (name, distro.capitalize(), release)) - pkg = max(lines, key=lambda x: Version(x['version'])) - - return FakeSPPH(pkg['source'], pkg['version'], pkg['component'], distro) - - def get_debian_srcpkg(name, release): - return _get_srcpkg('debian', name, release) + # Canonicalise release: + debian_info = DebianDistroInfo() + try: + codename = debian_info.codename(release, default=release) + return DebianSourcePackage(package=name, series=codename).lp_spph + except DistroDataOutdated as e: + Logger.warn(e) + except PackageNotFoundException: + pass + return DebianSourcePackage(package=name, series=release).lp_spph def get_ubuntu_srcpkg(name, release): - return _get_srcpkg('ubuntu', name, release) + return UbuntuSourcePackage(package=name, series=release).lp_spph def need_sponsorship(name, component, release): diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index f7f94f3..fb263d2 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -85,7 +85,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): self.workdir = tempfile.mkdtemp(prefix='udt-test') self._stubout('ubuntutools.archive.Distribution') - self._stubout('ubuntutools.archive.rmadison') self.mock_http = self._stubout('httplib2.Http.request') self.mock_http.side_effect = self.request_proxy From f4f16f95f76bf6f3a81bf59f91ccac45e4bb63ba Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 31 Mar 2017 15:48:32 -0400 Subject: [PATCH 10/30] pull-uca: update UbuntuCloudArchiveSourcePackage look in its PPA for its SPPHs look up the development (latest) UCA release if not specified use the 'series' param as the UCA release, instead of 'uca_release' param --- ubuntutools/archive.py | 50 ++++++++++++++++++++++++++++++++---- ubuntutools/lp/lpapicache.py | 7 +++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 06b0247..7aa9f19 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -45,7 +45,7 @@ import httplib2 from contextlib import closing from ubuntutools.config import UDTConfig -from ubuntutools.lp.lpapicache import (Launchpad, Distribution, +from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, SourcePackagePublishingHistory, BinaryPackagePublishingHistory) from ubuntutools.lp.udtexceptions import (PackageNotFoundException, @@ -146,6 +146,7 @@ class SourcePackage(object): self.workdir = workdir self.quiet = quiet self._series = series + self._use_series = True self._pocket = pocket self._dsc_source = dscfile @@ -185,7 +186,7 @@ class SourcePackage(object): if self._version: # if version was specified, use that params['version'] = self._version.full_version - else: + elif self._use_series: if self._series: # if version not specified, get the latest from this series series = distro.getSeries(self._series) @@ -709,15 +710,54 @@ class UbuntuSourcePackage(SourcePackage): class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): "Download / unpack an Ubuntu Cloud Archive source package" - def __init__(self, uca_release, *args, **kwargs): + _ppas = None + _ppa_names = None + + def __init__(self, *args, **kwargs): super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs) - self._uca_release = uca_release + self._use_series = False # UCA doesn't really use distro series + self._uca_release = self._series + self._series = None self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"] + @classmethod + def getArchives(cls): + if not cls._ppas: + ppas = filter(lambda p: p.name.endswith('-staging'), + PersonTeam.fetch('ubuntu-cloud-archive').getPPAs()) + cls._ppas = sorted(ppas, key=lambda p: p.name, reverse=True) + return cls._ppas + + @classmethod + def getReleaseNames(cls): + if not cls._ppa_names: + cls._ppa_names = [p.name.split('-', 1)[0] for p in cls.getArchives()] + return cls._ppa_names + + @classmethod + def getDevelopmentRelease(cls): + return cls.getReleaseNames()[0] + + @property + def uca_release(self): + if not self._uca_release: + self._uca_release = self.getDevelopmentRelease() + Logger.normal('Using UCA release %s', self._uca_release) + return self._uca_release + + def getArchive(self): + ppas = {p.name: p for p in self.getArchives()} + release = '{}-staging'.format(self.uca_release) + if release in ppas: + Logger.debug('UCA release {} at {}'.format(self.uca_release, + ppas[release]())) + return ppas[release] + raise SeriesNotFoundException('UCA release {} not found.'.format(self.uca_release)) + def _lp_url(self, filename): "Build a source package URL on Launchpad" return os.path.join('https://launchpad.net', "~ubuntu-cloud-archive", - '+archive', ("%s-staging" % self._uca_release), + '+archive', ("%s-staging" % self.uca_release), '+files', filename) diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index b5ca2af..3b4c736 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -846,6 +846,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam): def __init__(self, *args): # Don't share _upload between different PersonTeams + self._ppas = None if '_upload' not in self.__dict__: self._upload = dict() @@ -923,6 +924,12 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam): return canUpload + def getPPAs(self): + if not self._ppas: + ppas = Launchpad.load(self._lpobject.ppas_collection_link).entries + self._ppas = [Archive(a['self_link']) for a in ppas] + return self._ppas + class Build(BaseWrapper): ''' From 3dabf05370651d82ce0e8add12223002de731527 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 31 Mar 2017 15:50:19 -0400 Subject: [PATCH 11/30] pull-pkg: add debian and ubuntu ddebs mirror urls --- ubuntutools/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ubuntutools/config.py b/ubuntutools/config.py index ba029fe..dfa53a3 100644 --- a/ubuntutools/config.py +++ b/ubuntutools/config.py @@ -37,10 +37,12 @@ class UDTConfig(object): 'BUILDER': 'pbuilder', 'DEBIAN_MIRROR': 'http://deb.debian.org/debian', 'DEBSEC_MIRROR': 'http://security.debian.org', + 'DEBIAN_DDEBS_MIRROR': 'http://debug.mirrors.debian.org/debian-debug', 'LPINSTANCE': 'production', 'MIRROR_FALLBACK': True, 'UBUNTU_MIRROR': 'http://archive.ubuntu.com/ubuntu', 'UBUNTU_PORTS_MIRROR': 'http://ports.ubuntu.com', + 'UBUNTU_DDEBS_MIRROR': 'http://ddebs.ubuntu.com', 'UPDATE_BUILDER': False, 'WORKDIR': None, 'KEYID': None, From ec72cf15389b599fd51d370f948a54f05155bc2e Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 23 Mar 2017 13:47:07 -0400 Subject: [PATCH 12/30] pull-pkg: rename pull-lp-source to pull-pkg no changes to the file, just a rename, to allow the next commit to show the changes to the file. --- pull-lp-source => pull-pkg | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pull-lp-source => pull-pkg (100%) diff --git a/pull-lp-source b/pull-pkg similarity index 100% rename from pull-lp-source rename to pull-pkg From d7bcb012f62e724f166bb41ec67169096f485039 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 14 Sep 2017 19:59:53 -0400 Subject: [PATCH 13/30] pull-pkg: update to use previous SourcePackage improvements New pull-pkg allows pulling source, debs, ddebs, or udebs, or just listing all package files. Also, package lookup by binary name is done automatically. --- pull-pkg | 281 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 205 insertions(+), 76 deletions(-) diff --git a/pull-pkg b/pull-pkg index 1de837e..ed84208 100755 --- a/pull-pkg +++ b/pull-pkg @@ -1,10 +1,11 @@ #!/usr/bin/python3 # -# pull-lp-source -- pull a source package from Launchpad -# Basic usage: pull-lp-source [] +# pull-pkg -- pull package files for debian/ubuntu/uca +# Basic usage: pull-pkg -D distro -p type [version] [release] # # Copyright (C) 2008, Iain Lane , # 2010-2011, Stefano Rivera +# 2017, Dan Streetman # # ################################################################## # @@ -23,124 +24,252 @@ # ################################################################## -import json -import os +import re import sys -import urllib.error -import urllib.request from optparse import OptionParser -from distro_info import UbuntuDistroInfo, DistroDataOutdated +from distro_info import DebianDistroInfo -from ubuntutools.archive import UbuntuSourcePackage, DownloadError +from ubuntutools.archive import (UbuntuSourcePackage, DebianSourcePackage, + UbuntuCloudArchiveSourcePackage, + DownloadError) from ubuntutools.config import UDTConfig -from ubuntutools.lp.lpapicache import Distribution, Launchpad +from ubuntutools.lp.lpapicache import (Distribution, Launchpad) from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, PackageNotFoundException, PocketDoesNotExistError) from ubuntutools.logger import Logger -from ubuntutools.misc import split_release_pocket +from ubuntutools.misc import (split_release_pocket, host_architecture) + +PULL_SOURCE = 'source' +PULL_DEBS = 'debs' +PULL_DDEBS = 'ddebs' +PULL_UDEBS = 'udebs' +PULL_LIST = 'list' + +DEFAULT_PULL = PULL_SOURCE +VALID_PULLS = [PULL_SOURCE, PULL_DEBS, PULL_DDEBS, PULL_UDEBS, PULL_LIST] -def source_package_for(binary, release): - """Query DDE to find the source package for a particular binary - Should really do this with LP, but it's not possible LP: #597041 - """ - url = ('http://dde.debian.net/dde/q/udd/dist/d:ubuntu/r:%s/p:%s/?t=json' - % (release, binary)) - data = None +DISTRO_DEBIAN = 'debian' +DISTRO_UBUNTU = 'ubuntu' +DISTRO_UCA = 'uca' + +DEFAULT_DISTRO = DISTRO_UBUNTU +DISTRO_PKG_CLASS = { + DISTRO_DEBIAN: DebianSourcePackage, + DISTRO_UBUNTU: UbuntuSourcePackage, + DISTRO_UCA: UbuntuCloudArchiveSourcePackage, +} +VALID_DISTROS = DISTRO_PKG_CLASS.keys() + + +def parse_pull(pull): + if not pull: + pull = DEFAULT_PULL + Logger.normal("Defaulting to pull %s", pull) + + # allow 'dbgsym' as alias for 'ddebs' + if pull == 'dbgsym': + Logger.debug("Pulling '%s' for '%s'", PULL_DDEBS, pull) + pull = PULL_DDEBS + # assume anything starting with 'bin' means 'debs' + if str(pull).startswith('bin'): + Logger.debug("Pulling '%s' for '%s'", PULL_DEBS, pull) + pull = PULL_DEBS + # verify pull action is valid + if pull not in VALID_PULLS: + Logger.error("Invalid pull action '%s'", pull) + sys.exit(1) + + return pull + + +def parse_distro(distro): + if not distro: + distro = DEFAULT_DISTRO + Logger.normal("Defaulting to distro %s", distro) + + distro = distro.lower() + + # allow 'lp' for 'ubuntu' + if distro == 'lp': + Logger.debug("Using distro '%s' for '%s'", DISTRO_UBUNTU, distro) + distro = DISTRO_UBUNTU + # assume anything with 'cloud' is UCA + if re.match(r'.*cloud.*', distro): + Logger.debug("Using distro '%s' for '%s'", DISTRO_UCA, distro) + distro = DISTRO_UCA + # verify distro is valid + if distro not in VALID_DISTROS: + Logger.error("Invalid distro '%s'", distro) + sys.exit(1) + + return distro + + +def parse_release(release, distro): + if distro == DISTRO_UCA: + # UCA is special; it is specified UBUNTURELEASE-UCARELEASE or just + # UCARELEASE. The user could also specify UCARELEASE-POCKET. But UCA + # archives always correspond to only one UBUNTURELEASE, and UCA archives + # have only the Release pocket, so only UCARELEASE matters to us. + for r in release.split('-'): + if r in UbuntuCloudArchiveSourcePackage.getReleaseNames(): + Logger.debug("Using UCA release '%s'", r) + return (r, None) + raise SeriesNotFoundException('UCA release {} not found.'.format(release)) + + # Check if release[-pocket] is specified + (release, pocket) = split_release_pocket(release, default=None) + Logger.debug("Parsed release '%s' pocket '%s'", release, pocket) + + if distro == DISTRO_DEBIAN: + # This converts from the aliases like 'unstable' + debian_info = DebianDistroInfo() + codename = debian_info.codename(release) + if codename: + Logger.normal("Using release '%s' for '%s'", codename, release) + release = codename + try: - data = json.load(urllib.request.urlopen(url))['r'] - except urllib.error.URLError as e: - Logger.error('Unable to retrieve package information from DDE: ' - '%s (%s)', url, str(e)) - except ValueError as e: - Logger.error('Unable to parse JSON response from DDE: ' - '%s (%s)', url, str(e)) - if not data: - return None - return data[0]['source'] + d = Distribution(distro) + Logger.debug("Distro '%s' is valid", distro) + except: + Logger.debug("Distro '%s' not valid", distro) + raise SeriesNotFoundException("Distro {} not found".format(distro)) + + # let SeriesNotFoundException flow up + d.getSeries(release) + + Logger.debug("Using distro '%s' release '%s' pocket '%s'", + distro, release, pocket) + return (release, pocket) def main(): - usage = "Usage: %prog [release|version]" + usage = "Usage: %prog [release[-pocket]|version]" opt_parser = OptionParser(usage) + opt_parser.add_option('-v', '--verbose', + dest='verbose', default=False, + action='store_true', + help="Print verbose/debug messages") opt_parser.add_option('-d', '--download-only', dest='download_only', default=False, action='store_true', help="Do not extract the source package") - opt_parser.add_option('-m', '--mirror', metavar='UBUNTU_MIRROR', - dest='ubuntu_mirror', - help='Preferred Ubuntu mirror (default: Launchpad)') + opt_parser.add_option('-m', '--mirror', dest='mirror', + help='Preferred mirror') opt_parser.add_option('--no-conf', dest='no_conf', default=False, action='store_true', help="Don't read config files or environment " "variables") + opt_parser.add_option('-a', '--arch', + dest='arch', default=None, + help="Get binary packages for specified architecture " + "(default: {})".format(host_architecture())) + opt_parser.add_option('-p', '--pull', + dest='pull', default=None, + help="What to pull: {} (default: {})" + .format(", ".join(VALID_PULLS), DEFAULT_PULL)) + opt_parser.add_option('-D', '--distro', + dest='distro', default=None, + help="Pull from: {} (default: {})" + .format(", ".join(VALID_DISTROS), DEFAULT_DISTRO)) (options, args) = opt_parser.parse_args() if not args: opt_parser.error("Must specify package name") + distro = parse_distro(options.distro) + mirrors = [] + config = UDTConfig(options.no_conf) - if options.ubuntu_mirror is None: - options.ubuntu_mirror = config.get_value('UBUNTU_MIRROR') + if options.mirror is None: + options.mirror = config.get_value(distro.upper() + '_MIRROR') + if options.mirror: + mirrors.append(options.mirror) + + pull = parse_pull(options.pull) + if pull == PULL_DDEBS: + ddebs_mirror = config.get_value(distro.upper() + '_DDEBS_MIRROR') + if ddebs_mirror: + mirrors.append(ddebs_mirror) # Login anonymously to LP Launchpad.login_anonymously() + Logger.set_verbosity(options.verbose) + package = str(args[0]).lower() - - ubuntu_info = UbuntuDistroInfo() - if len(args) > 1: # Custom distribution specified. - version = str(args[1]) - else: - try: - version = os.getenv('DIST') or ubuntu_info.devel() - except DistroDataOutdated as e: - Logger.warn("%s\nOr specify a distribution.", e) - sys.exit(1) - component = None - - # Release, not package version number: + version = None release = None pocket = None - try: - (release, pocket) = split_release_pocket(version, default=None) - except PocketDoesNotExistError: - pass - if release in ubuntu_info.all: - archive = Distribution('ubuntu').getArchive() + + if len(args) > 1: try: - spph = archive.getSourcePackage(package, release, pocket) - except SeriesNotFoundException as e: - Logger.error(str(e)) - sys.exit(1) - except PackageNotFoundException as e: - source_package = source_package_for(package, release) - if source_package is not None and source_package != package: + (release, pocket) = parse_release(args[1], distro) + if len(args) > 2: + version = args[2] + except (SeriesNotFoundException, PocketDoesNotExistError): + version = args[1] + Logger.debug("Param '%s' not valid series, must be version", version) + if len(args) > 2: try: - spph = archive.getSourcePackage(source_package, release, - pocket) - package = source_package - except PackageNotFoundException: - Logger.error(str(e)) + (release, pocket) = parse_release(args[2], distro) + except (SeriesNotFoundException, PocketDoesNotExistError): + Logger.error("Can't find series for '%s' or '%s'", + args[1], args[2]) sys.exit(1) - else: - Logger.error(str(e)) - sys.exit(1) - version = spph.getVersion() - component = spph.getComponent() - - Logger.normal('Downloading %s version %s', package, version) - srcpkg = UbuntuSourcePackage(package, version, component=component, - mirrors=[options.ubuntu_mirror]) try: - srcpkg.pull() + pkgcls = DISTRO_PKG_CLASS[distro] + srcpkg = pkgcls(package=package, version=version, + series=release, pocket=pocket, + mirrors=mirrors) + spph = srcpkg.lp_spph + except PackageNotFoundException as e: + Logger.error(str(e)) + sys.exit(1) + + Logger.normal('Found %s', spph.display_name) + + if pull == PULL_LIST: + Logger.normal("Source files:") + for f in srcpkg.dsc['Files']: + Logger.normal(" %s", f['name']) + Logger.normal("Binary files:") + for f in spph.getBinaries(options.arch): + Logger.normal(" %s", f.getFileName()) + sys.exit(0) + + try: + if pull == PULL_SOURCE: + srcpkg.pull() + if not options.download_only: + srcpkg.unpack() + else: + name = '.*' + if package != spph.getPackageName(): + Logger.normal("Pulling binary package '%s'", package) + Logger.normal("Use package name '%s' to pull all binary packages", + spph.getPackageName()) + name = package + if pull == PULL_DEBS: + name = r'{}(? Date: Fri, 15 Sep 2017 17:45:13 -0400 Subject: [PATCH 14/30] pull-pkg: rename pull-pkg to ubuntutools/pullpkg.py module rename only with no content changes, so next commit shows code changes converting script pull-pkg to module pullpkg.py This also changes the file mode from 755 to 644, since we're changing an executable script into a module. --- pull-pkg => ubuntutools/pullpkg.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pull-pkg => ubuntutools/pullpkg.py (100%) mode change 100755 => 100644 diff --git a/pull-pkg b/ubuntutools/pullpkg.py old mode 100755 new mode 100644 similarity index 100% rename from pull-pkg rename to ubuntutools/pullpkg.py From cdaf5d07617f91266bcc7f010a1ac4b53834a108 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 14 Sep 2017 20:05:12 -0400 Subject: [PATCH 15/30] pull-pkg: change pull-pkg script code into modular code --- ubuntutools/pullpkg.py | 244 ++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index ed84208..70fc55a 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -1,7 +1,5 @@ -#!/usr/bin/python3 -# -# pull-pkg -- pull package files for debian/ubuntu/uca -# Basic usage: pull-pkg -D distro -p type [version] [release] +# pullpkg.py -- pull package files for debian/ubuntu/uca +# modified from ../pull-lp-source and converted to module # # Copyright (C) 2008, Iain Lane , # 2010-2011, Stefano Rivera @@ -25,19 +23,18 @@ import re -import sys -from optparse import OptionParser +from argparse import ArgumentParser from distro_info import DebianDistroInfo from ubuntutools.archive import (UbuntuSourcePackage, DebianSourcePackage, - UbuntuCloudArchiveSourcePackage, - DownloadError) + UbuntuCloudArchiveSourcePackage) from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Distribution, Launchpad) from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, PackageNotFoundException, - PocketDoesNotExistError) + PocketDoesNotExistError, + InvalidDistroValueError) from ubuntutools.logger import Logger from ubuntutools.misc import (split_release_pocket, host_architecture) @@ -47,15 +44,12 @@ PULL_DDEBS = 'ddebs' PULL_UDEBS = 'udebs' PULL_LIST = 'list' -DEFAULT_PULL = PULL_SOURCE VALID_PULLS = [PULL_SOURCE, PULL_DEBS, PULL_DDEBS, PULL_UDEBS, PULL_LIST] - DISTRO_DEBIAN = 'debian' DISTRO_UBUNTU = 'ubuntu' DISTRO_UCA = 'uca' -DEFAULT_DISTRO = DISTRO_UBUNTU DISTRO_PKG_CLASS = { DISTRO_DEBIAN: DebianSourcePackage, DISTRO_UBUNTU: UbuntuSourcePackage, @@ -64,10 +58,46 @@ DISTRO_PKG_CLASS = { VALID_DISTROS = DISTRO_PKG_CLASS.keys() +class InvalidPullValueError(ValueError): + """ Thrown when --pull value is invalid """ + pass + + +def create_argparser(default_pull=None, default_distro=None, default_arch=None): + help_default_pull = "What to pull: " + ", ".join(VALID_PULLS) + if default_pull: + help_default_pull += (" (default: %s)" % default_pull) + help_default_distro = "Pull from: " + ", ".join(VALID_DISTROS) + if default_distro: + help_default_distro += (" (default: %s)" % default_distro) + if not default_arch: + default_arch = host_architecture() + help_default_arch = ("Get binary packages for arch (default: %s)" % default_arch) + + parser = ArgumentParser() + parser.add_argument('-v', '--verbose', action='store_true', + help="Print verbose/debug messages") + parser.add_argument('-d', '--download-only', action='store_true', + help="Do not extract the source package") + parser.add_argument('-m', '--mirror', action='append', + help='Preferred mirror(s)') + parser.add_argument('--no-conf', action='store_true', + help="Don't read config files or environment variables") + parser.add_argument('-a', '--arch', default=default_arch, + help=help_default_arch) + parser.add_argument('-p', '--pull', default=default_pull, + help=help_default_pull) + parser.add_argument('-D', '--distro', default=default_distro, + help=help_default_distro) + parser.add_argument('package', help="Package name to pull") + parser.add_argument('release', nargs='?', help="Release to pull from") + parser.add_argument('version', nargs='?', help="Package version to pull") + return parser + + def parse_pull(pull): if not pull: - pull = DEFAULT_PULL - Logger.normal("Defaulting to pull %s", pull) + raise InvalidPullValueError("Must specify --pull") # allow 'dbgsym' as alias for 'ddebs' if pull == 'dbgsym': @@ -79,16 +109,14 @@ def parse_pull(pull): pull = PULL_DEBS # verify pull action is valid if pull not in VALID_PULLS: - Logger.error("Invalid pull action '%s'", pull) - sys.exit(1) + raise InvalidPullValueError("Invalid pull action '%s'" % pull) return pull def parse_distro(distro): if not distro: - distro = DEFAULT_DISTRO - Logger.normal("Defaulting to distro %s", distro) + raise InvalidDistroValueError("Must specify --distro") distro = distro.lower() @@ -102,8 +130,7 @@ def parse_distro(distro): distro = DISTRO_UCA # verify distro is valid if distro not in VALID_DISTROS: - Logger.error("Invalid distro '%s'", distro) - sys.exit(1) + raise InvalidDistroValueError("Invalid distro '%s'" % distro) return distro @@ -132,12 +159,7 @@ def parse_release(release, distro): Logger.normal("Using release '%s' for '%s'", codename, release) release = codename - try: - d = Distribution(distro) - Logger.debug("Distro '%s' is valid", distro) - except: - Logger.debug("Distro '%s' not valid", distro) - raise SeriesNotFoundException("Distro {} not found".format(distro)) + d = Distribution(distro) # let SeriesNotFoundException flow up d.getSeries(release) @@ -147,89 +169,76 @@ def parse_release(release, distro): return (release, pocket) -def main(): - usage = "Usage: %prog [release[-pocket]|version]" - opt_parser = OptionParser(usage) - opt_parser.add_option('-v', '--verbose', - dest='verbose', default=False, - action='store_true', - help="Print verbose/debug messages") - opt_parser.add_option('-d', '--download-only', - dest='download_only', default=False, - action='store_true', - help="Do not extract the source package") - opt_parser.add_option('-m', '--mirror', dest='mirror', - help='Preferred mirror') - opt_parser.add_option('--no-conf', - dest='no_conf', default=False, action='store_true', - help="Don't read config files or environment " - "variables") - opt_parser.add_option('-a', '--arch', - dest='arch', default=None, - help="Get binary packages for specified architecture " - "(default: {})".format(host_architecture())) - opt_parser.add_option('-p', '--pull', - dest='pull', default=None, - help="What to pull: {} (default: {})" - .format(", ".join(VALID_PULLS), DEFAULT_PULL)) - opt_parser.add_option('-D', '--distro', - dest='distro', default=None, - help="Pull from: {} (default: {})" - .format(", ".join(VALID_DISTROS), DEFAULT_DISTRO)) - (options, args) = opt_parser.parse_args() - if not args: - opt_parser.error("Must specify package name") +def pull(options): + # required options asserted below + # 'release' and 'version' are optional strings + # 'mirror' is optional list of strings + # these are type bool + assert hasattr(options, 'verbose') + assert hasattr(options, 'download_only') + assert hasattr(options, 'no_conf') + # these are type string + assert hasattr(options, 'arch') + assert hasattr(options, 'pull') + assert hasattr(options, 'distro') + assert hasattr(options, 'package') - distro = parse_distro(options.distro) - mirrors = [] + Logger.set_verbosity(options.verbose) - config = UDTConfig(options.no_conf) - if options.mirror is None: - options.mirror = config.get_value(distro.upper() + '_MIRROR') - if options.mirror: - mirrors.append(options.mirror) - - pull = parse_pull(options.pull) - if pull == PULL_DDEBS: - ddebs_mirror = config.get_value(distro.upper() + '_DDEBS_MIRROR') - if ddebs_mirror: - mirrors.append(ddebs_mirror) + Logger.debug("pullpkg options: %s", options) # Login anonymously to LP Launchpad.login_anonymously() - Logger.set_verbosity(options.verbose) + pull = parse_pull(options.pull) - package = str(args[0]).lower() - version = None - release = None + distro = parse_distro(options.distro) + + config = UDTConfig(options.no_conf) + + mirrors = [] + if hasattr(options, 'mirror') and options.mirror: + mirrors += options.mirror + if pull == PULL_DDEBS: + ddebs_mirror = config.get_value(distro.upper() + '_DDEBS_MIRROR') + if ddebs_mirror: + mirrors.append(ddebs_mirror) + if mirrors: + Logger.debug("using mirrors %s", ", ".join(mirrors)) + + package = options.package + release = getattr(options, 'release', None) + version = getattr(options, 'version', None) pocket = None + dscfile = None - if len(args) > 1: + if package.endswith('.dsc') and not release and not version: + dscfile = package + package = None + + if release: try: - (release, pocket) = parse_release(args[1], distro) - if len(args) > 2: - version = args[2] + (release, pocket) = parse_release(release, distro) except (SeriesNotFoundException, PocketDoesNotExistError): - version = args[1] - Logger.debug("Param '%s' not valid series, must be version", version) - if len(args) > 2: + Logger.debug("Param '%s' not valid series, must be version", release) + release, version = version, release + if release: try: - (release, pocket) = parse_release(args[2], distro) + (release, pocket) = parse_release(release, distro) except (SeriesNotFoundException, PocketDoesNotExistError): Logger.error("Can't find series for '%s' or '%s'", - args[1], args[2]) - sys.exit(1) + release, version) + raise try: pkgcls = DISTRO_PKG_CLASS[distro] srcpkg = pkgcls(package=package, version=version, series=release, pocket=pocket, - mirrors=mirrors) + mirrors=mirrors, dscfile=dscfile) spph = srcpkg.lp_spph except PackageNotFoundException as e: Logger.error(str(e)) - sys.exit(1) + raise Logger.normal('Found %s', spph.display_name) @@ -240,40 +249,31 @@ def main(): Logger.normal("Binary files:") for f in spph.getBinaries(options.arch): Logger.normal(" %s", f.getFileName()) - sys.exit(0) + return - try: - if pull == PULL_SOURCE: - srcpkg.pull() - if not options.download_only: - srcpkg.unpack() + # allow DownloadError to flow up to caller + if pull == PULL_SOURCE: + srcpkg.pull() + if options.download_only: + Logger.debug("--download-only specified, not extracting") else: - name = '.*' - if package != spph.getPackageName(): - Logger.normal("Pulling binary package '%s'", package) - Logger.normal("Use package name '%s' to pull all binary packages", - spph.getPackageName()) - name = package - if pull == PULL_DEBS: - name = r'{}(? Date: Thu, 14 Sep 2017 19:59:20 -0400 Subject: [PATCH 16/30] pull-pkg: create pull-[lp|debian|uca]-* scripts for backwards compatibility (i.e. to keep pull-lp-source, pull-debian-source, and pull-uca-source) and for ease of use, create scripts that default the pull-pkg -p and -D params re-create pull-pkg script also, to call ubuntutools/pullpkg.py without any default pull or distro parameters --- debian/control | 8 ++- pull-debian-ddebs | 11 ++++ pull-debian-debs | 11 ++++ pull-debian-source | 147 ++---------------------------------------- pull-debian-udebs | 11 ++++ pull-lp-ddebs | 11 ++++ pull-lp-debs | 11 ++++ pull-lp-source | 11 ++++ pull-lp-udebs | 11 ++++ pull-pkg | 29 +++++++++ pull-uca-ddebs | 11 ++++ pull-uca-debs | 11 ++++ pull-uca-source | 156 ++------------------------------------------- pull-uca-udebs | 11 ++++ setup.py | 10 +++ 15 files changed, 166 insertions(+), 294 deletions(-) create mode 100755 pull-debian-ddebs create mode 100755 pull-debian-debs create mode 100755 pull-debian-udebs create mode 100755 pull-lp-ddebs create mode 100755 pull-lp-debs create mode 100755 pull-lp-source create mode 100755 pull-lp-udebs create mode 100755 pull-pkg create mode 100755 pull-uca-ddebs create mode 100755 pull-uca-debs create mode 100755 pull-uca-udebs diff --git a/debian/control b/debian/control index 1d39a8b..6baef43 100644 --- a/debian/control +++ b/debian/control @@ -104,7 +104,13 @@ Description: useful tools for Ubuntu developers a Debian package and its immediate parent to generate a debdiff. - pull-debian-source - downloads the latest source package available in Debian of a package. - - pull-lp-source - downloads latest source package from Launchpad. + - pull-lp-source - downloads source package from Launchpad. + - pull-lp-debs - downloads debs package(s) from Launchpad. + - pull-lp-ddebs - downloads dbgsym/ddebs package(s) from Launchpad. + - pull-lp-udebs - downloads udebs package(s) from Launchpad. + - pull-debian-* - same as pull-lp-* but for Debian packages. + - pull-uca-* - same as pull-lp-* but for Ubuntu Cloud Archive packages. + - pull-pkg - common script that provides above pull-* functionality. - pull-revu-source - downloads the latest source package from REVU - requestbackport - file a backporting request. - requestsync - files a sync request with Debian changelog and rationale. diff --git a/pull-debian-ddebs b/pull-debian-ddebs new file mode 100755 index 0000000..330f315 --- /dev/null +++ b/pull-debian-ddebs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-debian-ddebs -- pull ddeb package files for debian +# Basic usage: pull-debian-ddebs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='debian', pull='ddebs') diff --git a/pull-debian-debs b/pull-debian-debs new file mode 100755 index 0000000..4f92407 --- /dev/null +++ b/pull-debian-debs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-debian-debs -- pull deb package files for debian +# Basic usage: pull-debian-debs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='debian', pull='debs') diff --git a/pull-debian-source b/pull-debian-source index 6bfc5fb..3ffb2dc 100755 --- a/pull-debian-source +++ b/pull-debian-source @@ -1,148 +1,11 @@ #!/usr/bin/python3 # -# pull-debian-source -- pull a source package from Launchpad -# Copyright (C) 2011, Stefano Rivera -# Inspired by a tool of the same name by Nathan Handler. +# pull-debian-source -- pull source package files for debian +# Basic usage: pull-debian-source [version|release] # -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -import json -import optparse -import sys -import urllib.request -import urllib.error - -from distro_info import DebianDistroInfo, DistroDataOutdated - -from ubuntutools.archive import DebianSourcePackage, DownloadError, rmadison -from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger - - -def is_suite(version): - """If version could be considered to be a Debian suite, return the - canonical suite name. Otherwise None - """ - debian_info = DebianDistroInfo() - debian_releases = debian_info.all + ['experimental'] - - if '-' in version: - release, pocket = version.split('-', 1) - release = debian_info.codename(release, default=release) - if release in debian_releases: - if pocket in ('proposed-updates', 'p-u'): - return (release + '-proposed-updates') - elif pocket == 'security': - return (release + '-security') - else: - release = debian_info.codename(version, default=version) - if release in debian_releases: - return release - return None - - -def source_package_for(binary, release): - """Query DDE to find the source package for a particular binary""" - try: - release = DebianDistroInfo().codename(release, default=release) - except DistroDataOutdated as e: - Logger.warn(e) - url = ('http://dde.debian.net/dde/q/udd/dist/d:debian/r:%s/p:%s/?t=json' - % (release, binary)) - data = None - try: - data = json.load(urllib.request.urlopen(url))['r'] - except urllib.error.URLError as e: - Logger.error('Unable to retrieve package information from DDE: ' - '%s (%s)', url, str(e)) - except ValueError as e: - Logger.error('Unable to parse JSON response from DDE: ' - '%s (%s)', url, str(e)) - if not data: - return None - return data[0]['source'] - - -def main(): - usage = 'Usage: %prog [release|version]' - parser = optparse.OptionParser(usage) - parser.add_option('-d', '--download-only', - dest='download_only', default=False, action='store_true', - help='Do not extract the source package') - parser.add_option('-m', '--mirror', metavar='DEBIAN_MIRROR', - dest='debian_mirror', - help='Preferred Debian mirror (default: %s)' - % UDTConfig.defaults['DEBIAN_MIRROR']) - parser.add_option('-s', '--security-mirror', metavar='DEBSEC_MIRROR', - dest='debsec_mirror', - help='Preferred Debian Security mirror (default: %s)' - % UDTConfig.defaults['DEBSEC_MIRROR']) - parser.add_option('--no-conf', - dest='no_conf', default=False, action='store_true', - help="Don't read config files or environment variables") - parser.add_option('--no-verify-signature', - dest='verify_signature', default=True, - action='store_false', - help="Allow signature verification failure") - (options, args) = parser.parse_args() - if not args: - parser.error('Must specify package name') - elif len(args) > 2: - parser.error('Too many arguments. ' - 'Must only specify package and (optionally) release.') - - config = UDTConfig(options.no_conf) - if options.debian_mirror is None: - options.debian_mirror = config.get_value('DEBIAN_MIRROR') - if options.debsec_mirror is None: - options.debsec_mirror = config.get_value('DEBSEC_MIRROR') - - package = args[0].lower() - - version = args[1] if len(args) > 1 else 'unstable' - component = None - - suite = is_suite(version) - if suite is not None: - line = list(rmadison('debian', package, suite, 'source')) - if not line: - source_package = source_package_for(package, suite) - if source_package is not None and package != source_package: - package = source_package - line = list(rmadison('debian', package, suite, 'source')) - if not line: - Logger.error('Unable to find %s in Debian suite "%s".', package, - suite) - sys.exit(1) - line = line[-1] - version = line['version'] - component = line['component'] - - Logger.normal('Downloading %s version %s', package, version) - srcpkg = DebianSourcePackage(package, version, component=component, - mirrors=[options.debian_mirror, - options.debsec_mirror]) - try: - srcpkg.pull(verify_signature=options.verify_signature) - except DownloadError as e: - Logger.error('Failed to download: %s', str(e)) - sys.exit(1) - if not options.download_only: - srcpkg.unpack() +# See pull-pkg +from ubuntutools.pullpkg import PullPkg if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - Logger.normal('User abort.') + PullPkg.main(distro='debian', pull='source') diff --git a/pull-debian-udebs b/pull-debian-udebs new file mode 100755 index 0000000..b839f88 --- /dev/null +++ b/pull-debian-udebs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-debian-udebs -- pull udeb package files for debian +# Basic usage: pull-debian-udebs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='debian', pull='udebs') diff --git a/pull-lp-ddebs b/pull-lp-ddebs new file mode 100755 index 0000000..16aab20 --- /dev/null +++ b/pull-lp-ddebs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-lp-ddebs -- pull ddeb package files for ubuntu +# Basic usage: pull-lp-ddebs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ubuntu', pull='ddebs') diff --git a/pull-lp-debs b/pull-lp-debs new file mode 100755 index 0000000..33ad826 --- /dev/null +++ b/pull-lp-debs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-lp-debs -- pull deb package files for ubuntu +# Basic usage: pull-lp-debs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ubuntu', pull='debs') diff --git a/pull-lp-source b/pull-lp-source new file mode 100755 index 0000000..6b399fd --- /dev/null +++ b/pull-lp-source @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-lp-source -- pull source package files for ubuntu +# Basic usage: pull-lp-source [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ubuntu', pull='source') diff --git a/pull-lp-udebs b/pull-lp-udebs new file mode 100755 index 0000000..580c65c --- /dev/null +++ b/pull-lp-udebs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-lp-udebs -- pull udeb package files for ubuntu +# Basic usage: pull-lp-udebs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ubuntu', pull='udebs') diff --git a/pull-pkg b/pull-pkg new file mode 100755 index 0000000..8ed38b8 --- /dev/null +++ b/pull-pkg @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +# +# pull-pkg -- pull package files for debian/ubuntu/uca/ppa +# Basic usage: pull-pkg -D distro -p type [version|release] +# +# Copyright (C) 2008, Iain Lane , +# 2010-2011, Stefano Rivera +# 2017-2018, Dan Streetman +# +# ################################################################## +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# See file /usr/share/common-licenses/GPL for more details. +# +# ################################################################## + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main() diff --git a/pull-uca-ddebs b/pull-uca-ddebs new file mode 100755 index 0000000..bd55ac5 --- /dev/null +++ b/pull-uca-ddebs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-uca-ddebs -- pull ddeb package files for ubuntu cloud archive +# Basic usage: pull-uca-ddebs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='uca', pull='ddebs') diff --git a/pull-uca-debs b/pull-uca-debs new file mode 100755 index 0000000..096cf8f --- /dev/null +++ b/pull-uca-debs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-uca-debs -- pull deb package files for ubuntu cloud archive +# Basic usage: pull-uca-debs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='uca', pull='debs') diff --git a/pull-uca-source b/pull-uca-source index 0bd97d2..6aa6676 100755 --- a/pull-uca-source +++ b/pull-uca-source @@ -1,157 +1,11 @@ #!/usr/bin/python3 # -# pull-uca-source -- pull a source package from Ubuntu Cloud Archive -# Basic usage: pull-uca-source [version] +# pull-uca-source -- pull source package files for ubuntu cloud archive +# Basic usage: pull-uca-source [version|release] # -# Copyright (C) 2008, Iain Lane , -# 2010-2011, Stefano Rivera -# 2016, Corey Bryant -# 2016, Dan Streetman -# -# ################################################################## -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 3 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# See file /usr/share/common-licenses/GPL for more details. -# -# ################################################################## - - -import re -import sys -from optparse import OptionParser - -from ubuntutools.archive import UbuntuCloudArchiveSourcePackage, DownloadError -from ubuntutools.config import UDTConfig -from ubuntutools.lp.lpapicache import Launchpad -from ubuntutools.lp.udtexceptions import PocketDoesNotExistError -from ubuntutools.logger import Logger -from ubuntutools.misc import split_release_pocket - -from lazr.restfulclient.errors import NotFound - -from launchpadlib.launchpad import Launchpad as LP - - -def showOpenstackReleases(uca): - releases = [] - for p in uca.ppas: - if re.match(r"\w*-staging", p.name): - releases.append(re.sub("-staging", "", p.name)) - Logger.error("Openstack releases are:\n\t%s", ", ".join(releases)) - - -def getSPPH(lp, archive, package, version=None, series=None, pocket=None, try_binary=True): - params = {'exact_match': True, 'order_by_date': True} - if pocket: - params['pocket'] = pocket - if series: - params['distro_series'] = series() - elif version: - params['version'] = version - Logger.normal("checking %s version %s pocket %s", package, version, pocket) - spphs = archive.getPublishedSources(source_name=package, **params) - if spphs: - return spphs[0] - if not try_binary: - return None - - # Didn't find any, maybe the package is a binary package name - if series: - del params['distro_series'] - archs = lp.load(series().architectures_collection_link).entries - params['distro_arch_series'] = archs[0]['self_link'] - bpphs = archive.getPublishedBinaries(binary_name=package, **params) - if bpphs: - bpph_build = lp.load(bpphs[0].build_link) - source_package = bpph_build.source_package_name - return getSPPH(lp, archive, source_package, version, series, pocket, - try_binary=False) - - return None - - -def main(): - usage = "Usage: %prog [version]" - opt_parser = OptionParser(usage) - opt_parser.add_option('-d', '--download-only', - dest='download_only', default=False, - action='store_true', - help="Do not extract the source package") - opt_parser.add_option('-m', '--mirror', metavar='OPENSTACK_MIRROR', - dest='openstack_mirror', - help='Preferred Openstack mirror (default: Launchpad)') - opt_parser.add_option('--no-conf', - dest='no_conf', default=False, action='store_true', - help="Don't read config files or environment " - "variables") - (options, args) = opt_parser.parse_args() - if len(args) < 2: - opt_parser.error("Must specify package name and openstack release") - - config = UDTConfig(options.no_conf) - if options.openstack_mirror is None: - options.openstack_mirror = config.get_value('OPENSTACK_MIRROR') - mirrors = [] - if options.openstack_mirror: - mirrors.append(options.openstack_mirror) - - # Login anonymously to LP - Launchpad.login_anonymously() - lp = LP.login_anonymously("pull-uca-source", "production") - uca = lp.people("ubuntu-cloud-archive") - - package = str(args[0]).lower() - release = str(args[1]).lower() - version = None - if len(args) > 2: - version = str(args[2]) - - pocket = None - try: - (release, pocket) = split_release_pocket(release, default=None) - except PocketDoesNotExistError: - pass - - try: - archive = uca.getPPAByName(name="%s-staging" % release) - except NotFound: - Logger.error('Archive does not exist for Openstack release: %s', - release) - showOpenstackReleases(uca) - sys.exit(1) - - spph = getSPPH(lp, archive, package, version, pocket=pocket) - if not spph: - Logger.error("Package %s in %s not found.", package, release) - sys.exit(1) - - package = spph.source_package_name - version = spph.source_package_version - component = spph.component_name - Logger.normal('Downloading %s version %s component %s', package, version, component) - srcpkg = UbuntuCloudArchiveSourcePackage(release, package, version, component=component, - mirrors=mirrors) - - try: - srcpkg.pull() - except DownloadError as e: - Logger.error('Failed to download: %s', str(e)) - sys.exit(1) - if not options.download_only: - srcpkg.unpack() +# See pull-pkg +from ubuntutools.pullpkg import PullPkg if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - Logger.normal('User abort.') + PullPkg.main(distro='uca', pull='source') diff --git a/pull-uca-udebs b/pull-uca-udebs new file mode 100755 index 0000000..95b0626 --- /dev/null +++ b/pull-uca-udebs @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +# +# pull-uca-udebs -- pull udeb package files for ubuntu cloud archive +# Basic usage: pull-uca-udebs [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='uca', pull='udebs') diff --git a/setup.py b/setup.py index ce8c966..35fdddc 100755 --- a/setup.py +++ b/setup.py @@ -27,11 +27,21 @@ scripts = [ 'mk-sbuild', 'pbuilder-dist', 'pbuilder-dist-simple', + 'pull-pkg', 'pull-debian-debdiff', 'pull-debian-source', + 'pull-debian-debs', + 'pull-debian-ddebs', + 'pull-debian-udebs', 'pull-lp-source', + 'pull-lp-debs', + 'pull-lp-ddebs', + 'pull-lp-udebs', 'pull-revu-source', 'pull-uca-source', + 'pull-uca-debs', + 'pull-uca-ddebs', + 'pull-uca-udebs', 'requestbackport', 'requestsync', 'reverse-build-depends', From 41a6c47ac2550479f344568686049792544f75aa Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 15 Sep 2017 09:50:29 -0400 Subject: [PATCH 17/30] pull-pkg: update man pages to all point to single pull-pkg.1 man page remove the pull-lp-source.1 and pull-debian-source.1 man pages, consolidating them both into a pull-pkg.1 man page. Also create symlinks for all associated scripts pointing to pull-pkg.1 man page. --- doc/pull-debian-ddebs.1 | 1 + doc/pull-debian-debs.1 | 1 + doc/pull-debian-source.1 | 90 +------------------------- doc/pull-debian-udebs.1 | 1 + doc/pull-lp-ddebs.1 | 1 + doc/pull-lp-debs.1 | 1 + doc/pull-lp-source.1 | 80 +---------------------- doc/pull-lp-udebs.1 | 1 + doc/pull-pkg.1 | 134 +++++++++++++++++++++++++++++++++++++++ doc/pull-uca-ddebs.1 | 1 + doc/pull-uca-debs.1 | 1 + doc/pull-uca-source.1 | 1 + doc/pull-uca-udebs.1 | 1 + 13 files changed, 146 insertions(+), 168 deletions(-) create mode 120000 doc/pull-debian-ddebs.1 create mode 120000 doc/pull-debian-debs.1 mode change 100644 => 120000 doc/pull-debian-source.1 create mode 120000 doc/pull-debian-udebs.1 create mode 120000 doc/pull-lp-ddebs.1 create mode 120000 doc/pull-lp-debs.1 mode change 100644 => 120000 doc/pull-lp-source.1 create mode 120000 doc/pull-lp-udebs.1 create mode 100644 doc/pull-pkg.1 create mode 120000 doc/pull-uca-ddebs.1 create mode 120000 doc/pull-uca-debs.1 create mode 120000 doc/pull-uca-source.1 create mode 120000 doc/pull-uca-udebs.1 diff --git a/doc/pull-debian-ddebs.1 b/doc/pull-debian-ddebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-debian-ddebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-debian-debs.1 b/doc/pull-debian-debs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-debian-debs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-debian-source.1 b/doc/pull-debian-source.1 deleted file mode 100644 index 468724a..0000000 --- a/doc/pull-debian-source.1 +++ /dev/null @@ -1,89 +0,0 @@ -.\" Copyright (C) 2010-2011, Stefano Rivera -.\" -.\" Permission to use, copy, modify, and/or distribute this software for any -.\" purpose with or without fee is hereby granted, provided that the above -.\" copyright notice and this permission notice appear in all copies. -.\" -.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -.\" REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -.\" AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -.\" INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -.\" LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -.\" OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -.\" PERFORMANCE OF THIS SOFTWARE. -.TH PULL\-DEBIAN\-SOURCE "1" "22 January 2011" "ubuntu\-dev\-tools" - -.SH NAME -pull\-debian\-source \- download and extract a source package from Debian - -.SH SYNOPSIS -.B pull\-debian\-source \fR[\fIoptions\fR] <\fIsource package\fR> -[\fIrelease\fR|\fIversion\fR] - -.SH DESCRIPTION -\fBpull\-debian\-source\fR downloads and extracts the specified -\fIversion\fR of \fIsource package\fR, or the latest version in the -specified Debian \fIrelease\fR. -.P -\fBpull\-debian\-source\fR will try the preferred mirror, default -mirror, security mirror, and fall back to \fBLaunchpad\fR or -\fBsnapshot.debian.org\fR, in search of the requested version. - -.SH OPTIONS -.TP -.I source package -The source package to download from Debian. -.TP -.I release -The release to download the source package from. Defaults to -\fBunstable\fR. -.TP -.I version -The specific version of the package to download. -.TP -.BR \-d ", " \-\-download\-only -Do not extract the source package. -.TP -.B \-m \fIDEBIAN_MIRROR\fR, \fB\-\-mirror\fR=\fIDEBIAN_MIRROR\fR -Use the specified mirror. -Should be in the form \fBhttp://ftp.debian.org/debian\fR. -If the package isn't found on this mirror, \fBpull\-debian\-source\fR -will fall back to the default mirror. -.TP -.B \-s \fIDEBSEC_MIRROR\fR, \fB\-\-security\-mirror\fR=\fIDEBSEC_MIRROR\fR -Use the specified mirror. -Should be in the form \fBhttp://security.debian.org\fR. -If the package isn't found on this mirror, \fBpull\-debian\-source\fR -will fall back to the default mirror. -.TP -.B \-\-no\-conf -Do not read any configuration files, or configuration from environment -variables. -.TP -.BR \-h ", " \-\-help -Display the usage instructions and exit. - -.SH ENVIRONMENT -All of the \fBCONFIGURATION VARIABLES\fR below are also supported as -environment variables. -Variables in the environment take precedence to those in configuration -files. - -.SH CONFIGURATION VARIABLES -The following variables can be set in the environment or in -.BR ubuntu\-dev\-tools (5) -configuration files. -In each case, the script\-specific variable takes precedence over the -package\-wide variable. -.TP -.BR PULL_DEBIAN_SOURCE_DEBIAN_MIRROR ", " UBUNTUTOOLS_DEBIAN_MIRROR -The default value for \fB\-\-mirror\fR. -.TP -.BR PULL_DEBIAN_SOURCE_DEBSEC_MIRROR ", " UBUNTUTOOLS_DEBSEC_MIRROR -The default value for \fB\-\-security\-mirror\fR. - -.SH SEE ALSO -.BR dget (1), -.BR pull\-debian\-debdiff (1), -.BR pull\-lp\-source (1), -.BR ubuntu\-dev\-tools (5) diff --git a/doc/pull-debian-source.1 b/doc/pull-debian-source.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-debian-source.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-debian-udebs.1 b/doc/pull-debian-udebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-debian-udebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-lp-ddebs.1 b/doc/pull-lp-ddebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-lp-ddebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-lp-debs.1 b/doc/pull-lp-debs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-lp-debs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-lp-source.1 b/doc/pull-lp-source.1 deleted file mode 100644 index 757857a..0000000 --- a/doc/pull-lp-source.1 +++ /dev/null @@ -1,79 +0,0 @@ -.TH PULL\-LP\-SOURCE "1" "4 August 2008" "ubuntu-dev-tools" - -.SH NAME -pull\-lp\-source \- download a source package from Launchpad - -.SH SYNOPSIS -.B pull\-lp\-source \fR[\fIoptions\fR]\fB \fBsource package\fR -[\fIrelease\fR|\fIversion\fR] - -.SH DESCRIPTION -\fBpull\-lp\-source\fR downloads and extracts the specified -\fIversion\fR of <\fBsource package\fR> from Launchpad, or the latest -version of the specified \fIrelease\fR. -To request a version from a particular pocket say -\fIrelease\fB\-\fIpocket\fR (with a magic \fB\-release\fR for only the -release pocket). -If no \fIversion\fR or \fIrelease\fR is specified, the latest version in -the development release will be downloaded. - -.SH OPTIONS -Listed below are the command line options for pull\-lp\-source: -.TP -.B source package -This is the source package that you would like to be downloaded from Launchpad. -.TP -.B version -This is the version of the source package to be downloaded. -.TP -.B release -This is the release that you would like the source package to be downloaded from. -This value defaults to the current development release. -.TP -.BR \-h ", " \-\-help -Display a help message and exit. -.TP -.BR \-d ", " \-\-download\-only -Do not extract the source package. -.TP -.B \-m \fIUBUNTU_MIRROR\fR, \fB\-\-mirror\fR=\fIUBUNTU_MIRROR\fR -Use the specified Ubuntu mirror. -Should be in the form \fBhttp://archive.ubuntu.com/ubuntu\fR. -If the package isn't found on this mirror, \fBpull\-lp\-source\fR will -fall back to Launchpad, as its name implies. -.TP -.B \-\-no\-conf -Do not read any configuration files, or configuration from environment -variables. - -.SH ENVIRONMENT -All of the \fBCONFIGURATION VARIABLES\fR below are also supported as -environment variables. -Variables in the environment take precedence to those in configuration -files. -.TP -.B -DIST -Specifies the default target. - -.SH CONFIGURATION VARIABLES -The following variables can be set in the environment or in -.BR ubuntu\-dev\-tools (5) -configuration files. -In each case, the script\-specific variable takes precedence over the -package\-wide variable. -.TP -.BR PULL_LP_SOURCE_UBUNTU_MIRROR ", " UBUNTUTOOLS_UBUNTU_MIRROR -The default value for \fB\-\-mirror\fR. - -.SH SEE ALSO -.BR dget (1), -.BR pull\-debian\-source (1), -.BR pull\-debian\-debdiff (1), -.BR ubuntu\-dev\-tools (5) - -.SH AUTHOR -.PP -\fBpull\-lp\-source\fR and this manual page were written by Iain Lane -. -Both are released under the GNU General Public License, version 3 or later. diff --git a/doc/pull-lp-source.1 b/doc/pull-lp-source.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-lp-source.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-lp-udebs.1 b/doc/pull-lp-udebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-lp-udebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-pkg.1 b/doc/pull-pkg.1 new file mode 100644 index 0000000..f62aa11 --- /dev/null +++ b/doc/pull-pkg.1 @@ -0,0 +1,134 @@ +.TH PULL\-PKG "1" "28 August 2017" "ubuntu-dev-tools" + +.SH NAME +pull\-pkg \- download a package for Debian, Ubuntu, or UCA + +.SH SYNOPSIS +.B pull\-pkg \fR[\fIoptions\fR]\fR <\fIpackage name\fR> +[\fIrelease\fR|\fIversion\fR] + +.SH DESCRIPTION +\fBpull\-pkg\fR downloads the specified \fIversion\fR of +<\fIpackage name\fR>, or the latest version from the +specified \fIrelease\fR. To request a version from +a particular pocket say \fIrelease\fB\-\fIpocket\fR (with a magic +\fB\-release\fR for only the release pocket). +If no \fIversion\fR or \fIrelease\fR is specified, the latest version in +the development release will be downloaded. + +There are convenience scripts that set pull type and distribution +appropriately: these are +\fBpull\-lp\-source\fR, \fBpull\-lp\-debs\fR, \fBpull\-lp\-ddebs\fR, +and \fBpull\-lp\-udebs\fR, which all pull Ubuntu packages; +\fBpull\-debian\-source\fR, \fBpull\-debian\-debs\fR, \fBpull\-debian\-ddebs\fR, +and \fBpull\-debian\-udebs\fR, which all pull Debian packages; +and \fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR, +and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages. +Each script pulls the file type in its name, i.e. +\fIsource\fR, \fIdebs\fR, \fIddebs\fR, or \fIudebs\fR. + +.SH OPTIONS +Listed below are the command line options for pull\-pkg: +.TP +.I package name +This is name of the package to downloaded. +You can use either the source package name, or binary package name. +.TP +.I version +This is the version of the package to downloaded. +.TP +.I release +This is the release to downloaded from. +For debian, you can use either the release name like \fBjessie\fR +or \fBsid\fR, or you can use the special release names \fBunstable\fR, +\fBstable\fR, or \fBtesting\fR. +For ubuntu, you can use either the release name like \fBxenial\fR +or the release-pocket like \fBxenial-proposed\fR. +For ubuntu cloud archive (uca) you can use either the uca release +name like \fBmitaka\fR or the ubuntu and uca release names like +\fBtrusty-mitaka\fR. +Defaults to the current development release. +.TP +.BR \-h ", " \-\-help +Display a help message and exit. +.TP +.BR \-v ", " \-\-verbose +Be verbose about what is being done. +.TP +.BR \-d ", " \-\-download\-only +Do not extract the source package (applies only to source packages). +.TP +.B \-m \fIMIRROR\fR, \fB\-\-mirror\fR=\fIMIRROR\fR +Use the specified mirror server. +Should be in the form \fBhttp://archive.ubuntu.com/ubuntu\fR or +\fBhttp://deb.debian.org/debian\fR. If not specified or if the +package is not found on the specified mirror, this will fall +back to the default mirror(s) and/or mirror(s) from environment +variables, and then will fall back to Launchpad or Debian Snapshot. +This can be specified multiple times to try multiple mirrors. +.TP +.B \-\-no\-conf +Do not use mirrors from the default configuration, or from +any environment variables. +.TP +.B \-a \fIARCH\fR, \fB\-\-arch\fR=\fIARCH\fR +Get binary packages from the \fIARCH\fR architecture. +Defaults to the local architecture, if it can be deteected. +.TP +.B \-p \fIPULL\fR, \fB\-\-pull\fR=\fIPULL\fR +What to pull: \fBsource\fR, \fBdebs\fR, \fBddebs\fR, \fBudebs\fR, +or \fBlist\fR. The \fBlist\fR action only lists all a package's +source and binary files, but does not actually download any. +Defaults to \fBsource\fR. +.TP +.B \-D \fIDISTRO\fR, \fB\-\-distro\fR=\fIDISTRO\fR +Pull from: \fBdebian\fR, \fBuca\fR, or \fBubuntu\fR. +\fBlp\fR can be used instead of \fBubuntu\fR. +Any string containing \fBcloud\fR can be used instead of \fBuca\fR. +Deafults to \fBubuntu\fR. + +.SH ENVIRONMENT +All of the \fBCONFIGURATION VARIABLES\fR below are also supported as +environment variables. +Variables in the environment take precedence to those in configuration +files. + +.SH CONFIGURATION VARIABLES +The following variables can be set in the environment or in +.BR ubuntu\-dev\-tools (5) +configuration files. +In each case, the script\-specific variable takes precedence over the +package\-wide variable. +.TP +.BR UBUNTUTOOLS_UBUNTU_MIRROR +The default mirror. +.TP +.BR PULL_PKG_UBUNTU_MIRROR +The default mirror when using the \fBpull\-pkg\fR script. +.TP +.BR PULL_[LP|DEBIAN|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR +The default mirror when using the associated script. + +.SH SEE ALSO +.BR dget (1), +.BR pull\-lp\-source (1), +.BR pull\-lp\-debs (1), +.BR pull\-lp\-ddebs (1), +.BR pull\-lp\-udebs (1), +.BR pull\-debian\-source (1), +.BR pull\-debian\-debs (1), +.BR pull\-debian\-ddebs (1), +.BR pull\-debian\-udebs (1), +.BR pull\-uca\-source (1), +.BR pull\-uca\-debs (1), +.BR pull\-uca\-ddebs (1), +.BR pull\-uca\-udebs (1), +.BR pull\-debian\-debdiff (1), +.BR ubuntu\-dev\-tools (5) + +.SH AUTHOR +.PP +\fBpull\-pkg\fR was written by Dan Streetman , +based on the original \fBpull\-lp\-source\fR; it and this manual page +were written by Iain Lane . +All are released under the GNU General Public License, version 3 or later. diff --git a/doc/pull-uca-ddebs.1 b/doc/pull-uca-ddebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-uca-ddebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-uca-debs.1 b/doc/pull-uca-debs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-uca-debs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-uca-source.1 b/doc/pull-uca-source.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-uca-source.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-uca-udebs.1 b/doc/pull-uca-udebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-uca-udebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file From 3a413760f3761e829b2e6504ada822ed973f5c8e Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 28 Feb 2018 15:47:53 -0500 Subject: [PATCH 18/30] ubuntutools: add pull-* --no-verify-signature option, don't fail if no pub key Change dsc verification to fail only if the public key was available, but signature verification failed. If no public key is available for the dsc, print warning only. (LP: #1700846) Also add pull-* parameter --no-verify-signature to manually prevent failure when signature verification fails. --- ubuntutools/archive.py | 35 +++++++++----- ubuntutools/pullpkg.py | 6 ++- ubuntutools/test/test_archive.py | 82 +++++++++++++++++++++----------- 3 files changed, 82 insertions(+), 41 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 7aa9f19..7cccca4 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -129,7 +129,7 @@ class SourcePackage(object): def __init__(self, package=None, version=None, component=None, dscfile=None, lp=None, mirrors=(), workdir='.', quiet=False, - series=None, pocket=None): + series=None, pocket=None, verify_signature=False): """Can be initialised using either package or dscfile. If package is specified, either the version or series can also be specified; using version will get the specific package version, @@ -149,6 +149,7 @@ class SourcePackage(object): self._use_series = True self._pocket = pocket self._dsc_source = dscfile + self._verify_signature = verify_signature # Cached values: self._distribution = None @@ -321,7 +322,7 @@ class SourcePackage(object): continue yield (bpph.getFileName(), bpph.getUrl(), 0) - def pull_dsc(self, verify_signature=False): + def pull_dsc(self): "Retrieve dscfile and parse" if self._dsc_source: parsed = urlparse(self._dsc_source) @@ -333,16 +334,18 @@ class SourcePackage(object): url = self._lp_url(self.dsc_name) self._download_dsc(url) - self._check_dsc(verify_signature=verify_signature) + self._check_dsc() def _download_dsc(self, url): "Download specified dscfile and parse" parsed = urlparse(url) if parsed.scheme == 'file': - with open(parsed.path, 'r') as f: + Logger.debug("Using dsc file '%s'" % parsed.path) + with open(parsed.path, 'rb') as f: body = f.read() else: try: + Logger.debug("Trying dsc url '%s'" % url) response, body = httplib2.Http().request(url) except httplib2.HttpLib2Error as e: raise DownloadError(e) @@ -351,13 +354,14 @@ class SourcePackage(object): response.reason)) self._dsc = Dsc(body) - def _check_dsc(self, verify_signature=False): + def _check_dsc(self): "Check that the dsc matches what we are expecting" assert self._dsc is not None self.source = self.dsc['Source'] self._version = Version(self.dsc['Version']) valid = False + no_pub_key = False message = None gpg_info = None try: @@ -381,19 +385,24 @@ class SourcePackage(object): % (gpg_info['GOODSIG'][1], gpg_info['GOODSIG'][0])) elif 'VALIDSIG' in gpg_info: message = 'Valid signature by 0x%s' % gpg_info['VALIDSIG'][0] - if verify_signature: + elif 'NO_PUBKEY' in gpg_info: + no_pub_key = True + message = 'Public key not found, could not verify signature' + if self._verify_signature: if valid: Logger.normal(message) + elif no_pub_key: + Logger.warn(message) else: Logger.error(message) raise DownloadError(message) else: Logger.info(message) - def _write_dsc(self, verify_signature=True): + def _write_dsc(self): "Write dsc file to workdir" if self._dsc is None: - self.pull_dsc(verify_signature=verify_signature) + self.pull_dsc() with open(self.dsc_pathname, 'wb') as f: f.write(self.dsc.raw_text) @@ -477,9 +486,9 @@ class SourcePackage(object): return False return True - def pull(self, verify_signature=False): + def pull(self): "Pull into workdir" - self._write_dsc(verify_signature=verify_signature) + self._write_dsc() for entry in self.dsc['Files']: name = entry['name'] for url in self._source_urls(name): @@ -648,10 +657,10 @@ class DebianSourcePackage(SourcePackage): continue yield (f.name, f.getUrl(), f.size) - def pull_dsc(self, verify_signature=True): + def pull_dsc(self): "Retrieve dscfile and parse" try: - super(DebianSourcePackage, self).pull_dsc(verify_signature) + super(DebianSourcePackage, self).pull_dsc() return except DownloadError: pass @@ -666,7 +675,7 @@ class DebianSourcePackage(SourcePackage): break else: raise DownloadError('dsc could not be found anywhere') - self._check_dsc(verify_signature=verify_signature) + self._check_dsc() # Local methods: @property diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index 70fc55a..c159c4a 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -83,6 +83,8 @@ def create_argparser(default_pull=None, default_distro=None, default_arch=None): help='Preferred mirror(s)') parser.add_argument('--no-conf', action='store_true', help="Don't read config files or environment variables") + parser.add_argument('--no-verify-signature', action='store_true', + help="Don't fail if dsc signature can't be verified") parser.add_argument('-a', '--arch', default=default_arch, help=help_default_arch) parser.add_argument('-p', '--pull', default=default_pull, @@ -177,6 +179,7 @@ def pull(options): assert hasattr(options, 'verbose') assert hasattr(options, 'download_only') assert hasattr(options, 'no_conf') + assert hasattr(options, 'no_verify_signature') # these are type string assert hasattr(options, 'arch') assert hasattr(options, 'pull') @@ -234,7 +237,8 @@ def pull(options): pkgcls = DISTRO_PKG_CLASS[distro] srcpkg = pkgcls(package=package, version=version, series=release, pocket=pocket, - mirrors=mirrors, dscfile=dscfile) + mirrors=mirrors, dscfile=dscfile, + verify_signature=(not options.no_verify_signature)) spph = srcpkg.lp_spph except PackageNotFoundException as e: Logger.error(str(e)) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index fb263d2..9483b8d 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -145,11 +145,14 @@ class LocalSourcePackageTestCase(unittest.TestCase): return self.request_404(url) def test_local_copy(self): - pkg = self.SourcePackage('example', '1.0-1', 'main', + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', dscfile='test-data/example_1.0-1.dsc', - workdir=self.workdir) + workdir=self.workdir, + verify_signature=False) pkg.quiet = True - pkg.pull(verify_signature=False) + pkg.pull() pkg.unpack() def test_workdir_srcpkg_noinfo(self): @@ -159,9 +162,10 @@ class LocalSourcePackageTestCase(unittest.TestCase): pkg = self.SourcePackage(dscfile=os.path.join(self.workdir, 'example_1.0-1.dsc'), - workdir=self.workdir) + workdir=self.workdir, + verify_signature=False) pkg.quiet = True - pkg.pull(verify_signature=False) + pkg.pull() pkg.unpack() def test_workdir_srcpkg_info(self): @@ -169,12 +173,14 @@ class LocalSourcePackageTestCase(unittest.TestCase): shutil.copy2('test-data/example_1.0.orig.tar.gz', self.workdir) shutil.copy2('test-data/example_1.0-1.debian.tar.xz', self.workdir) - pkg = self.SourcePackage('example', '1.0-1', 'main', + pkg = self.SourcePackage(package='example', version='1.0-1', + component='main', dscfile=os.path.join(self.workdir, 'example_1.0-1.dsc'), - workdir=self.workdir) + workdir=self.workdir, + verify_signature=False) pkg.quiet = True - pkg.pull(verify_signature=False) + pkg.pull() pkg.unpack() def test_verification(self): @@ -185,19 +191,25 @@ class LocalSourcePackageTestCase(unittest.TestCase): 'r+b') as f: f.write(b'CORRUPTION') - pkg = self.SourcePackage('example', '1.0-1', 'main', + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', dscfile='test-data/example_1.0-1.dsc', - workdir=self.workdir) + workdir=self.workdir, + verify_signature=False) pkg.quiet = True - pkg.pull(verify_signature=False) + pkg.pull() def test_pull(self): - pkg = self.SourcePackage('example', '1.0-1', 'main', - workdir=self.workdir) + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', + workdir=self.workdir, + verify_signature=False) pkg.url_opener = self.url_opener pkg.quiet = True - pkg.pull(verify_signature=False) + pkg.pull() def test_mirrors(self): mirror = 'http://mirror' @@ -209,15 +221,21 @@ class LocalSourcePackageTestCase(unittest.TestCase): url_opener = mock.MagicMock(spec=OpenerDirector) url_opener.open.side_effect = _callable_iter - pkg = self.SourcePackage('example', '1.0-1', 'main', - workdir=self.workdir, mirrors=[mirror]) + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', + workdir=self.workdir, + mirrors=[mirror], + verify_signature=False) pkg.url_opener = url_opener pkg.quiet = True - pkg.pull(verify_signature=False) + pkg.pull() def test_dsc_missing(self): self.mock_http.side_effect = self.request_404 - pkg = self.SourcePackage('example', '1.0-1', 'main', + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', workdir=self.workdir) pkg.quiet = True self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) @@ -243,12 +261,15 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): url_opener = mock.MagicMock(spec=OpenerDirector) url_opener.open.side_effect = _callable_iter - pkg = self.SourcePackage('example', '1.0-1', 'main', - workdir=self.workdir, mirrors=[debian_mirror, - debsec_mirror]) + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', + workdir=self.workdir, + mirrors=[debian_mirror, debsec_mirror], + verify_signature=False) pkg.quiet = True pkg.url_opener = url_opener - pkg.pull(verify_signature=False) + pkg.pull() pkg.unpack() def test_dsc_missing(self): @@ -262,10 +283,14 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): '[GNUPG:] GOODSIG DEADBEEF Joe Developer ' '') - pkg = self.SourcePackage('example', '1.0-1', 'main', - workdir=self.workdir, mirrors=[mirror]) + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', + workdir=self.workdir, + mirrors=[mirror], + verify_signature=False) pkg.url_opener = self.url_opener - pkg.pull(verify_signature=False) + pkg.pull() def test_dsc_badsig(self): mirror = 'http://mirror' @@ -277,8 +302,11 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): mock_gpg_info.return_value = debian.deb822.GpgInfo.from_output( '[GNUPG:] ERRSIG DEADBEEF') - pkg = self.SourcePackage('example', '1.0-1', 'main', - workdir=self.workdir, mirrors=[mirror]) + pkg = self.SourcePackage(package='example', + version='1.0-1', + component='main', + workdir=self.workdir, + mirrors=[mirror]) try: self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) except URLError: From fb750e38bb217e0bd625dbadbd080c04256effb1 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 21 Nov 2017 14:37:42 -0500 Subject: [PATCH 19/30] lpapicache: find binaries from lp Also update getBinaries() to allow retreival by binary name This significantly speeds up binary file lookup for non-published package versions, since we can get the list of binary urls, but then have to look up the BPPH for each of those urls, which is slow. If the user only wants a specific binary package and/or arch, this speeds up getting that, especially for packages with a lot of binary files (like the kernel). --- ubuntutools/archive.py | 40 +++++------ ubuntutools/lp/lpapicache.py | 134 ++++++++++++++++++++++++++++------- 2 files changed, 127 insertions(+), 47 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 7cccca4..d230bf4 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -317,9 +317,7 @@ class SourcePackage(object): yield default_url def _binary_files_info(self, arch, name): - for bpph in self.lp_spph.getBinaries(arch): - if name and not re.match(name, bpph.binary_package_name): - continue + for bpph in self.lp_spph.getBinaries(arch=arch, name=name): yield (bpph.getFileName(), bpph.getUrl(), 0) def pull_dsc(self): @@ -585,13 +583,11 @@ class DebianSPPH(SourcePackagePublishingHistory): """ resource_type = 'source_package_publishing_history' - def getBinaries(self, arch=None): - if not self._binaries: - Logger.normal('Using Snapshot to find binary packages') - srcpkg = Snapshot.getSourcePackage(self.getPackageName(), - version=self.getVersion()) - self._binaries = [b.getBPPH() for b in srcpkg.getBinaryFiles()] - return super(DebianSPPH, self).getBinaries(arch) + def getBinaries(self, arch, name=None): + Logger.normal('Using Snapshot to find binary packages') + srcpkg = Snapshot.getSourcePackage(self.getPackageName(), + version=self.getVersion()) + return srcpkg.getSPPH().getBinaries(arch=arch, name=name) class DebianSourcePackage(SourcePackage): @@ -652,9 +648,7 @@ class DebianSourcePackage(SourcePackage): yield self.snapshot_files[name] def _binary_files_info(self, arch, name): - for f in self.snapshot_package.getBinaryFiles(arch): - if name and not re.match(name, f.package_name): - continue + for f in self.snapshot_package.getBinaryFiles(arch=arch, name=name): yield (f.name, f.getUrl(), f.size) def pull_dsc(self): @@ -924,7 +918,7 @@ class SnapshotSourcePackage(SnapshotPackage): def getAllFiles(self): return self.getFiles() + self.getBinaryFiles() - def getBinaryFiles(self, arch=None): + def getBinaryFiles(self, arch=None, name=None): if not self._binary_files: url = "/mr/package/{}/{}/allfiles".format(self.name, self.version) response = Snapshot.load("{}?fileinfo=1".format(url)) @@ -934,9 +928,12 @@ class SnapshotSourcePackage(SnapshotPackage): r['architecture'], self.name) for b in response['result']['binaries'] for r in b['files']] self._binary_files = files - if not arch: - return list(self._binary_files) - return filter(lambda f: f.isArch(arch), self._binary_files) + bins = self._binary_files.copy() + if arch: + bins = filter(lambda b: b.isArch(arch), bins) + if name: + bins = filter(lambda b: re.match(name, b.name), bins) + return bins def getFiles(self): if not self._files: @@ -970,6 +967,8 @@ class SnapshotBinaryPackage(SnapshotPackage): f = self.getFiles(arch) if not f: return None + if not arch: + raise RuntimeError("Must specify arch") # Can only be 1 binary file for this pkg name/version/arch return f[0].getBPPH() @@ -983,7 +982,7 @@ class SnapshotBinaryPackage(SnapshotPackage): r['architecture'], self.source) for r in response['result']] if not arch: - return list(self._files) + return self._files.copy() return filter(lambda f: f.isArch(arch), self._files) @@ -1145,8 +1144,9 @@ class SnapshotSPPH(object): new_entries.append(str(block)) return ''.join(new_entries) - def getBinaries(self, arch=None): - return [b.getBPPH() for b in self._pkg.getBinaryFiles(arch)] + def getBinaries(self, arch, name=None): + return [b.getBPPH() + for b in self._pkg.getBinaryFiles(arch=arch, name=name)] class SnapshotBPPH(object): diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 3b4c736..f69f456 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -26,6 +26,7 @@ # httplib2.debuglevel = 1 import collections +import re import sys from debian.changelog import Changelog @@ -364,6 +365,7 @@ class Archive(BaseWrapper): series defaults to the current development series if not specified. series must be either a series name string, or DistroArchSeries object. + series may be omitted if version is specified. pocket may be a string or a list. If a list, the highest version will be returned. It defaults to all pockets except backports. @@ -423,16 +425,18 @@ class Archive(BaseWrapper): pass elif series: series = dist.getSeries(series) - else: + elif not version: series = dist.getDevelopmentSeries() if binary: - if arch_series is None: + if arch_series is None and series: arch_series = series.getArchSeries(archtag=archtag) - if archtag is None: + if archtag is None and arch_series: archtag = arch_series.architecture_tag + if archtag is None: + archtag = host_architecture() - index = (name, series.name, archtag, pockets, status, version) + index = (name, getattr(series, 'name', None), archtag, pockets, status, version) if index not in cache: params = { @@ -443,9 +447,9 @@ class Archive(BaseWrapper): if status: params['status'] = status - if binary: + if arch_series: params['distro_arch_series'] = arch_series() - else: + elif series: params['distro_series'] = series() if len(pockets) == 1: @@ -460,9 +464,11 @@ class Archive(BaseWrapper): for record in records: if record.pocket not in pockets: continue - if latest is None or (Version(latest.source_package_version) - < Version(record.source_package_version)): - latest = record + r = wrapper(record) + if binary and archtag and archtag != r.arch: + continue + if latest is None or latest.getVersion() < r.getVersion(): + latest = r if latest is None: if name_key == 'binary_name': @@ -471,19 +477,23 @@ class Archive(BaseWrapper): package_type = "source package" else: package_type = "package" - msg = ("The %s '%s' does not exist in the %s %s archive" % - (package_type, name, dist.display_name, self.name)) + msg = "The %s '%s' " % (package_type, name) + if version: + msg += "version %s " % version + msg += ("does not exist in the %s %s archive" % + (dist.display_name, self.name)) if binary: msg += " for architecture %s" % archtag - pockets = [series.name if pocket == 'Release' - else '%s-%s' % (series.name, pocket.lower()) - for pocket in pockets] - if len(pockets) > 1: - pockets[-2:] = [' or '.join(pockets[-2:])] - msg += " in " + ', '.join(pockets) + if series: + pockets = [series.name if pocket == 'Release' + else '%s-%s' % (series.name, pocket.lower()) + for pocket in pockets] + if len(pockets) > 1: + pockets[-2:] = [' or '.join(pockets[-2:])] + msg += " in " + ', '.join(pockets) raise PackageNotFoundException(msg) - cache[index] = wrapper(latest) + cache[index] = latest return cache[index] def copyPackage(self, source_name, version, from_archive, to_pocket, @@ -557,7 +567,8 @@ class SourcePackagePublishingHistory(BaseWrapper): def __init__(self, *args): self._archive = None self._changelog = None - self._binaries = None + self._binaries = {} + self._have_all_binaries = False self._distro_series = None # Don't share _builds between different # SourcePackagePublishingHistory objects @@ -653,19 +664,88 @@ class SourcePackagePublishingHistory(BaseWrapper): new_entries.append(str(block)) return ''.join(new_entries) - def getBinaries(self, arch=None): + def getBinaries(self, arch, name=None): ''' Returns the resulting BinaryPackagePublishingHistorys. - If arch is specified, only returns BPPH for that arch. + Must specify arch, or use 'all' to get all archs. + If name is specified, only returns BPPH matching that (regex) name. ''' - if self._binaries is None: - self._binaries = [BinaryPackagePublishingHistory(bpph) - for bpph in - self._lpobject.getPublishedBinaries()] if not arch: - return list(self._binaries) + raise RuntimeError("Must specify arch") - return [b for b in self._binaries if b.arch == arch] + # debs with arch 'all' have to be categorized as a specific arch + # so use requested arch if not 'all', or system arch + fallback_arch = arch + if fallback_arch == 'all': + fallback_arch = host_architecture() + + if self._have_all_binaries: + # Great! + pass + elif self.status in ["Pending", "Published"]: + # Published, great! Directly query the list of binaries + binaries = map(BinaryPackagePublishingHistory, + self._lpobject.getPublishedBinaries()) + for b in binaries: + a = b.arch + if a == 'all': + a = fallback_arch + if a not in self._binaries: + self._binaries[a] = {} + self._binaries[a][b.binary_package_name] = b + self._have_all_binaries = True + else: + # Older version, so we have to go the long way :( + print("Please wait, this may take some time...") + archive = self.getArchive() + urls = self.binaryFileUrls() + for url in urls: + # strip out the URL leading text. + filename = url.rsplit('/', 1)[1] + # strip the file suffix + pkgname = filename.rsplit('.', 1)[0] + # split into name, version, arch + (n, v, a) = pkgname.rsplit('_', 2) + if a == 'all': + a = fallback_arch + # Only check the arch requested - saves time + if arch != 'all' and arch != a: + continue + # Only check the name requested - saves time + if name and not re.match(name, n): + continue + # If we already have this BPPH, keep going + if a in self._binaries and n in self._binaries[a]: + continue + # we ignore the version, as it may be missing epoch + # also we can't use series, as some package versions + # span multiple series! (e.g. for different archs) + params = {'name': n, + 'archtag': a, + 'version': self.getVersion()} + try: + bpph = archive.getBinaryPackage(**params) + except PackageNotFoundException: + print("Could not find pkg in archive: %s" % filename) + continue + if a not in self._binaries: + self._binaries[a] = {} + self._binaries[a][n] = bpph + if not name and arch == 'all': + # We must have got them all + self._have_all_binaries = True + + bpphs = [] + if arch == 'all': + for a in self._binaries.values(): + bpphs += a.values() + elif arch in self._binaries: + bpphs = self._binaries[arch].copy().values() + + if name: + bpphs = filter(lambda b: re.match(name, b.binary_package_name), bpphs) + + return bpphs def _fetch_builds(self): '''Populate self._builds with the build records.''' From 3491b0cff920e5c2dfa6e7816e918d42ebf407dd Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 10 Jul 2018 14:20:44 -0400 Subject: [PATCH 20/30] pull-pkg: change pullpkg into class PullPkg instead of pullpkg.py containing a simple method to call, change it into a normal class PullPkg that callers can create and use. --- ubuntutools/archive.py | 12 +- ubuntutools/pullpkg.py | 466 ++++++++++++++++++++++------------------- 2 files changed, 266 insertions(+), 212 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index d230bf4..47a7e6a 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -128,8 +128,7 @@ class SourcePackage(object): spph_class = SourcePackagePublishingHistory def __init__(self, package=None, version=None, component=None, - dscfile=None, lp=None, mirrors=(), workdir='.', quiet=False, - series=None, pocket=None, verify_signature=False): + *args, **kwargs): """Can be initialised using either package or dscfile. If package is specified, either the version or series can also be specified; using version will get the specific package version, @@ -137,6 +136,15 @@ class SourcePackage(object): Specifying only the package with no version or series will get the latest version from the development series. """ + dscfile = kwargs.get('dscfile') + lp = kwargs.get('lp') + mirrors = kwargs.get('mirrors', ()) + workdir = kwargs.get('workdir', '.') + quiet = kwargs.get('quiet', False) + series = kwargs.get('series') + pocket = kwargs.get('pocket') + verify_signature = kwargs.get('verify_signature', False) + assert (package is not None or dscfile is not None) self.source = package diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index c159c4a..9704a6e 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -23,6 +23,8 @@ import re +import errno + from argparse import ArgumentParser from distro_info import DebianDistroInfo @@ -63,221 +65,265 @@ class InvalidPullValueError(ValueError): pass -def create_argparser(default_pull=None, default_distro=None, default_arch=None): - help_default_pull = "What to pull: " + ", ".join(VALID_PULLS) - if default_pull: - help_default_pull += (" (default: %s)" % default_pull) - help_default_distro = "Pull from: " + ", ".join(VALID_DISTROS) - if default_distro: - help_default_distro += (" (default: %s)" % default_distro) - if not default_arch: - default_arch = host_architecture() - help_default_arch = ("Get binary packages for arch (default: %s)" % default_arch) +class PullPkg(object): + """Class used to pull file(s) associated with a specific package""" + @classmethod + def main(cls, *args, **kwargs): + """For use by stand-alone cmdline scripts. - parser = ArgumentParser() - parser.add_argument('-v', '--verbose', action='store_true', - help="Print verbose/debug messages") - parser.add_argument('-d', '--download-only', action='store_true', - help="Do not extract the source package") - parser.add_argument('-m', '--mirror', action='append', - help='Preferred mirror(s)') - parser.add_argument('--no-conf', action='store_true', - help="Don't read config files or environment variables") - parser.add_argument('--no-verify-signature', action='store_true', - help="Don't fail if dsc signature can't be verified") - parser.add_argument('-a', '--arch', default=default_arch, - help=help_default_arch) - parser.add_argument('-p', '--pull', default=default_pull, - help=help_default_pull) - parser.add_argument('-D', '--distro', default=default_distro, - help=help_default_distro) - parser.add_argument('package', help="Package name to pull") - parser.add_argument('release', nargs='?', help="Release to pull from") - parser.add_argument('version', nargs='?', help="Package version to pull") - return parser - - -def parse_pull(pull): - if not pull: - raise InvalidPullValueError("Must specify --pull") - - # allow 'dbgsym' as alias for 'ddebs' - if pull == 'dbgsym': - Logger.debug("Pulling '%s' for '%s'", PULL_DDEBS, pull) - pull = PULL_DDEBS - # assume anything starting with 'bin' means 'debs' - if str(pull).startswith('bin'): - Logger.debug("Pulling '%s' for '%s'", PULL_DEBS, pull) - pull = PULL_DEBS - # verify pull action is valid - if pull not in VALID_PULLS: - raise InvalidPullValueError("Invalid pull action '%s'" % pull) - - return pull - - -def parse_distro(distro): - if not distro: - raise InvalidDistroValueError("Must specify --distro") - - distro = distro.lower() - - # allow 'lp' for 'ubuntu' - if distro == 'lp': - Logger.debug("Using distro '%s' for '%s'", DISTRO_UBUNTU, distro) - distro = DISTRO_UBUNTU - # assume anything with 'cloud' is UCA - if re.match(r'.*cloud.*', distro): - Logger.debug("Using distro '%s' for '%s'", DISTRO_UCA, distro) - distro = DISTRO_UCA - # verify distro is valid - if distro not in VALID_DISTROS: - raise InvalidDistroValueError("Invalid distro '%s'" % distro) - - return distro - - -def parse_release(release, distro): - if distro == DISTRO_UCA: - # UCA is special; it is specified UBUNTURELEASE-UCARELEASE or just - # UCARELEASE. The user could also specify UCARELEASE-POCKET. But UCA - # archives always correspond to only one UBUNTURELEASE, and UCA archives - # have only the Release pocket, so only UCARELEASE matters to us. - for r in release.split('-'): - if r in UbuntuCloudArchiveSourcePackage.getReleaseNames(): - Logger.debug("Using UCA release '%s'", r) - return (r, None) - raise SeriesNotFoundException('UCA release {} not found.'.format(release)) - - # Check if release[-pocket] is specified - (release, pocket) = split_release_pocket(release, default=None) - Logger.debug("Parsed release '%s' pocket '%s'", release, pocket) - - if distro == DISTRO_DEBIAN: - # This converts from the aliases like 'unstable' - debian_info = DebianDistroInfo() - codename = debian_info.codename(release) - if codename: - Logger.normal("Using release '%s' for '%s'", codename, release) - release = codename - - d = Distribution(distro) - - # let SeriesNotFoundException flow up - d.getSeries(release) - - Logger.debug("Using distro '%s' release '%s' pocket '%s'", - distro, release, pocket) - return (release, pocket) - - -def pull(options): - # required options asserted below - # 'release' and 'version' are optional strings - # 'mirror' is optional list of strings - # these are type bool - assert hasattr(options, 'verbose') - assert hasattr(options, 'download_only') - assert hasattr(options, 'no_conf') - assert hasattr(options, 'no_verify_signature') - # these are type string - assert hasattr(options, 'arch') - assert hasattr(options, 'pull') - assert hasattr(options, 'distro') - assert hasattr(options, 'package') - - Logger.set_verbosity(options.verbose) - - Logger.debug("pullpkg options: %s", options) - - # Login anonymously to LP - Launchpad.login_anonymously() - - pull = parse_pull(options.pull) - - distro = parse_distro(options.distro) - - config = UDTConfig(options.no_conf) - - mirrors = [] - if hasattr(options, 'mirror') and options.mirror: - mirrors += options.mirror - if pull == PULL_DDEBS: - ddebs_mirror = config.get_value(distro.upper() + '_DDEBS_MIRROR') - if ddebs_mirror: - mirrors.append(ddebs_mirror) - if mirrors: - Logger.debug("using mirrors %s", ", ".join(mirrors)) - - package = options.package - release = getattr(options, 'release', None) - version = getattr(options, 'version', None) - pocket = None - dscfile = None - - if package.endswith('.dsc') and not release and not version: - dscfile = package - package = None - - if release: + This will handle catching certain exceptions or kbd interrupts, + and printing out (via Logger) the error message, instead of + allowing the exception to flow up to the script. This does + not catch unexpected exceptions, such as internal errors. + On (expected) error, this will call sys.exit(error); + unexpected errors will flow up to the caller. + On success, this simply returns. + """ try: - (release, pocket) = parse_release(release, distro) + cls(*args, **kwargs).pull() + return + except KeyboardInterrupt: + Logger.normal('User abort.') + except (PackageNotFoundException, SeriesNotFoundException, + PocketDoesNotExistError, InvalidDistroValueError) as e: + Logger.error(str(e)) + sys.exit(errno.ENOENT) + + def __init__(self, *args, **kwargs): + self._default_pull = kwargs.get('pull') + self._default_distro = kwargs.get('distro') + self._default_arch = kwargs.get('arch', host_architecture()) + self._parser = None + + @property + def argparser(self): + if self._parser: + return self._parser + + help_default_pull = "What to pull: " + ", ".join(VALID_PULLS) + if self._default_pull: + help_default_pull += (" (default: %s)" % self._default_pull) + help_default_distro = "Pull from: " + ", ".join(VALID_DISTROS) + if self._default_distro: + help_default_distro += (" (default: %s)" % self._default_distro) + help_default_arch = ("Get binary packages for arch") + help_default_arch += ("(default: %s)" % self._default_arch) + + parser = ArgumentParser() + parser.add_argument('-v', '--verbose', action='store_true', + help="Print verbose/debug messages") + parser.add_argument('-d', '--download-only', action='store_true', + help="Do not extract the source package") + parser.add_argument('-m', '--mirror', action='append', + help='Preferred mirror(s)') + parser.add_argument('--no-conf', action='store_true', + help="Don't read config files or environment variables") + parser.add_argument('--no-verify-signature', action='store_true', + help="Don't fail if dsc signature can't be verified") + parser.add_argument('-a', '--arch', default=self._default_arch, + help=help_default_arch) + parser.add_argument('-p', '--pull', default=self._default_pull, + help=help_default_pull) + parser.add_argument('-D', '--distro', default=self._default_distro, + help=help_default_distro) + parser.add_argument('package', help="Package name to pull") + parser.add_argument('release', nargs='?', help="Release to pull from") + parser.add_argument('version', nargs='?', help="Package version to pull") + self._parser = parser + return self._parser + + def parse_pull(self, pull): + if not pull: + raise InvalidPullValueError("Must specify --pull") + + # allow 'dbgsym' as alias for 'ddebs' + if pull == 'dbgsym': + Logger.debug("Pulling '%s' for '%s'", PULL_DDEBS, pull) + pull = PULL_DDEBS + # assume anything starting with 'bin' means 'debs' + if str(pull).startswith('bin'): + Logger.debug("Pulling '%s' for '%s'", PULL_DEBS, pull) + pull = PULL_DEBS + # verify pull action is valid + if pull not in VALID_PULLS: + raise InvalidPullValueError("Invalid pull action '%s'" % pull) + + return pull + + def parse_distro(self, distro): + if not distro: + raise InvalidDistroValueError("Must specify --distro") + + distro = distro.lower() + + # allow 'lp' for 'ubuntu' + if distro == 'lp': + Logger.debug("Using distro '%s' for '%s'", DISTRO_UBUNTU, distro) + distro = DISTRO_UBUNTU + # assume anything with 'cloud' is UCA + if re.match(r'.*cloud.*', distro): + Logger.debug("Using distro '%s' for '%s'", DISTRO_UCA, distro) + distro = DISTRO_UCA + # verify distro is valid + if distro not in VALID_DISTROS: + raise InvalidDistroValueError("Invalid distro '%s'" % distro) + + return distro + + def parse_release(self, distro, release): + if distro == DISTRO_UCA: + # UCA is special; it is specified UBUNTURELEASE-UCARELEASE or just + # UCARELEASE. The user could also specify UCARELEASE-POCKET. But UCA + # archives always correspond to only one UBUNTURELEASE, and UCA archives + # have only the Release pocket, so only UCARELEASE matters to us. + for r in release.split('-'): + if r in UbuntuCloudArchiveSourcePackage.getReleaseNames(): + Logger.debug("Using UCA release '%s'", r) + return (r, None) + raise SeriesNotFoundException('UCA release {} not found.'.format(release)) + + # Check if release[-pocket] is specified + (release, pocket) = split_release_pocket(release, default=None) + Logger.debug("Parsed release '%s' pocket '%s'", release, pocket) + + if distro == DISTRO_DEBIAN: + # This converts from the aliases like 'unstable' + debian_info = DebianDistroInfo() + codename = debian_info.codename(release) + if codename: + Logger.normal("Using release '%s' for '%s'", codename, release) + release = codename + + d = Distribution(distro) + + # let SeriesNotFoundException flow up + d.getSeries(release) + + Logger.debug("Using distro '%s' release '%s' pocket '%s'", + distro, release, pocket) + return (release, pocket) + + def parse_release_and_version(self, distro, release, version, try_swap=True): + # Verify specified release is valid, and params in correct order + pocket = None + try: + (release, pocket) = self.parse_release(distro, release) except (SeriesNotFoundException, PocketDoesNotExistError): - Logger.debug("Param '%s' not valid series, must be version", release) - release, version = version, release - if release: - try: - (release, pocket) = parse_release(release, distro) - except (SeriesNotFoundException, PocketDoesNotExistError): - Logger.error("Can't find series for '%s' or '%s'", - release, version) - raise + if try_swap: + Logger.debug("Param '%s' not valid series, must be version", release) + release, version = version, release + if release: + return self.parse_release_and_version(distro, release, version, False) + else: + Logger.error("Can't find series for '%s' or '%s'", release, version) + raise + return (release, version, pocket) - try: - pkgcls = DISTRO_PKG_CLASS[distro] - srcpkg = pkgcls(package=package, version=version, - series=release, pocket=pocket, - mirrors=mirrors, dscfile=dscfile, - verify_signature=(not options.no_verify_signature)) + def parse_options(self, options): + # if any of these fail, there is a problem with the parser + # they should all be provided, though the optional ones may be None + + # type bool + assert 'download_only' in options + assert 'no_conf' in options + assert 'no_verify_signature' in options + # type string + assert 'pull' in options + assert 'distro' in options + assert 'arch' in options + assert 'package' in options + # type string, optional + assert 'release' in options + assert 'version' in options + # type list of strings, optional + assert 'mirror' in options + + pull = self.parse_pull(options['pull']) + distro = self.parse_distro(options['distro']) + + params = {} + params['package'] = options['package'] + + if options['release']: + (r, v, p) = self.parse_release_and_version(distro, options['release'], + options['version']) + params['series'] = r + params['version'] = v + params['pocket'] = p + + if (params['package'].endswith('.dsc') and not params['series'] and not params['version']): + params['dscfile'] = params['package'] + params.pop('package') + + mirrors = [] + if options['mirror']: + mirrors.append(options['mirror']) + if pull == PULL_DDEBS: + config = UDTConfig(options['no_conf']) + ddebs_mirror = config.get_value(distro.upper() + '_DDEBS_MIRROR') + if ddebs_mirror: + mirrors.append(ddebs_mirror) + if mirrors: + Logger.debug("using mirrors %s", ", ".join(mirrors)) + params['mirrors'] = mirrors + + params['verify_signature'] = not options['no_verify_signature'] + + return (pull, distro, params) + + def pull(self, args=None): + """Pull (download) specified package file(s)""" + options = vars(self.argparser.parse_args(args)) + + assert 'verbose' in options + if options['verbose'] is not None: + Logger.set_verbosity(options['verbose']) + + Logger.debug("pullpkg options: %s", options) + + # Login anonymously to LP + Launchpad.login_anonymously() + + (pull, distro, params) = self.parse_options(options) + + # call implementation, and allow exceptions to flow up to caller + srcpkg = DISTRO_PKG_CLASS[distro](**params) spph = srcpkg.lp_spph - except PackageNotFoundException as e: - Logger.error(str(e)) - raise - Logger.normal('Found %s', spph.display_name) + Logger.normal('Found %s', spph.display_name) - if pull == PULL_LIST: - Logger.normal("Source files:") - for f in srcpkg.dsc['Files']: - Logger.normal(" %s", f['name']) - Logger.normal("Binary files:") - for f in spph.getBinaries(options.arch): - Logger.normal(" %s", f.getFileName()) - return - - # allow DownloadError to flow up to caller - if pull == PULL_SOURCE: - srcpkg.pull() - if options.download_only: - Logger.debug("--download-only specified, not extracting") + if pull == PULL_LIST: + Logger.normal("Source files:") + for f in srcpkg.dsc['Files']: + Logger.normal(" %s", f['name']) + Logger.normal("Binary files:") + for f in spph.getBinaries(options['arch']): + Logger.normal(" %s", f.getFileName()) + elif pull == PULL_SOURCE: + # allow DownloadError to flow up to caller + srcpkg.pull() + if options['download_only']: + Logger.debug("--download-only specified, not extracting") + else: + srcpkg.unpack() else: - srcpkg.unpack() - else: - name = '.*' - if package != spph.getPackageName(): - Logger.normal("Pulling only binary package '%s'", package) - Logger.normal("Use package name '%s' to pull all binary packages", - spph.getPackageName()) - name = package - if pull == PULL_DEBS: - name = r'{}(? Date: Tue, 10 Jul 2018 14:23:21 -0400 Subject: [PATCH 21/30] pull-pkg: add pull-ppa-* functionality Add functionality, and frontend pull-ppa-* scripts, to be able to pull from PPA archives. --- doc/pull-pkg.1 | 32 ++++++++++++++------- doc/pull-ppa-ddebs.1 | 1 + doc/pull-ppa-debs.1 | 1 + doc/pull-ppa-source.1 | 1 + doc/pull-ppa-udebs.1 | 1 + pull-ppa-ddebs | 12 ++++++++ pull-ppa-debs | 12 ++++++++ pull-ppa-source | 12 ++++++++ pull-ppa-udebs | 12 ++++++++ setup.py | 4 +++ ubuntutools/archive.py | 44 +++++++++++++++++++++++++---- ubuntutools/lp/lpapicache.py | 10 +++++-- ubuntutools/pullpkg.py | 54 ++++++++++++++++++++++++++++++++++-- 13 files changed, 176 insertions(+), 20 deletions(-) create mode 120000 doc/pull-ppa-ddebs.1 create mode 120000 doc/pull-ppa-debs.1 create mode 120000 doc/pull-ppa-source.1 create mode 120000 doc/pull-ppa-udebs.1 create mode 100755 pull-ppa-ddebs create mode 100755 pull-ppa-debs create mode 100755 pull-ppa-source create mode 100755 pull-ppa-udebs diff --git a/doc/pull-pkg.1 b/doc/pull-pkg.1 index f62aa11..08e4f68 100644 --- a/doc/pull-pkg.1 +++ b/doc/pull-pkg.1 @@ -1,7 +1,7 @@ .TH PULL\-PKG "1" "28 August 2017" "ubuntu-dev-tools" .SH NAME -pull\-pkg \- download a package for Debian, Ubuntu, or UCA +pull\-pkg \- download a package for Debian, Ubuntu, UCA, or a PPA .SH SYNOPSIS .B pull\-pkg \fR[\fIoptions\fR]\fR <\fIpackage name\fR> @@ -22,9 +22,11 @@ appropriately: these are and \fBpull\-lp\-udebs\fR, which all pull Ubuntu packages; \fBpull\-debian\-source\fR, \fBpull\-debian\-debs\fR, \fBpull\-debian\-ddebs\fR, and \fBpull\-debian\-udebs\fR, which all pull Debian packages; -and \fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR, -and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages. -Each script pulls the file type in its name, i.e. +\fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR, +and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages; +and \fBpull\-ppa\-source\fR, \fBpull\-ppa\-debs\fR, \fBpull\-ppa\-ddebs\fR, +and \fBpull\-ppa\-udebs\fR, which all pull from a specified Personal Package +Archive on Launchpad. Each script pulls the file type in its name, i.e. \fIsource\fR, \fIdebs\fR, \fIddebs\fR, or \fIudebs\fR. .SH OPTIONS @@ -46,8 +48,7 @@ For ubuntu, you can use either the release name like \fBxenial\fR or the release-pocket like \fBxenial-proposed\fR. For ubuntu cloud archive (uca) you can use either the uca release name like \fBmitaka\fR or the ubuntu and uca release names like -\fBtrusty-mitaka\fR. -Defaults to the current development release. +\fBtrusty-mitaka\fR. Defaults to the current development release. .TP .BR \-h ", " \-\-help Display a help message and exit. @@ -82,10 +83,17 @@ source and binary files, but does not actually download any. Defaults to \fBsource\fR. .TP .B \-D \fIDISTRO\fR, \fB\-\-distro\fR=\fIDISTRO\fR -Pull from: \fBdebian\fR, \fBuca\fR, or \fBubuntu\fR. +Pull from: \fBdebian\fR, \fBuca\fR, \fBubuntu\fR, or a \fBppa\fR. \fBlp\fR can be used instead of \fBubuntu\fR. Any string containing \fBcloud\fR can be used instead of \fBuca\fR. -Deafults to \fBubuntu\fR. +If pulling from a ppa, you must specify the PPA. Deafults to \fBubuntu\fR. +.TP +.B \-\-ppa\fR=ppa:\fIUSER/NAME\fR +Applies only when \fBdistro\fR is \fIppa\fR. Can be provided either as +a value to the \fB\-\-ppa\fR option parameter, or as a plain option +(like \fIrelease\fR or \fIversion\fR). When specified as a plain option, +the form must be \fBppa:USER/NAME\fR; when specified as a value to the +\fB\-\-ppa\fR option parameter, the leading \fBppa:\fR is optional. .SH ENVIRONMENT All of the \fBCONFIGURATION VARIABLES\fR below are also supported as @@ -106,7 +114,7 @@ The default mirror. .BR PULL_PKG_UBUNTU_MIRROR The default mirror when using the \fBpull\-pkg\fR script. .TP -.BR PULL_[LP|DEBIAN|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR +.BR PULL_[LP|DEBIAN|PPA|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR The default mirror when using the associated script. .SH SEE ALSO @@ -119,6 +127,10 @@ The default mirror when using the associated script. .BR pull\-debian\-debs (1), .BR pull\-debian\-ddebs (1), .BR pull\-debian\-udebs (1), +.BR pull\-ppa\-source (1), +.BR pull\-ppa\-debs (1), +.BR pull\-ppa\-ddebs (1), +.BR pull\-ppa\-udebs (1), .BR pull\-uca\-source (1), .BR pull\-uca\-debs (1), .BR pull\-uca\-ddebs (1), @@ -128,7 +140,7 @@ The default mirror when using the associated script. .SH AUTHOR .PP -\fBpull\-pkg\fR was written by Dan Streetman , +\fBpull\-pkg\fR was written by Dan Streetman , based on the original \fBpull\-lp\-source\fR; it and this manual page were written by Iain Lane . All are released under the GNU General Public License, version 3 or later. diff --git a/doc/pull-ppa-ddebs.1 b/doc/pull-ppa-ddebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-ddebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-ppa-debs.1 b/doc/pull-ppa-debs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-debs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-ppa-source.1 b/doc/pull-ppa-source.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-source.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-ppa-udebs.1 b/doc/pull-ppa-udebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-udebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/pull-ppa-ddebs b/pull-ppa-ddebs new file mode 100755 index 0000000..8b9c3cf --- /dev/null +++ b/pull-ppa-ddebs @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-ddebs -- pull ddeb package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-ddebs [version|release] +# pull-ppa-ddebs --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='ddebs') diff --git a/pull-ppa-debs b/pull-ppa-debs new file mode 100755 index 0000000..0804e29 --- /dev/null +++ b/pull-ppa-debs @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-debs -- pull deb package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-debs [version|release] +# pull-ppa-debs --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='debs') diff --git a/pull-ppa-source b/pull-ppa-source new file mode 100755 index 0000000..065bb2b --- /dev/null +++ b/pull-ppa-source @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-source -- pull source package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-source [version|release] +# pull-ppa-source --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='source') diff --git a/pull-ppa-udebs b/pull-ppa-udebs new file mode 100755 index 0000000..4e59c3a --- /dev/null +++ b/pull-ppa-udebs @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-udebs -- pull udeb package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-udebs [version|release] +# pull-ppa-udebs --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='udebs') diff --git a/setup.py b/setup.py index 35fdddc..7078931 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,10 @@ scripts = [ 'pull-lp-debs', 'pull-lp-ddebs', 'pull-lp-udebs', + 'pull-ppa-source', + 'pull-ppa-debs', + 'pull-ppa-ddebs', + 'pull-ppa-udebs', 'pull-revu-source', 'pull-uca-source', 'pull-uca-debs', diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 47a7e6a..3c78d82 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -719,6 +719,38 @@ class UbuntuSourcePackage(SourcePackage): distribution = 'ubuntu' +class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage): + "Download / unpack an Ubuntu Personal Package Archive source package" + def __init__(self, *args, **kwargs): + super(PersonalPackageArchiveSourcePackage, self).__init__(*args, **kwargs) + assert 'ppa' in kwargs + ppa = kwargs['ppa'].split('/') + if len(ppa) != 2: + raise ValueError('Invalid PPA value "%s",' + 'must be "/"' % kwargs['ppa']) + self._ppateam = ppa[0] + self._ppaname = ppa[1] + self.masters = [] + self._team = None + self._ppa = None + + def getArchive(self): + if not self._ppa: + try: + self._team = PersonTeam.fetch(self._ppateam) + except KeyError: + raise ValueError('No user/team "%s" found on Launchpad' % self._ppateam) + self._ppa = self._team.getPPAByName(self._ppaname) + Logger.debug('Using PPA %s' % self._ppa.web_link) + return self._ppa + + def _lp_url(self, filename): + "Build a source package URL on Launchpad" + return os.path.join('https://launchpad.net', '~' + self._ppateam, + '+archive', self.distribution, self._ppaname, + '+files', filename) + + class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): "Download / unpack an Ubuntu Cloud Archive source package" _ppas = None @@ -733,10 +765,12 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): @classmethod def getArchives(cls): - if not cls._ppas: - ppas = filter(lambda p: p.name.endswith('-staging'), - PersonTeam.fetch('ubuntu-cloud-archive').getPPAs()) - cls._ppas = sorted(ppas, key=lambda p: p.name, reverse=True) + if cls._ppas is None: + cls._ppas = [] + ppas = PersonTeam.fetch('ubuntu-cloud-archive').getPPAs() + for key in ppas.keys(): + if key.endswith('-staging'): + cls._ppas.append(ppas[key]) return cls._ppas @classmethod @@ -747,7 +781,7 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): @classmethod def getDevelopmentRelease(cls): - return cls.getReleaseNames()[0] + return sorted(cls.getReleaseNames(), reverse=True)[0] @property def uca_release(self): diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index f69f456..5215108 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -1005,11 +1005,15 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam): return canUpload def getPPAs(self): - if not self._ppas: - ppas = Launchpad.load(self._lpobject.ppas_collection_link).entries - self._ppas = [Archive(a['self_link']) for a in ppas] + if self._ppas is None: + ppas = [Archive(ppa['self_link']) for ppa in + Launchpad.load(self._lpobject.ppas_collection_link).entries] + self._ppas = {ppa.name: ppa for ppa in ppas} return self._ppas + def getPPAByName(self, name): + return self._lpobject.getPPAByName(name=name) + class Build(BaseWrapper): ''' diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index 9704a6e..ee7939e 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -23,6 +23,7 @@ import re +import sys import errno from argparse import ArgumentParser @@ -30,7 +31,8 @@ from argparse import ArgumentParser from distro_info import DebianDistroInfo from ubuntutools.archive import (UbuntuSourcePackage, DebianSourcePackage, - UbuntuCloudArchiveSourcePackage) + UbuntuCloudArchiveSourcePackage, + PersonalPackageArchiveSourcePackage) from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Distribution, Launchpad) from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, @@ -51,11 +53,13 @@ VALID_PULLS = [PULL_SOURCE, PULL_DEBS, PULL_DDEBS, PULL_UDEBS, PULL_LIST] DISTRO_DEBIAN = 'debian' DISTRO_UBUNTU = 'ubuntu' DISTRO_UCA = 'uca' +DISTRO_PPA = 'ppa' DISTRO_PKG_CLASS = { DISTRO_DEBIAN: DebianSourcePackage, DISTRO_UBUNTU: UbuntuSourcePackage, DISTRO_UCA: UbuntuCloudArchiveSourcePackage, + DISTRO_PPA: PersonalPackageArchiveSourcePackage, } VALID_DISTROS = DISTRO_PKG_CLASS.keys() @@ -94,6 +98,7 @@ class PullPkg(object): self._default_distro = kwargs.get('distro') self._default_arch = kwargs.get('arch', host_architecture()) self._parser = None + self._ppa_parser = None @property def argparser(self): @@ -126,12 +131,41 @@ class PullPkg(object): help=help_default_pull) parser.add_argument('-D', '--distro', default=self._default_distro, help=help_default_distro) + parser.add_argument('--ppa', help='PPA to pull from') parser.add_argument('package', help="Package name to pull") parser.add_argument('release', nargs='?', help="Release to pull from") parser.add_argument('version', nargs='?', help="Package version to pull") self._parser = parser return self._parser + def parse_ppa_args(self, args): + """When pulling from PPA, convert from bare ppa:USER/NAME to --ppa option""" + if not args: + myargs = sys.argv[1:] + + options = vars(self.argparser.parse_known_args(myargs)[0]) + # we use these, which all should be always provided by the parser, + # even if their value is None + assert 'distro' in options + assert 'ppa' in options + assert 'release' in options + assert 'version' in options + + # if we're not pulling from a PPA, or if we are but --ppa was given, + # then no change to the args is needed + if options['distro'] != DISTRO_PPA or options['ppa'] is not None: + return args + + # check if release or version is a ppa: + # if it is, move it to a --ppa param + for param in ['release', 'version']: + if str(options[param]).startswith('ppa:'): + myargs.remove(options[param]) + myargs.insert(0, options[param]) + myargs.insert(0, '--ppa') + + return myargs + def parse_pull(self, pull): if not pull: raise InvalidPullValueError("Must specify --pull") @@ -194,7 +228,11 @@ class PullPkg(object): Logger.normal("Using release '%s' for '%s'", codename, release) release = codename - d = Distribution(distro) + if distro == DISTRO_PPA: + # PPAs are part of Ubuntu distribution + d = Distribution(DISTRO_UBUNTU) + else: + d = Distribution(distro) # let SeriesNotFoundException flow up d.getSeries(release) @@ -233,6 +271,7 @@ class PullPkg(object): assert 'arch' in options assert 'package' in options # type string, optional + assert 'ppa' in options assert 'release' in options assert 'version' in options # type list of strings, optional @@ -255,6 +294,14 @@ class PullPkg(object): params['dscfile'] = params['package'] params.pop('package') + if options['ppa']: + if options['ppa'].startswith('ppa:'): + params['ppa'] = options['ppa'][4:] + else: + params['ppa'] = options['ppa'] + elif distro == DISTRO_PPA: + raise ValueError('Must specify PPA to pull from') + mirrors = [] if options['mirror']: mirrors.append(options['mirror']) @@ -273,6 +320,9 @@ class PullPkg(object): def pull(self, args=None): """Pull (download) specified package file(s)""" + # pull-ppa-* may need conversion from ppa:USER/NAME to --ppa USER/NAME + args = self.parse_ppa_args(args) + options = vars(self.argparser.parse_args(args)) assert 'verbose' in options From b11b83f0e2f22ffd5406b54d21b2f0ca289aea04 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 10 Jul 2018 15:32:57 -0400 Subject: [PATCH 22/30] pull-pkg: change UCA source pkg class to subclass of PPA source pkg class The UCA source pkg class was its own specific class, but with the addition of the generic PPA source pkg class, the UCA class is just a specific case of PPA. Changing it to a subclass of the PPA class simplifies code. --- ubuntutools/archive.py | 62 ++++++++++++++---------------------------- ubuntutools/pullpkg.py | 2 +- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 3c78d82..6e8e8a0 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -751,59 +751,37 @@ class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage): '+files', filename) -class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): +class UbuntuCloudArchiveSourcePackage(PersonalPackageArchiveSourcePackage): "Download / unpack an Ubuntu Cloud Archive source package" - _ppas = None - _ppa_names = None + _ppateam = 'ubuntu-cloud-archive' def __init__(self, *args, **kwargs): + series = kwargs.get('series') or UbuntuCloudArchiveSourcePackage.getDevelSeries() + kwargs.pop('series', None) + kwargs['ppa'] = ('%s/%s-staging' % + (UbuntuCloudArchiveSourcePackage._ppateam, series)) super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs) - self._use_series = False # UCA doesn't really use distro series - self._uca_release = self._series - self._series = None + self._use_series = False # each UCA series is for a single Ubuntu series + self.uca_release = series self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"] @classmethod - def getArchives(cls): - if cls._ppas is None: - cls._ppas = [] - ppas = PersonTeam.fetch('ubuntu-cloud-archive').getPPAs() - for key in ppas.keys(): - if key.endswith('-staging'): - cls._ppas.append(ppas[key]) - return cls._ppas + def getDevelSeries(cls): + ppas = PersonTeam.fetch(cls._ppateam).getPPAs() + for key in sorted(ppas.keys(), reverse=True): + if key.endswith('-staging'): + return key.rsplit('-', 1)[0] + raise SeriesNotFoundException('Internal Error: No UCA devel series found...?') @classmethod - def getReleaseNames(cls): - if not cls._ppa_names: - cls._ppa_names = [p.name.split('-', 1)[0] for p in cls.getArchives()] - return cls._ppa_names - - @classmethod - def getDevelopmentRelease(cls): - return sorted(cls.getReleaseNames(), reverse=True)[0] - - @property - def uca_release(self): - if not self._uca_release: - self._uca_release = self.getDevelopmentRelease() - Logger.normal('Using UCA release %s', self._uca_release) - return self._uca_release + def isValidRelease(cls, release): + return ('%s-staging' % release) in PersonTeam.fetch(cls._ppateam).getPPAs() def getArchive(self): - ppas = {p.name: p for p in self.getArchives()} - release = '{}-staging'.format(self.uca_release) - if release in ppas: - Logger.debug('UCA release {} at {}'.format(self.uca_release, - ppas[release]())) - return ppas[release] - raise SeriesNotFoundException('UCA release {} not found.'.format(self.uca_release)) - - def _lp_url(self, filename): - "Build a source package URL on Launchpad" - return os.path.join('https://launchpad.net', "~ubuntu-cloud-archive", - '+archive', ("%s-staging" % self.uca_release), - '+files', filename) + try: + return super(UbuntuCloudArchiveSourcePackage, self).getArchive() + except ValueError: + raise SeriesNotFoundException('UCA release {} not found.'.format(self.uca_release)) class _WebJSON(object): diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index ee7939e..e7064a5 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -211,7 +211,7 @@ class PullPkg(object): # archives always correspond to only one UBUNTURELEASE, and UCA archives # have only the Release pocket, so only UCARELEASE matters to us. for r in release.split('-'): - if r in UbuntuCloudArchiveSourcePackage.getReleaseNames(): + if UbuntuCloudArchiveSourcePackage.isValidRelease(r): Logger.debug("Using UCA release '%s'", r) return (r, None) raise SeriesNotFoundException('UCA release {} not found.'.format(release)) From 0f61836b108668c09af5d75c3a84cfb2d3a3bd06 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 24 Jul 2018 16:28:25 -0400 Subject: [PATCH 23/30] ubuntutools: update archive/lpapicache to optionally search all series For PPA and UCA repos, the latest build is not necessarily in the 'development' release; so for those SourcePackage classes, search all the 'active' series (starting with latest devel) for any matches to the provided package name. This allows not having to specify the series name when looking in PPA/UCA repos for the 'latest' version of a specific package. --- ubuntutools/archive.py | 105 ++++++------- ubuntutools/lp/lpapicache.py | 291 +++++++++++++++++++++++------------ 2 files changed, 242 insertions(+), 154 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 6e8e8a0..9e857e6 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -46,8 +46,7 @@ from contextlib import closing from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, - SourcePackagePublishingHistory, - BinaryPackagePublishingHistory) + SourcePackagePublishingHistory) from ubuntutools.lp.udtexceptions import (PackageNotFoundException, SeriesNotFoundException, InvalidDistroValueError) @@ -154,7 +153,6 @@ class SourcePackage(object): self.workdir = workdir self.quiet = quiet self._series = series - self._use_series = True self._pocket = pocket self._dsc_source = dscfile self._verify_signature = verify_signature @@ -188,65 +186,61 @@ class SourcePackage(object): else: Launchpad.login_anonymously() - distro = self.getDistribution() archive = self.getArchive() - series = None - params = {'exact_match': True, 'order_by_date': True} + params = {} if self._version: # if version was specified, use that params['version'] = self._version.full_version - elif self._use_series: - if self._series: - # if version not specified, get the latest from this series - series = distro.getSeries(self._series) - else: - # if no version or series, get the latest from devel series - series = distro.getDevelopmentSeries() - params['distro_series'] = series() + elif self._series: + # if version not specified, get the latest from this series + params['series'] = self._series + # note that if not specified, pocket defaults to all EXCEPT -backports if self._pocket: params['pocket'] = self._pocket - spphs = archive.getPublishedSources(source_name=self.source, **params) - if spphs: - self._spph = self.spph_class(spphs[0]) + else: + # We always want to search all series, if not specified + params['search_all_series'] = True + + try: + self._spph = archive.getSourcePackage(self.source, + wrapper=self.spph_class, + **params) return self._spph + except PackageNotFoundException as pnfe: + if not self.try_binary or self.binary: + # either we don't need to bother trying binary name lookup, + # or we've already tried + raise pnfe - if self.try_binary and not self.binary: - if series: - arch_series = series.getArchSeries() - params['distro_arch_series'] = arch_series() - del params['distro_series'] - bpphs = archive.getPublishedBinaries(binary_name=self.source, **params) - if bpphs: - bpph = BinaryPackagePublishingHistory(bpphs[0]) - self.binary = self.source - self.source = bpph.getSourcePackageName() - Logger.normal("Using source package '{}' for binary package '{}'" - .format(self.source, self.binary)) - try: - spph = bpph.getBuild().getSourcePackagePublishingHistory() - except Exception: - spph = None - if spph: - self._spph = spph - return self._spph - else: - # binary build didn't include source link, unfortunately - # so try again with the updated self.source name - if not self._version: - # Get version first if user didn't specify it, as some - # binaries have their version hardcoded in their name, - # such as the kernel package - self._version = Version(bpph.getVersion()) - return self.lp_spph + Logger.normal('Source package lookup failed, ' + 'trying lookup of binary package %s' % self.source) - msg = "No {} package found".format(self.source) - if self._version: - msg += " for version {}".format(self._version.full_version) - elif series: - msg += " in series {}".format(series.name) - if self._pocket: - msg += " pocket {}".format(self._pocket) - raise PackageNotFoundException(msg) + try: + bpph = archive.getBinaryPackage(self.source, **params) + except PackageNotFoundException as bpnfe: + # log binary lookup failure, in case it provides hints + Logger.normal(str(bpnfe)) + # raise the original exception for the source lookup + raise pnfe + + self.binary = self.source + self.source = bpph.getSourcePackageName() + Logger.normal("Using source package '{}' for binary package '{}'" + .format(self.source, self.binary)) + + spph = bpph.getBuild().getSourcePackagePublishingHistory() + if spph: + self._spph = self.spph_class(spph.self_link) + return self._spph + + # binary build didn't include source link, unfortunately + # so try again with the updated self.source name + if not self._version: + # Get version first if user didn't specify it, as some + # binaries have their version hardcoded in their name, + # such as the kernel package + self._version = Version(bpph.getVersion()) + return self.lp_spph @property def version(self): @@ -761,7 +755,6 @@ class UbuntuCloudArchiveSourcePackage(PersonalPackageArchiveSourcePackage): kwargs['ppa'] = ('%s/%s-staging' % (UbuntuCloudArchiveSourcePackage._ppateam, series)) super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs) - self._use_series = False # each UCA series is for a single Ubuntu series self.uca_release = series self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"] @@ -948,7 +941,7 @@ class SnapshotSourcePackage(SnapshotPackage): r['architecture'], self.name) for b in response['result']['binaries'] for r in b['files']] self._binary_files = files - bins = self._binary_files.copy() + bins = list(self._binary_files) if arch: bins = filter(lambda b: b.isArch(arch), bins) if name: @@ -1002,7 +995,7 @@ class SnapshotBinaryPackage(SnapshotPackage): r['architecture'], self.source) for r in response['result']] if not arch: - return self._files.copy() + return list(self._files) return filter(lambda f: f.isArch(arch), self._files) diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 5215108..5063c7e 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -37,7 +37,8 @@ from lazr.restfulclient.resource import Entry from ubuntutools.version import Version from ubuntutools.lp import (service, api_version) -from ubuntutools.misc import host_architecture +from ubuntutools.misc import (host_architecture, + DEFAULT_POCKETS, POCKETS) from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError, ArchiveNotFoundException, ArchSeriesNotFoundException, @@ -59,8 +60,6 @@ __all__ = [ 'SourcePackagePublishingHistory', ] -_POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports') - class _Launchpad(object): '''Singleton for LP API access.''' @@ -191,15 +190,23 @@ class Distribution(BaseWrapper): resource_type = 'distribution' def __init__(self, *args): - # Don't share _series and _archives between different Distributions - if '_series' not in self.__dict__: - self._series = dict() - if '_archives' not in self.__dict__: - self._archives = dict() + self._archives = dict() + self._series_by_name = dict() + self._series = dict() + self._dev_series = None + self._have_all_series = False def cache(self): self._cache[self.name] = self + def _cache_series(self, series): + ''' + Add the DistroSeries to the cache if needed. + ''' + if series.version not in self._series: + self._series_by_name[series.name] = series + self._series[series.version] = series + @classmethod def fetch(cls, dist): ''' @@ -246,28 +253,49 @@ class Distribution(BaseWrapper): (e.g. 'karmic') or version (e.g. '9.10'). If the series is not found: raise SeriesNotFoundException ''' - if name_or_version not in self._series: - try: - series = DistroSeries(self().getSeries(name_or_version=name_or_version)) - # Cache with name and version - self._series[series.name] = series - self._series[series.version] = series - except HTTPError: - message = "Release '%s' is unknown in '%s'." % \ - (name_or_version, self.display_name) - raise SeriesNotFoundException(message) - return self._series[name_or_version] + if name_or_version in self._series: + return self._series[name_or_version] + if name_or_version in self._series_by_name: + return self._series_by_name[name_or_version] + + try: + series = DistroSeries(self().getSeries(name_or_version=name_or_version)) + except HTTPError: + message = "Release '%s' is unknown in '%s'." % \ + (name_or_version, self.display_name) + raise SeriesNotFoundException(message) + + self._cache_series(series) + return series def getDevelopmentSeries(self): ''' Returns a DistroSeries object of the current development series. ''' - dev = DistroSeries(self.current_series_link) - # Cache it in _series if not already done - if dev.name not in self._series: - self._series[dev.name] = dev - self._series[dev.version] = dev - return dev + if not self._dev_series: + series = DistroSeries(self.current_series_link) + self._cache_series(series) + self._dev_series = series + return self._dev_series + + def getAllSeries(self, active=True): + ''' + Returns a list of all DistroSeries objects. + ''' + if not self._have_all_series: + for s in Launchpad.load(self.series_collection_link).entries: + series = DistroSeries(s['self_link']) + self._cache_series(series) + self._have_all_series = True + + allseries = filter(lambda s: s.active, self._series.values()) + allseries = sorted(allseries, + key=lambda s: float(s.version), + reverse=True) + Logger.debug("Found series: %s" % ", ".join(map(lambda s: "%s (%s)" % + (s.name, s.version), + allseries))) + return collections.OrderedDict((s.name, s) for s in allseries) class DistroArchSeries(BaseWrapper): @@ -329,7 +357,7 @@ class Archive(BaseWrapper): self._component_uploaders = {} def getSourcePackage(self, name, series=None, pocket=None, version=None, - status=None): + status=None, wrapper=None, search_all_series=False): ''' Returns a SourcePackagePublishingHistory object for the most recent source package in the distribution 'dist', series and @@ -338,11 +366,20 @@ class Archive(BaseWrapper): series defaults to the current development series if not specified. series must be either a series name string, or DistroSeries object. - pocket may be a string or a list. If a list, the highest version - will be returned. It defaults to all pockets except backports. - version may be specified to get only the exact version requested. + pocket may be a string or a list. If no version is provided, it + defaults to all pockets except 'Backports'; if searching for a + specific version, it defaults to all pockets. Pocket strings must + be capitalized. + + wrapper is the class to return an instance of; defaults to + SourcePackagePublishingHistory. + + search_all_series is used if series is None. If False, this will + search only the latest devel series, and if True all series + will be searched, in reverse order, starting with the latest + devel series. Defaults to False. status is optional, to restrict search to a given status only. If the requested source package doesn't exist a @@ -351,13 +388,15 @@ class Archive(BaseWrapper): return self._getPublishedItem(name, series, pocket, cache=self._srcpkgs, function='getPublishedSources', name_key='source_name', - wrapper=SourcePackagePublishingHistory, + wrapper=wrapper or SourcePackagePublishingHistory, version=version, status=status, + search_all_series=search_all_series, binary=False) def getBinaryPackage(self, name, archtag=None, series=None, pocket=None, - version=None, status=None): + version=None, status=None, wrapper=None, + search_all_series=False): ''' Returns a BinaryPackagePublishingHistory object for the most recent source package in the distribution 'dist', architecture @@ -367,11 +406,20 @@ class Archive(BaseWrapper): series must be either a series name string, or DistroArchSeries object. series may be omitted if version is specified. - pocket may be a string or a list. If a list, the highest version - will be returned. It defaults to all pockets except backports. - version may be specified to get only the exact version requested. + pocket may be a string or a list. If no version is provided, it + defaults to all pockets except 'Backports'; if searching for a + specific version, it defaults to all pockets. Pocket strings must + be capitalized. + + wrapper is the class to return an instance of; defaults to + BinaryPackagePublishingHistory. + + search_all_series is used if series is None. If False, this will + search only the latest devel series, and if True all series + will be searched, in reverse order, starting with the latest + devel series. Defaults to False. status is optional, to restrict search to a given status only. If the requested binary package doesn't exist a @@ -381,36 +429,40 @@ class Archive(BaseWrapper): cache=self._binpkgs, function='getPublishedBinaries', name_key='binary_name', - wrapper=BinaryPackagePublishingHistory, + wrapper=wrapper or BinaryPackagePublishingHistory, version=version, status=status, + search_all_series=search_all_series, binary=True) def _getPublishedItem(self, name, series, pocket, cache, function, name_key, wrapper, archtag=None, - version=None, status=None, + version=None, status=None, search_all_series=False, binary=False): ''' Common code between getSourcePackage and getBinaryPackage. Don't use this directly. ''' - if pocket is None: - pockets = frozenset(('Proposed', 'Updates', 'Security', 'Release')) + if not pocket: + if version and not series: + # check ALL pockets if specific version in any series + pockets = POCKETS + else: + # otherwise, check all pockets EXCEPT 'Backports' + pockets = DEFAULT_POCKETS elif isinstance(pocket, str): - pockets = frozenset((pocket,)) + pockets = (pocket,) else: - pockets = frozenset(pocket) + pockets = tuple(pocket) + + for p in pockets: + if p not in POCKETS: + raise PocketDoesNotExistError("Pocket '%s' does not exist." % p) - for pocket in pockets: - if pocket not in _POCKETS: - raise PocketDoesNotExistError("Pocket '%s' does not exist." % - pocket) dist = Distribution(self.distribution_link) - arch_series = None - # please don't pass DistroArchSeries as archtag! # but, the code was like that before so keep # backwards compatibility. @@ -418,83 +470,126 @@ class Archive(BaseWrapper): series = archtag archtag = None - if isinstance(series, DistroArchSeries): - arch_series = series - series = series.getSeries() - elif isinstance(series, DistroSeries): - pass - elif series: - series = dist.getSeries(series) - elif not version: - series = dist.getDevelopmentSeries() + series_to_check = [series] + if not version and not series: + # if neither version or series are specified, use either the + # devel series or search all series + if search_all_series: + series_to_check = dist.getAllSeries().values() + else: + series_to_check = [dist.getDevelopmentSeries()] - if binary: - if arch_series is None and series: - arch_series = series.getArchSeries(archtag=archtag) - if archtag is None and arch_series: - archtag = arch_series.architecture_tag - if archtag is None: - archtag = host_architecture() + # check each series - if only version was provided, series will be None + for series in series_to_check: + arch_series = None - index = (name, getattr(series, 'name', None), archtag, pockets, status, version) + if isinstance(series, DistroArchSeries): + arch_series = series + series = series.getSeries() + elif isinstance(series, DistroSeries): + pass + elif series: + series = dist.getSeries(series) + + if binary: + if arch_series is None and series: + arch_series = series.getArchSeries(archtag=archtag) + if archtag is None and arch_series: + archtag = arch_series.architecture_tag + if archtag is None: + archtag = host_architecture() + + index = (name, getattr(series, 'name', None), archtag, pockets, status, version) + + if index in cache: + return cache[index] - if index not in cache: params = { name_key: name, 'exact_match': True, } - if status: - params['status'] = status - if arch_series: params['distro_arch_series'] = arch_series() elif series: params['distro_series'] = series() if len(pockets) == 1: - params['pocket'] = list(pockets)[0] + params['pocket'] = pockets[0] + if version: params['version'] = version + Logger.debug('Calling %s(%s)' % (function, + ', '.join(['%s=%s' % (k, v) + for (k, v) in params.items()]))) records = getattr(self, function)(**params) - latest = None + err_msg = ("does not exist in the %s %s archive" % + (dist.display_name, self.name)) + for record in records: + if binary: + rversion = getattr(record, 'binary_package_version', None) + else: + rversion = getattr(record, 'source_package_version', None) + skipmsg = ('Skipping version %s: ' % rversion) + if record.pocket not in pockets: + err_msg = 'pocket %s not in (%s)' % (record.pocket, ','.join(pockets)) + Logger.debug(skipmsg + err_msg) + continue continue r = wrapper(record) if binary and archtag and archtag != r.arch: + err_msg = 'arch %s does not match requested arch %s' % (r.arch, archtag) + Logger.debug(skipmsg + err_msg) continue - if latest is None or latest.getVersion() < r.getVersion(): - latest = r + # results are ordered so first is latest + cache[index] = r + return r - if latest is None: - if name_key == 'binary_name': - package_type = "binary package" - elif name_key == 'source_name': - package_type = "source package" - else: - package_type = "package" - msg = "The %s '%s' " % (package_type, name) - if version: - msg += "version %s " % version - msg += ("does not exist in the %s %s archive" % - (dist.display_name, self.name)) - if binary: - msg += " for architecture %s" % archtag - if series: - pockets = [series.name if pocket == 'Release' - else '%s-%s' % (series.name, pocket.lower()) - for pocket in pockets] - if len(pockets) > 1: - pockets[-2:] = [' or '.join(pockets[-2:])] - msg += " in " + ', '.join(pockets) - raise PackageNotFoundException(msg) + version_with_epoch = None + if version and version == Version(version).strip_epoch() and len(records) == 0: + # a specific version was asked for, but we found none; + # check if one exists with an epoch to give a hint in error msg + for epoch in range(1, 9): + v = Version(version) + v.epoch = epoch + params['version'] = v.full_version + if len(getattr(self, function)(**params)) > 0: + version_with_epoch = v.full_version + Logger.debug('Found version with epoch %s' % version_with_epoch) + break - cache[index] = latest - return cache[index] + if name_key == 'binary_name': + package_type = "binary package" + elif name_key == 'source_name': + package_type = "source package" + else: + package_type = "package" + msg = "The %s '%s' " % (package_type, name) + if version: + msg += "version %s " % version + msg += err_msg + if binary and archtag: + msg += " for architecture %s" % archtag + if len(series_to_check) > 1: + msg += " in any release" + if len(pockets) == 1: + msg += " for pocket %s" % pockets[0] + elif len(pockets) != len(POCKETS): + msg += " for pockets " + ', '.join(pockets) + elif series: + msg += " in %s" % series.name + if len(pockets) == 1: + msg += "-%s" % pockets[0] + elif len(pockets) != len(POCKETS): + msg += " for pockets " + ', '.join(pockets) + if version_with_epoch: + msg += " (did you forget the epoch? try %s)" % version_with_epoch + raise PackageNotFoundException(msg) def copyPackage(self, source_name, version, from_archive, to_pocket, to_series=None, sponsored=None, include_binaries=False): @@ -695,7 +790,7 @@ class SourcePackagePublishingHistory(BaseWrapper): self._binaries[a][b.binary_package_name] = b self._have_all_binaries = True else: - # Older version, so we have to go the long way :( + # we have to go the long way :( print("Please wait, this may take some time...") archive = self.getArchive() urls = self.binaryFileUrls() @@ -976,7 +1071,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam): if package is None and component is None: raise ValueError('Either a source package name or a component has ' 'to be specified.') - if pocket not in _POCKETS: + if pocket not in POCKETS: raise PocketDoesNotExistError("Pocket '%s' does not exist." % pocket) @@ -1012,7 +1107,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam): return self._ppas def getPPAByName(self, name): - return self._lpobject.getPPAByName(name=name) + return Archive(self._lpobject.getPPAByName(name=name)) class Build(BaseWrapper): From 7c097b19bad1efff794982ed20cf488e33dc46af Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 6 Nov 2019 17:07:42 -0500 Subject: [PATCH 24/30] ubuntutools: add --status param to pull-pkg Allow specifying what package statuses should be searched for. By default search only for Pending and Published, unless a specific version number is being searched for. --- ubuntutools/archive.py | 4 ++++ ubuntutools/lp/lpapicache.py | 41 ++++++++++++++++++++++++++++++++---- ubuntutools/misc.py | 3 +++ ubuntutools/pullpkg.py | 13 ++++++++++-- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 9e857e6..375f308 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -142,6 +142,7 @@ class SourcePackage(object): quiet = kwargs.get('quiet', False) series = kwargs.get('series') pocket = kwargs.get('pocket') + status = kwargs.get('status') verify_signature = kwargs.get('verify_signature', False) assert (package is not None or dscfile is not None) @@ -154,6 +155,7 @@ class SourcePackage(object): self.quiet = quiet self._series = series self._pocket = pocket + self._status = status self._dsc_source = dscfile self._verify_signature = verify_signature @@ -201,6 +203,8 @@ class SourcePackage(object): # We always want to search all series, if not specified params['search_all_series'] = True + params['status'] = self._status + try: self._spph = archive.getSourcePackage(self.source, wrapper=self.spph_class, diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 5063c7e..a821ebc 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -38,7 +38,8 @@ from lazr.restfulclient.resource import Entry from ubuntutools.version import Version from ubuntutools.lp import (service, api_version) from ubuntutools.misc import (host_architecture, - DEFAULT_POCKETS, POCKETS) + DEFAULT_POCKETS, POCKETS, + DEFAULT_STATUSES, STATUSES) from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError, ArchiveNotFoundException, ArchSeriesNotFoundException, @@ -373,6 +374,11 @@ class Archive(BaseWrapper): specific version, it defaults to all pockets. Pocket strings must be capitalized. + status may be a string or a list. If no version is provided, it + defaults to only 'Pending' and 'Published'; if searching for a + specific version, it defaults to all statuses. Status strings must + be capitalized. + wrapper is the class to return an instance of; defaults to SourcePackagePublishingHistory. @@ -380,7 +386,6 @@ class Archive(BaseWrapper): search only the latest devel series, and if True all series will be searched, in reverse order, starting with the latest devel series. Defaults to False. - status is optional, to restrict search to a given status only. If the requested source package doesn't exist a PackageNotFoundException is raised. @@ -413,6 +418,11 @@ class Archive(BaseWrapper): specific version, it defaults to all pockets. Pocket strings must be capitalized. + status may be a string or a list. If no version is provided, it + defaults to only 'Pending' and 'Published'; if searching for a + specific version, it defaults to all statuses. Status strings must + be capitalized. + wrapper is the class to return an instance of; defaults to BinaryPackagePublishingHistory. @@ -420,7 +430,6 @@ class Archive(BaseWrapper): search only the latest devel series, and if True all series will be searched, in reverse order, starting with the latest devel series. Defaults to False. - status is optional, to restrict search to a given status only. If the requested binary package doesn't exist a PackageNotFoundException is raised. @@ -460,6 +469,21 @@ class Archive(BaseWrapper): if p not in POCKETS: raise PocketDoesNotExistError("Pocket '%s' does not exist." % p) + if not status: + if version: + # check ALL statuses if specific version + statuses = STATUSES + else: + # otherwise, only check 'Pending' and 'Published' + statuses = DEFAULT_STATUSES + elif isinstance(status, str): + statuses = (status,) + else: + statuses = tuple(status) + + for s in statuses: + if s not in STATUSES: + raise ValueError("Status '%s' is not valid." % s) dist = Distribution(self.distribution_link) @@ -499,7 +523,7 @@ class Archive(BaseWrapper): if archtag is None: archtag = host_architecture() - index = (name, getattr(series, 'name', None), archtag, pockets, status, version) + index = (name, getattr(series, 'name', None), archtag, pockets, statuses, version) if index in cache: return cache[index] @@ -517,6 +541,8 @@ class Archive(BaseWrapper): if len(pockets) == 1: params['pocket'] = pockets[0] + if len(statuses) == 1: + params['status'] = statuses[0] if version: params['version'] = version @@ -540,6 +566,9 @@ class Archive(BaseWrapper): err_msg = 'pocket %s not in (%s)' % (record.pocket, ','.join(pockets)) Logger.debug(skipmsg + err_msg) continue + if record.status not in statuses: + err_msg = 'status %s not in (%s)' % (record.status, ','.join(statuses)) + Logger.debug(skipmsg + err_msg) continue r = wrapper(record) if binary and archtag and archtag != r.arch: @@ -587,6 +616,10 @@ class Archive(BaseWrapper): msg += "-%s" % pockets[0] elif len(pockets) != len(POCKETS): msg += " for pockets " + ', '.join(pockets) + if len(statuses) == 1: + msg += " with status %s" % statuses[0] + elif len(statuses) != len(STATUSES): + msg += " with status in " + ', '.join(statuses) if version_with_epoch: msg += " (did you forget the epoch? try %s)" % version_with_epoch raise PackageNotFoundException(msg) diff --git a/ubuntutools/misc.py b/ubuntutools/misc.py index e322422..1a285fd 100644 --- a/ubuntutools/misc.py +++ b/ubuntutools/misc.py @@ -35,6 +35,9 @@ from ubuntutools.lp.udtexceptions import PocketDoesNotExistError DEFAULT_POCKETS = ('Release', 'Security', 'Updates', 'Proposed') POCKETS = DEFAULT_POCKETS + ('Backports',) +DEFAULT_STATUSES = ('Pending', 'Published') +STATUSES = DEFAULT_STATUSES + ('Superseded', 'Deleted', 'Obsolete') + _system_distribution_chain = [] diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index e7064a5..11f1bd8 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -40,7 +40,7 @@ from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, PocketDoesNotExistError, InvalidDistroValueError) from ubuntutools.logger import Logger -from ubuntutools.misc import (split_release_pocket, host_architecture) +from ubuntutools.misc import (split_release_pocket, host_architecture, STATUSES) PULL_SOURCE = 'source' PULL_DEBS = 'debs' @@ -114,7 +114,11 @@ class PullPkg(object): help_default_arch = ("Get binary packages for arch") help_default_arch += ("(default: %s)" % self._default_arch) - parser = ArgumentParser() + epilog = ("Note on --status: if a version is provided, all status types " + "will be searched; if no version is provided, by default only " + "'Pending' and 'Published' status will be searched.") + + parser = ArgumentParser(epilog=epilog) parser.add_argument('-v', '--verbose', action='store_true', help="Print verbose/debug messages") parser.add_argument('-d', '--download-only', action='store_true', @@ -125,6 +129,8 @@ class PullPkg(object): help="Don't read config files or environment variables") parser.add_argument('--no-verify-signature', action='store_true', help="Don't fail if dsc signature can't be verified") + parser.add_argument('-s', '--status', action='append', default=[], + help="Search for packages with specific status(es)") parser.add_argument('-a', '--arch', default=self._default_arch, help=help_default_arch) parser.add_argument('-p', '--pull', default=self._default_pull, @@ -265,6 +271,7 @@ class PullPkg(object): assert 'download_only' in options assert 'no_conf' in options assert 'no_verify_signature' in options + assert 'status' in options # type string assert 'pull' in options assert 'distro' in options @@ -316,6 +323,8 @@ class PullPkg(object): params['verify_signature'] = not options['no_verify_signature'] + params['status'] = STATUSES if 'all' in options['status'] else options['status'] + return (pull, distro, params) def pull(self, args=None): From c9c7fed1f69182f511503bc2bf33acdcc44ef780 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 13 Sep 2018 17:46:19 -0400 Subject: [PATCH 25/30] pull-uca-*: search in reverse order of UCA releases, if none specified if only the version of a UCA package is specified, search each UCA archive from latest backwards. This avoids having to specify both the package version *and* UCA release. --- ubuntutools/archive.py | 64 +++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 375f308..96ed316 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -726,11 +726,8 @@ class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage): if len(ppa) != 2: raise ValueError('Invalid PPA value "%s",' 'must be "/"' % kwargs['ppa']) - self._ppateam = ppa[0] - self._ppaname = ppa[1] + self._set_ppa(ppa[0], ppa[1]) self.masters = [] - self._team = None - self._ppa = None def getArchive(self): if not self._ppa: @@ -742,6 +739,12 @@ class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage): Logger.debug('Using PPA %s' % self._ppa.web_link) return self._ppa + def _set_ppa(self, team, name): + self._ppateam = team + self._ppaname = name + self._team = None + self._ppa = None + def _lp_url(self, filename): "Build a source package URL on Launchpad" return os.path.join('https://launchpad.net', '~' + self._ppateam, @@ -752,33 +755,66 @@ class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage): class UbuntuCloudArchiveSourcePackage(PersonalPackageArchiveSourcePackage): "Download / unpack an Ubuntu Cloud Archive source package" _ppateam = 'ubuntu-cloud-archive' + _ppas = None def __init__(self, *args, **kwargs): - series = kwargs.get('series') or UbuntuCloudArchiveSourcePackage.getDevelSeries() - kwargs.pop('series', None) + series = kwargs.pop('series', None) + check_all_series = series is None + if not series: + series = UbuntuCloudArchiveSourcePackage.getDevelSeries() kwargs['ppa'] = ('%s/%s-staging' % (UbuntuCloudArchiveSourcePackage._ppateam, series)) super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs) - self.uca_release = series + self._uca_release = series + self._check_all_series = check_all_series self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"] @classmethod def getDevelSeries(cls): - ppas = PersonTeam.fetch(cls._ppateam).getPPAs() - for key in sorted(ppas.keys(), reverse=True): - if key.endswith('-staging'): - return key.rsplit('-', 1)[0] - raise SeriesNotFoundException('Internal Error: No UCA devel series found...?') + return cls.ppas()[0] + + @classmethod + def ppas(cls): + if not cls._ppas: + ppas = PersonTeam.fetch(cls._ppateam).getPPAs().keys() + ppas = filter(lambda p: p.endswith('-staging'), ppas) + ppas = map(lambda p: p.rsplit('-', 1)[0], ppas) + ppas = sorted(ppas, reverse=True) + if not ppas: + raise SeriesNotFoundException('Internal Error: No UCA series found...?') + cls._ppas = ppas + return list(cls._ppas) @classmethod def isValidRelease(cls, release): - return ('%s-staging' % release) in PersonTeam.fetch(cls._ppateam).getPPAs() + return release in cls.ppas() + + @property + def lp_spph(self): + "Return the LP Source Package Publishing History entry" + while True: + try: + return super(UbuntuCloudArchiveSourcePackage, self).lp_spph + except PackageNotFoundException as pnfe: + if self._check_all_series and self._set_next_release(): + continue + raise pnfe + + def _set_next_release(self): + ppas = UbuntuCloudArchiveSourcePackage.ppas() + try: + r = ppas[ppas.index(self._uca_release) + 1] + except IndexError: + return False + self._uca_release = r + self._set_ppa(UbuntuCloudArchiveSourcePackage._ppateam, '%s-staging' % r) + return True def getArchive(self): try: return super(UbuntuCloudArchiveSourcePackage, self).getArchive() except ValueError: - raise SeriesNotFoundException('UCA release {} not found.'.format(self.uca_release)) + raise SeriesNotFoundException('UCA release {} not found.'.format(self._uca_release)) class _WebJSON(object): From 90e8fe81e1b2610e352c82c0301076ffc7da5ac0 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 12 Oct 2018 18:54:07 -0400 Subject: [PATCH 26/30] replace ubuntutools.logger with standard python logging --- backportpackage | 8 +- bitesize | 4 +- grep-merges | 11 ++- hugdaylist | 25 +++--- import-bug-from-debian | 6 +- merge-changelog | 7 +- pbuilder-dist | 19 +++-- pull-debian-debdiff | 12 +-- requestbackport | 31 ++++---- requestsync | 76 +++++++++--------- reverse-depends | 50 ++++++------ seeded-in-ubuntu | 24 +++--- sponsor-patch | 10 ++- submittodebian | 17 ++-- syncpackage | 86 +++++++++++---------- ubuntu-build | 68 ++++++++-------- ubuntu-iso | 8 +- ubuntu-upload-permission | 56 +++++++------- ubuntutools/__init__.py | 17 ++++ ubuntutools/archive.py | 49 ++++++------ ubuntutools/builder.py | 21 ++--- ubuntutools/config.py | 13 ++-- ubuntutools/logger.py | 76 ------------------ ubuntutools/lp/lpapicache.py | 19 +++-- ubuntutools/pullpkg.py | 42 ++++++---- ubuntutools/requestsync/lp.py | 6 +- ubuntutools/requestsync/mail.py | 16 ++-- ubuntutools/sponsor_patch/bugtask.py | 6 +- ubuntutools/sponsor_patch/patch.py | 14 ++-- ubuntutools/sponsor_patch/source_package.py | 42 +++++----- ubuntutools/sponsor_patch/sponsor_patch.py | 50 ++++++------ ubuntutools/test/test_archive.py | 12 --- ubuntutools/test/test_config.py | 35 ++++----- ubuntutools/test/test_logger.py | 54 ------------- ubuntutools/test/test_update_maintainer.py | 25 +++--- ubuntutools/update_maintainer.py | 8 +- 36 files changed, 478 insertions(+), 545 deletions(-) delete mode 100644 ubuntutools/logger.py delete mode 100644 ubuntutools/test/test_logger.py diff --git a/backportpackage b/backportpackage index c8f1ef0..1df7129 100755 --- a/backportpackage +++ b/backportpackage @@ -36,11 +36,13 @@ from ubuntutools.builder import get_builder from ubuntutools.lp.lpapicache import (Launchpad, Distribution, SeriesNotFoundException, PackageNotFoundException) -from ubuntutools.logger import Logger from ubuntutools.misc import (system_distribution, vendor_to_distroinfo, codename_to_distribution) from ubuntutools.question import YesNoQuestion +from ubuntutools import getLogger +Logger = getLogger(__name__) + def error(msg): Logger.error(msg) @@ -48,7 +50,7 @@ def error(msg): def check_call(cmd, *args, **kwargs): - Logger.command(cmd) + Logger.debug(' '.join(cmd)) ret = subprocess.call(cmd, *args, **kwargs) if ret != 0: error('%s returned %d.' % (cmd[0], ret)) @@ -302,7 +304,7 @@ def orig_needed(upload, workdir, pkg): not headers['content-location'].startswith('https://launchpadlibrarian.net')): return True except HttpLib2Error as e: - Logger.info(e) + Logger.debug(e) return True return False diff --git a/bitesize b/bitesize index 6fd1b47..5a2c132 100755 --- a/bitesize +++ b/bitesize @@ -28,7 +28,9 @@ from launchpadlib.launchpad import Launchpad from launchpadlib.errors import HTTPError from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger + +from ubuntutools import getLogger +Logger = getLogger(__name__) def error_out(msg): diff --git a/grep-merges b/grep-merges index 3658a1b..7c75cd4 100755 --- a/grep-merges +++ b/grep-merges @@ -27,6 +27,9 @@ from httplib2 import Http, HttpLib2Error import ubuntutools.misc +from ubuntutools import getLogger +Logger = getLogger(__name__) + def main(): parser = optparse.OptionParser( @@ -52,11 +55,11 @@ def main(): try: headers, page = Http().request(url) except HttpLib2Error as e: - print(str(e), file=sys.stderr) + Logger.exception(e) sys.exit(1) if headers.status != 200: - print("%s: %s %s" % (url, headers.status, headers.reason), - file=sys.stderr) + Logger.error("%s: %s %s" % (url, headers.status, + headers.reason)) sys.exit(1) for merge in json.loads(page): @@ -71,7 +74,7 @@ def main(): pretty_uploader = '{} {}'.format(author, uploader) if (match is None or match in package or match in author or match in uploader or match in teams): - print('%s\t%s' % (package, pretty_uploader)) + Logger.info('%s\t%s' % (package, pretty_uploader)) if __name__ == '__main__': diff --git a/hugdaylist b/hugdaylist index 37f531f..5140ae1 100755 --- a/hugdaylist +++ b/hugdaylist @@ -36,6 +36,9 @@ from launchpadlib.launchpad import Launchpad from ubuntutools.lp.libsupport import translate_web_api +from ubuntutools import getLogger +Logger = getLogger(__name__) + def check_args(): howmany = -1 @@ -57,8 +60,7 @@ def check_args(): # Check that we have an URL. if not args: - print("An URL pointing to a Launchpad bug list is required.", - file=sys.stderr) + Logger.error("An URL pointing to a Launchpad bug list is required.") opt_parser.print_help() sys.exit(1) else: @@ -87,15 +89,14 @@ def main(): if len(url.split("?", 1)) == 2: # search options not supported, because there is no mapping web ui # options <-> API options - print("Options in url are not supported, url: %s" % url, - file=sys.stderr) + Logger.error("Options in url are not supported, url: %s" % url) sys.exit(1) launchpad = None try: launchpad = Launchpad.login_with("ubuntu-dev-tools", 'production') except IOError as error: - print(error) + Logger.exception(error) sys.exit(1) api_url = translate_web_api(url, launchpad) @@ -104,8 +105,8 @@ def main(): except Exception as error: response = getattr(error, "response", {}) if response.get("status", None) == "404": - print(("The URL at '%s' does not appear to be a valid url to a " - "product") % url, file=sys.stderr) + Logger.error("The URL at '%s' does not appear to be a " + "valid url to a product" % url) sys.exit(1) else: raise @@ -113,12 +114,12 @@ def main(): bug_list = [b for b in product.searchTasks() if filter_unsolved(b)] if not bug_list: - print("Bug list of %s is empty." % url) + Logger.info("Bug list of %s is empty." % url) sys.exit(0) if howmany == -1: howmany = len(bug_list) - print(""" + Logger.info(""" ## || This task is done || somebody || || ## || This task is assigned || somebody || || ## || This task isn't || ... || || @@ -128,13 +129,13 @@ def main(): for i in list(bug_list)[:howmany]: bug = i.bug - print('|| [%s %s] || %s || ||' - % (bug.web_link, bug.id, bug.title)) + Logger.info('|| [%s %s] || %s || ||' % + (bug.web_link, bug.id, bug.title)) if __name__ == '__main__': try: main() except KeyboardInterrupt: - print("Aborted.", file=sys.stderr) + Logger.error("Aborted.") sys.exit(1) diff --git a/import-bug-from-debian b/import-bug-from-debian index df3d21d..079ce3c 100755 --- a/import-bug-from-debian +++ b/import-bug-from-debian @@ -30,7 +30,9 @@ import webbrowser from launchpadlib.launchpad import Launchpad from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger + +from ubuntutools import getLogger +Logger = getLogger(__name__) try: import debianbts @@ -134,7 +136,7 @@ def main(): d_watch = u_bug.addWatch(remote_bug=bug_num, bug_tracker=lp_debbugs) d_task.bug_watch = d_watch d_task.lp_save() - Logger.normal("Opened %s", u_bug.web_link) + Logger.info("Opened %s", u_bug.web_link) if not options.browserless: webbrowser.open(u_bug.web_link) diff --git a/merge-changelog b/merge-changelog index 9d0b67e..f14d4db 100755 --- a/merge-changelog +++ b/merge-changelog @@ -22,9 +22,12 @@ import sys from debian.changelog import Changelog +from ubuntutools import getLogger +Logger = getLogger(__name__) + def usage(exit_code=1): - print('''Usage: merge-changelog + Logger.info('''Usage: merge-changelog merge-changelog takes two changelogs that once shared a common source, merges them back together, and prints the merged result to stdout. This @@ -61,7 +64,7 @@ def merge_changelog(left_changelog, right_changelog): assert block.version == version - print(str(block).strip(), end='\n\n') + Logger.info(str(block).strip() + '\n\n') def main(): diff --git a/pbuilder-dist b/pbuilder-dist index 4fff772..0971b0a 100755 --- a/pbuilder-dist +++ b/pbuilder-dist @@ -39,9 +39,11 @@ from distro_info import DebianDistroInfo, UbuntuDistroInfo, DistroDataOutdated import ubuntutools.misc import ubuntutools.version from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger from ubuntutools.question import YesNoQuestion +from ubuntutools import getLogger +Logger = getLogger(__name__) + class PbuilderDist(object): def __init__(self, builder): @@ -103,9 +105,10 @@ class PbuilderDist(object): '~/pbuilder/')) if 'SUDO_USER' in os.environ: - Logger.warn('Running under sudo. ' - 'This is probably not what you want. ' - 'pbuilder-dist will use sudo itself, when necessary.') + Logger.warning('Running under sudo. ' + 'This is probably not what you want. ' + 'pbuilder-dist will use sudo itself, ' + 'when necessary.') if os.stat(os.environ['HOME']).st_uid != os.getuid(): Logger.error("You don't own $HOME") sys.exit(1) @@ -280,7 +283,7 @@ class PbuilderDist(object): codename = debian_info.codename(self.target_distro, default=self.target_distro) except DistroDataOutdated as error: - Logger.warn(error) + Logger.warning(error) if codename in (debian_info.devel(), 'experimental'): self.enable_security = False self.enable_updates = False @@ -307,7 +310,7 @@ class PbuilderDist(object): try: dev_release = self.target_distro == UbuntuDistroInfo().devel() except DistroDataOutdated as error: - Logger.warn(error) + Logger.warning(error) dev_release = True if dev_release: @@ -396,7 +399,7 @@ def show_help(exit_code=0): Print a help message for pbuilder-dist, and exit with the given code. """ - print('See man pbuilder-dist for more information.') + Logger.info('See man pbuilder-dist for more information.') sys.exit(exit_code) @@ -498,7 +501,7 @@ def main(): if '--debug-echo' not in args: sys.exit(subprocess.call(app.get_command(args))) else: - print(app.get_command([arg for arg in args if arg != '--debug-echo'])) + Logger.info(app.get_command([arg for arg in args if arg != '--debug-echo'])) if __name__ == '__main__': diff --git a/pull-debian-debdiff b/pull-debian-debdiff index 9685c80..50f15ce 100755 --- a/pull-debian-debdiff +++ b/pull-debian-debdiff @@ -24,9 +24,11 @@ import debian.changelog from ubuntutools.archive import DebianSourcePackage, DownloadError from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger from ubuntutools.version import Version +from ubuntutools import getLogger +Logger = getLogger(__name__) + def previous_version(package, version, distance): "Given an (extracted) package, determine the version distance versions ago" @@ -79,7 +81,7 @@ def main(): opts.debsec_mirror = config.get_value('DEBSEC_MIRROR') mirrors = [opts.debsec_mirror, opts.debian_mirror] - Logger.normal('Downloading %s %s', package, version) + Logger.info('Downloading %s %s', package, version) newpkg = DebianSourcePackage(package, version, mirrors=mirrors) try: @@ -96,7 +98,7 @@ def main(): if not oldversion: Logger.error('No previous version could be found') sys.exit(1) - Logger.normal('Downloading %s %s', package, oldversion) + Logger.info('Downloading %s %s', package, oldversion) oldpkg = DebianSourcePackage(package, oldversion, mirrors=mirrors) try: @@ -104,11 +106,11 @@ def main(): except DownloadError as e: Logger.error('Failed to download: %s', str(e)) sys.exit(1) - print('file://' + oldpkg.debdiff(newpkg, diffstat=True)) + Logger.info('file://' + oldpkg.debdiff(newpkg, diffstat=True)) if __name__ == '__main__': try: main() except KeyboardInterrupt: - Logger.normal('User abort.') + Logger.info('User abort.') diff --git a/requestbackport b/requestbackport index e660f51..a6e30f3 100755 --- a/requestbackport +++ b/requestbackport @@ -25,11 +25,13 @@ from distro_info import UbuntuDistroInfo from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import Launchpad, Distribution from ubuntutools.lp.udtexceptions import PackageNotFoundException -from ubuntutools.logger import Logger from ubuntutools.question import (YesNoQuestion, EditBugReport, confirmation_prompt) from ubuntutools.rdepends import query_rdepends, RDependsException +from ubuntutools import getLogger +Logger = getLogger(__name__) + class DestinationException(Exception): pass @@ -106,16 +108,11 @@ def check_existing(package, destinations): if not bugs: return - Logger.normal("There are existing bug reports that look similar to your " - "request. Please check before continuing:") + Logger.info("There are existing bug reports that look similar to your " + "request. Please check before continuing:") - by_id = {} - for bug_task in bugs: - bug = bug_task.bug - by_id[bug.id] = bug - - for id_, bug in sorted(by_id.items()): - Logger.normal(" * LP: #%-7i: %s %s", bug.id, bug.title, bug.web_link) + for bug in sorted(set(bug_task.bug for bug_task in bugs)): + Logger.info(" * LP: #%-7i: %s %s", bug.id, bug.title, bug.web_link) confirmation_prompt() @@ -190,8 +187,8 @@ def locate_package(package, distribution): except KeyError: continue package = apt_pkg.candidate.source_name - Logger.normal("Binary package specified, considering its source " - "package instead: %s", package) + Logger.info("Binary package specified, considering its source " + "package instead: %s", package) def request_backport(package_spph, source, destinations): @@ -204,8 +201,8 @@ def request_backport(package_spph, source, destinations): Logger.error("%s (%s) has no published binaries in %s. ", package_spph.getPackageName(), package_spph.getVersion(), source) - Logger.normal("Is it stuck in bin-NEW? It can't be backported until " - "the binaries have been accepted.") + Logger.info("Is it stuck in bin-NEW? It can't be backported until " + "the binaries have been accepted.") sys.exit(1) testing = [] @@ -256,8 +253,8 @@ def request_backport(package_spph, source, destinations): editor.edit() subject, body = editor.get_report() - Logger.normal('The final report is:\nSummary: %s\nDescription:\n%s\n', - subject, body) + Logger.info('The final report is:\nSummary: %s\nDescription:\n%s\n', + subject, body) if YesNoQuestion().ask("Request this backport", "yes") == "no": sys.exit(1) @@ -268,7 +265,7 @@ def request_backport(package_spph, source, destinations): for target in targets[1:]: bug.addTask(target=target) - Logger.normal("Backport request filed as %s", bug.web_link) + Logger.info("Backport request filed as %s", bug.web_link) def main(): diff --git a/requestsync b/requestsync index 6d77bb8..a12a669 100755 --- a/requestsync +++ b/requestsync @@ -38,6 +38,9 @@ from ubuntutools.misc import require_utf8 from ubuntutools.question import confirmation_prompt, EditBugReport from ubuntutools.version import Version +from ubuntutools import getLogger +Logger = getLogger(__name__) + # # entry point # @@ -97,7 +100,7 @@ def main(): config = UDTConfig(options.no_conf) if options.deprecated_lp_flag: - print("The --lp flag is now default, ignored.") + Logger.info("The --lp flag is now default, ignored.") if options.email: options.lpapi = False else: @@ -115,8 +118,8 @@ def main(): elif options.lpinstance == 'staging': bug_mail_domain = 'bugs.staging.launchpad.net' else: - print('Error: Unknown launchpad instance: %s' % options.lpinstance, - file=sys.stderr) + Logger.error('Error: Unknown launchpad instance: %s' + % options.lpinstance) sys.exit(1) mailserver_host = config.get_value('SMTP_SERVER', @@ -130,8 +133,8 @@ def main(): firstmx = mxlist[0] mailserver_host = firstmx[1] except ImportError: - print('Please install python3-dns to support Launchpad mail ' - 'server lookup.', file=sys.stderr) + Logger.error('Please install python-dns to support ' + 'Launchpad mail server lookup.') sys.exit(1) mailserver_port = config.get_value('SMTP_PORT', default=25, @@ -167,9 +170,8 @@ def main(): get_ubuntu_delta_changelog, mail_bug, need_sponsorship) if not any(x in os.environ for x in ('UBUMAIL', 'DEBEMAIL', 'EMAIL')): - print('E: The environment variable UBUMAIL, DEBEMAIL or EMAIL ' - 'needs to be set to let this script mail the sync request.', - file=sys.stderr) + Logger.error('The environment variable UBUMAIL, DEBEMAIL or EMAIL needs ' + 'to be set to let this script mail the sync request.') sys.exit(1) newsource = options.newpkg @@ -187,15 +189,14 @@ def main(): else: ubu_info = UbuntuDistroInfo() release = ubu_info.devel() - print('W: Target release missing - assuming %s' % release, - file=sys.stderr) + Logger.warning('Target release missing - assuming %s' % release) elif len(args) == 2: release = args[1] elif len(args) == 3: release = args[1] force_base_version = Version(args[2]) else: - print('E: Too many arguments.', file=sys.stderr) + Logger.error('Too many arguments.') parser.print_help() sys.exit(1) @@ -210,13 +211,12 @@ def main(): ubuntu_version = Version('~') ubuntu_component = None # Set after getting the Debian info if not newsource: - print(("'%s' doesn't exist in 'Ubuntu %s'.\n" - "Do you want to sync a new package?") - % (srcpkg, release)) + Logger.info("'%s' doesn't exist in 'Ubuntu %s'." % (srcpkg, release)) + Logger.info("Do you want to sync a new package?") confirmation_prompt() newsource = True except udtexceptions.SeriesNotFoundException as error: - print("E: %s" % error, file=sys.stderr) + Logger.error(error) sys.exit(1) # Get the requested Debian source package @@ -225,10 +225,10 @@ def main(): debian_version = Version(debian_srcpkg.getVersion()) debian_component = debian_srcpkg.getComponent() except udtexceptions.PackageNotFoundException as error: - print("E: %s" % error, file=sys.stderr) + Logger.error(error) sys.exit(1) except udtexceptions.SeriesNotFoundException as error: - print("E: %s" % error, file=sys.stderr) + Logger.error(error) sys.exit(1) if ubuntu_component is None: @@ -246,17 +246,17 @@ def main(): debian_version = Version(debian_srcpkg.getVersion()) debian_component = debian_srcpkg.getComponent() except udtexceptions.PackageNotFoundException as error: - print("E: %s" % error, file=sys.stderr) + Logger.error(error) sys.exit(1) if ubuntu_version == debian_version: - print('E: The versions in Debian and Ubuntu are the same already ' - '(%s). Aborting.' % ubuntu_version, file=sys.stderr) + Logger.error('The versions in Debian and Ubuntu are the ' + 'same already (%s). Aborting.' % ubuntu_version) sys.exit(1) if ubuntu_version > debian_version: - print(('E: The version in Ubuntu (%s) is newer than the version in ' - 'Debian (%s). Aborting.') - % (ubuntu_version, debian_version), file=sys.stderr) + Logger.error('The version in Ubuntu (%s) is newer than ' + 'the version in Debian (%s). Aborting.' + % (ubuntu_version, debian_version)) sys.exit(1) # -s flag not specified - check if we do need sponsorship @@ -264,8 +264,8 @@ def main(): sponsorship = need_sponsorship(srcpkg, ubuntu_component, release) if not sponsorship and not ffe: - print('Consider using syncpackage(1) for syncs that do not require ' - 'feature freeze exceptions.', file=sys.stderr) + Logger.error('Consider using syncpackage(1) for syncs that ' + 'do not require feature freeze exceptions.') # Check for existing package reports if not newsource: @@ -283,9 +283,9 @@ def main(): if 'ubuntu' in str(ubuntu_version): need_interaction = True - print('Changes have been made to the package in Ubuntu.\n' - 'Please edit the report and give an explanation.\n' - 'Not saving the report file will abort the request.') + Logger.info('Changes have been made to the package in Ubuntu.') + Logger.info('Please edit the report and give an explanation.') + Logger.info('Not saving the report file will abort the request.') report += ('Explanation of the Ubuntu delta and why it can be ' 'dropped:\n%s\n>>> ENTER_EXPLANATION_HERE <<<\n\n' % get_ubuntu_delta_changelog(ubuntu_srcpkg)) @@ -293,9 +293,9 @@ def main(): if ffe: need_interaction = True - print('To approve FeatureFreeze exception, you need to state\n' - 'the reason why you feel it is necessary.\n' - 'Not saving the report file will abort the request.') + Logger.info('To approve FeatureFreeze exception, you need to state') + Logger.info('the reason why you feel it is necessary.') + Logger.info('Not saving the report file will abort the request.') report += ('Explanation of FeatureFreeze exception:\n' '>>> ENTER_EXPLANATION_HERE <<<\n\n') @@ -312,10 +312,10 @@ def main(): changelog = debian_srcpkg.getChangelog(since_version=base_version) if not changelog: if not options.missing_changelog_ok: - print("E: Did not retrieve any changelog entries. " - "Do you need to specify '-C'? " - "Was the package recently uploaded? (check " - "http://packages.debian.org/changelogs/)", file=sys.stderr) + Logger.error("Did not retrieve any changelog entries. " + "Do you need to specify '-C'? " + "Was the package recently uploaded? (check " + "http://packages.debian.org/changelogs/)") sys.exit(1) else: need_interaction = True @@ -327,8 +327,8 @@ def main(): title, report = editor.get_report() if 'XXX FIXME' in report: - print("E: changelog boilerplate found in report, please manually add " - "changelog when using '-C'", file=sys.stderr) + Logger.error("changelog boilerplate found in report, " + "please manually add changelog when using '-C'") sys.exit(1) # bug status and bug subscriber @@ -359,5 +359,5 @@ if __name__ == '__main__': try: main() except KeyboardInterrupt: - print("\nUser abort.") + Logger.error("User abort.") sys.exit(2) diff --git a/reverse-depends b/reverse-depends index e8d06f9..d5ca6b8 100755 --- a/reverse-depends +++ b/reverse-depends @@ -19,11 +19,13 @@ import sys from distro_info import DistroDataOutdated -from ubuntutools.logger import Logger from ubuntutools.misc import (system_distribution, vendor_to_distroinfo, codename_to_distribution) from ubuntutools.rdepends import query_rdepends, RDependsException +from ubuntutools import getLogger +Logger = getLogger(__name__) + DEFAULT_MAX_DEPTH = 10 # We want avoid any infinite loop... @@ -32,7 +34,7 @@ def main(): try: default_release = system_distro_info.devel() except DistroDataOutdated as e: - Logger.warn(e) + Logger.warning(e) default_release = 'unstable' description = ("List reverse-dependencies of package. " @@ -85,7 +87,7 @@ def main(): options.release = distro_info.codename(options.release, default=options.release) except DistroDataOutdated: - # We already printed a warning + # We already logged a warning pass if options.build_depends: @@ -143,14 +145,14 @@ def main(): def display_verbose(package, values): if not values: - print("No reverse dependencies found") + Logger.info("No reverse dependencies found") return - def print_field(field): - print(field) - print('=' * len(field)) + def log_field(field): + Logger.info(field) + Logger.info('=' * len(field)) - def print_package(values, package, arch, dependency, offset=0): + def log_package(values, package, arch, dependency, offset=0): line = ' ' * offset + '* %s' % package if all_archs and set(arch) != all_archs: line += ' [%s]' % ' '.join(sorted(arch)) @@ -158,17 +160,17 @@ def display_verbose(package, values): if len(line) < 30: line += ' ' * (30 - len(line)) line += ' (for %s)' % dependency - print(line) + Logger.info(line) data = values.get(package) if data: offset = offset + 1 for rdeps in data.values(): for rdep in rdeps: - print_package(values, - rdep['Package'], - rdep.get('Architectures', all_archs), - rdep.get('Dependency'), - offset) + log_package(values, + rdep['Package'], + rdep.get('Architectures', all_archs), + rdep.get('Dependency'), + offset) all_archs = set() # This isn't accurate, but we make up for it by displaying what we found @@ -179,19 +181,19 @@ def display_verbose(package, values): all_archs.update(rdep['Architectures']) for field, rdeps in values[package].items(): - print_field(field) + Logger.info(field) rdeps.sort(key=lambda x: x['Package']) for rdep in rdeps: - print_package(values, - rdep['Package'], - rdep.get('Architectures', all_archs), - rdep.get('Dependency')) - print() + log_package(values, + rdep['Package'], + rdep.get('Architectures', all_archs), + rdep.get('Dependency')) + Logger.info("") if all_archs: - print("Packages without architectures listed are " - "reverse-dependencies in: %s" - % ', '.join(sorted(list(all_archs)))) + Logger.info("Packages without architectures listed are " + "reverse-dependencies in: %s" + % ', '.join(sorted(list(all_archs)))) def display_consise(values): @@ -201,7 +203,7 @@ def display_consise(values): for rdep in rdeps: result.add(rdep['Package']) - print('\n'.join(sorted(list(result)))) + Logger.info('\n'.join(sorted(list(result)))) if __name__ == '__main__': diff --git a/seeded-in-ubuntu b/seeded-in-ubuntu index 7c3b849..846f92e 100755 --- a/seeded-in-ubuntu +++ b/seeded-in-ubuntu @@ -24,7 +24,9 @@ import urllib.request from ubuntutools.lp.lpapicache import (Distribution, Launchpad, PackageNotFoundException) -from ubuntutools.logger import Logger + +from ubuntutools import getLogger +Logger = getLogger(__name__) DATA_URL = 'http://qa.ubuntuwire.org/ubuntu-seeded-packages/seeded.json.gz' @@ -88,28 +90,28 @@ def output_binaries(index, binaries): '''Print binaries found in index''' for binary in binaries: if binary in index: - print("%s is seeded in:" % binary) - print(present_on(index[binary])) + Logger.info("%s is seeded in:" % binary) + Logger.info(present_on(index[binary])) else: - print("%s is not seeded (and may not exist)." % binary) + Logger.info("%s is not seeded (and may not exist)." % binary) def output_by_source(index, by_source): - '''Print binaries found in index. Grouped by source''' + '''Logger.Info(binaries found in index. Grouped by source''' for source, binaries in by_source.items(): seen = False if not binaries: - print("Status unknown: No binary packages built by the latest " - "%s.\nTry again using -b and the expected binary packages." - % source) + Logger.info("Status unknown: No binary packages built by the latest " + "%s.\nTry again using -b and the expected binary packages." + % source) continue for binary in binaries: if binary in index: seen = True - print("%s (from %s) is seeded in:" % (binary, source)) - print(present_on(index[binary])) + Logger.info("%s (from %s) is seeded in:" % (binary, source)) + Logger.info(present_on(index[binary])) if not seen: - print("%s's binaries are not seeded." % source) + Logger.info("%s's binaries are not seeded." % source) def main(): diff --git a/sponsor-patch b/sponsor-patch index fe50146..1e77852 100755 --- a/sponsor-patch +++ b/sponsor-patch @@ -19,12 +19,15 @@ import os import shutil import sys import tempfile +import logging from ubuntutools.builder import get_builder from ubuntutools.config import UDTConfig -from ubuntutools.logger import Logger from ubuntutools.sponsor_patch.sponsor_patch import sponsor_patch, check_dependencies +from ubuntutools import getLogger +Logger = getLogger(__name__) + def parse(script_name): """Parse the command line parameters.""" @@ -64,7 +67,8 @@ def parse(script_name): "temporary directory, deleted afterwards).") (options, args) = parser.parse_args() - Logger.set_verbosity(options.verbose) + if options.verbose: + Logger.setLevel(logging.DEBUG) check_dependencies() if len(args) == 0: @@ -123,7 +127,7 @@ def main(): options.keyid, options.lpinstance, options.update, options.upload, workdir) except KeyboardInterrupt: - print("\nUser abort.") + Logger.error("User abort.") sys.exit(2) finally: if options.workdir is None: diff --git a/submittodebian b/submittodebian index 2599ff4..2238521 100755 --- a/submittodebian +++ b/submittodebian @@ -39,6 +39,9 @@ from ubuntutools.config import ubu_email from ubuntutools.question import YesNoQuestion, EditFile from ubuntutools.update_maintainer import update_maintainer, restore_maintainer +from ubuntutools import getLogger +Logger = getLogger(__name__) + def get_most_recent_debian_version(changelog): for block in changelog: @@ -89,7 +92,7 @@ def gen_debdiff(tmpdir, changelog): diff_cmd = ['bzr', 'diff', '-r', 'tag:' + str(oldver)] if call(diff_cmd, stdout=DEVNULL, stderr=DEVNULL) == 1: - print("Extracting bzr diff between %s and %s" % (oldver, newver)) + Logger.info("Extracting bzr diff between %s and %s" % (oldver, newver)) else: if oldver.epoch is not None: oldver = str(oldver)[str(oldver).index(":") + 1:] @@ -102,7 +105,7 @@ def gen_debdiff(tmpdir, changelog): check_file(olddsc) check_file(newdsc) - print("Generating debdiff between %s and %s" % (oldver, newver)) + Logger.info("Generating debdiff between %s and %s" % (oldver, newver)) diff_cmd = ['debdiff', olddsc, newdsc] with Popen(diff_cmd, stdout=PIPE, encoding='utf-8') as diff: @@ -119,7 +122,7 @@ def check_file(fname, critical=True): else: if not critical: return False - print("Couldn't find «%s».\n" % fname) + Logger.info("Couldn't find «%s».\n" % fname) sys.exit(1) @@ -127,7 +130,7 @@ def submit_bugreport(body, debdiff, deb_version, changelog): try: devel = UbuntuDistroInfo().devel() except DistroDataOutdated as e: - print(str(e)) + Logger.info(str(e)) devel = '' if os.path.dirname(sys.argv[0]).startswith('/usr/bin'): @@ -197,7 +200,7 @@ no-cc with open(fn, 'w') as f: f.write(reportbugrc) - print("""\ + Logger.info("""\ You have not configured reportbug. Assuming this is the first time you have used it. Writing a ~/.reportbugrc that will use Debian's mail server, and CC the bug to you at <%s> @@ -221,8 +224,8 @@ def main(): parser.parse_args() if not os.path.exists('/usr/bin/reportbug'): - print("This utility requires the «reportbug» package, which isn't " - "currently installed.") + Logger.error("This utility requires the «reportbug» package, which isn't " + "currently installed.") sys.exit(1) check_reportbug_config() diff --git a/syncpackage b/syncpackage index 8dc296d..1268e1a 100755 --- a/syncpackage +++ b/syncpackage @@ -21,6 +21,7 @@ # ################################################################## import fnmatch +import logging import optparse import os import shutil @@ -37,7 +38,6 @@ from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.lp import udtexceptions from ubuntutools.lp.lpapicache import (Distribution, Launchpad, PersonTeam, SourcePackagePublishingHistory) -from ubuntutools.logger import Logger from ubuntutools.misc import split_release_pocket from ubuntutools.question import YesNoQuestion from ubuntutools.requestsync.mail import ( @@ -45,6 +45,9 @@ from ubuntutools.requestsync.mail import ( from ubuntutools.requestsync.lp import get_debian_srcpkg, get_ubuntu_srcpkg from ubuntutools.version import Version +from ubuntutools import getLogger +Logger = getLogger(__name__) + def remove_signature(dscname): '''Removes the signature from a .dsc file if the .dsc file is signed.''' @@ -116,7 +119,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, ubuntu_ver = Version('~') ubu_pkg = None need_orig = True - Logger.normal('%s does not exist in Ubuntu.', name) + Logger.info('%s does not exist in Ubuntu.', name) Logger.debug('Source %s: current version %s, new version %s', src_pkg.source, ubuntu_ver, new_ver) @@ -128,9 +131,9 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, Logger.error('--force is required to discard Ubuntu changes.') sys.exit(1) - Logger.warn('Overwriting modified Ubuntu version %s, ' - 'setting current version to %s', - ubuntu_ver.full_version, cur_ver.full_version) + Logger.warning('Overwriting modified Ubuntu version %s, ' + 'setting current version to %s', + ubuntu_ver.full_version, cur_ver.full_version) if simulate: return @@ -144,7 +147,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, needs_fakesync = not (need_orig or ubu_pkg.verify_orig()) if needs_fakesync and fakesync: - Logger.warn('Performing a fakesync') + Logger.warning('Performing a fakesync') elif not needs_fakesync and fakesync: Logger.error('Fakesync not required, aborting.') sys.exit(1) @@ -163,7 +166,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, # change into package directory directory = src_pkg.source + '-' + new_ver.upstream_version - Logger.command(('cd', directory)) + Logger.debug('cd' + directory) os.chdir(directory) # read Debian distribution from debian/changelog if not specified @@ -183,9 +186,9 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, cmd.append("-sa") else: cmd.append("-sd") - if not Logger.verbose: + if not Logger.isEnabledFor(logging.DEBUG): cmd += ["-q"] - Logger.command(cmd + ['>', '../' + changes_filename]) + Logger.debug(' '.join(cmd) + '> ../' + changes_filename) changes = subprocess.check_output(cmd, encoding='utf-8') # Add additional bug numbers @@ -193,7 +196,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, changes = add_fixed_bugs(changes, bugs) # remove extracted (temporary) files - Logger.command(('cd', '..')) + Logger.debug('cd ..') os.chdir('..') shutil.rmtree(directory, True) @@ -208,7 +211,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, cmd = ["debsign", changes_filename] if keyid is not None: cmd.insert(1, "-k" + keyid) - Logger.command(cmd) + Logger.debug(' '.join(cmd)) subprocess.check_call(cmd) else: # Create fakesync changelog entry @@ -223,14 +226,14 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, cmd = ['dch', '-v', new_ver.full_version, '--force-distribution', '-D', release, message] env = {'DEBFULLNAME': name, 'DEBEMAIL': email} - Logger.command(cmd) + Logger.debug(' '.join(cmd)) subprocess.check_call(cmd, env=env) # update the Maintainer field cmd = ["update-maintainer"] - if not Logger.verbose: + if not Logger.isEnabledFor(logging.DEBUG): cmd.append("-q") - Logger.command(cmd) + Logger.debug(' '.join(cmd)) subprocess.check_call(cmd) # Build source package @@ -240,7 +243,7 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, cmd += ['-sa'] if keyid: cmd += ["-k" + keyid] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) returncode = subprocess.call(cmd) if returncode != 0: Logger.error('Source-only build with debuild failed. ' @@ -339,9 +342,9 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False): ubuntu_spph.getComponent(), mirrors=[]) - Logger.normal('Source %s -> %s/%s: current version %s, new version %s', - src_pkg.source, ubuntu_series, ubuntu_pocket, - ubuntu_pkg.version, src_pkg.version) + Logger.info('Source %s -> %s/%s: current version %s, new version %s', + src_pkg.source, ubuntu_series, ubuntu_pocket, + ubuntu_pkg.version, src_pkg.version) ubuntu_version = Version(ubuntu_pkg.version.full_version) base_version = ubuntu_version.get_related_debian_version() @@ -358,21 +361,21 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False): sys.exit(1) except udtexceptions.PackageNotFoundException: base_version = Version('~') - Logger.normal('Source %s -> %s/%s: not in Ubuntu, new version %s', - src_pkg.source, ubuntu_series, ubuntu_pocket, - src_pkg.version) + Logger.info('Source %s -> %s/%s: not in Ubuntu, new version %s', + src_pkg.source, ubuntu_series, ubuntu_pocket, + src_pkg.version) changes = debian_spph.getChangelog(since_version=base_version) if changes: changes = changes.strip() - Logger.normal("New changes:\n%s", changes) + Logger.info("New changes:\n%s", changes) if simulate: return if sponsoree: - Logger.normal("Sponsoring this sync for %s (%s)", - sponsoree.display_name, sponsoree.name) + Logger.info("Sponsoring this sync for %s (%s)", + sponsoree.display_name, sponsoree.name) answer = YesNoQuestion().ask("Sync this package", "no") if answer != "yes": return @@ -392,14 +395,14 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False): Logger.error(error.content) sys.exit(1) - Logger.normal('Request succeeded; you should get an e-mail once it is ' - 'processed.') + Logger.info('Request succeeded; you should get an e-mail once it is ' + 'processed.') bugs = sorted(set(bugs)) if bugs: - Logger.normal("Launchpad bugs to be closed: %s", - ', '.join(str(bug) for bug in bugs)) - Logger.normal('Please wait for the sync to be successful before ' - 'closing bugs.') + Logger.info("Launchpad bugs to be closed: %s", + ', '.join(str(bug) for bug in bugs)) + Logger.info('Please wait for the sync to be successful before ' + 'closing bugs.') answer = YesNoQuestion().ask("Close bugs", "yes") if answer == "yes": close_bugs(bugs, src_pkg.source, src_pkg.version.full_version, @@ -468,7 +471,7 @@ def close_bugs(bugs, package, version, changes, sponsoree): if target == ubuntu or (target.name == package and getattr(target, 'distribution', None) == ubuntu): if task.status != 'Fix Released': - Logger.normal("Closed bug %s", task.web_link) + Logger.info("Closed bug %s", task.web_link) task.status = 'Fix Released' task.lp_save() bug.newMessage(content=message) @@ -597,7 +600,8 @@ def main(): '''Handle parameters and get the ball rolling''' (options, package) = parse() - Logger.verbose = options.verbose + if options.verbose: + Logger.setLevel('DEBUG') config = UDTConfig(options.no_conf) if options.debian_mirror is None: options.debian_mirror = config.get_value('DEBIAN_MIRROR') @@ -626,10 +630,10 @@ def main(): options.release = "%s-proposed" % ubuntu.current_series.name if not options.fakesync and not options.lp: - Logger.warn("The use of --no-lp is not recommended for uploads " - "targeted at Ubuntu. " - "The archive-admins discourage its use, except for " - "fakesyncs.") + Logger.warning("The use of --no-lp is not recommended for uploads " + "targeted at Ubuntu. " + "The archive-admins discourage its use, except for " + "fakesyncs.") sponsoree = None if options.sponsoree: @@ -687,17 +691,17 @@ def main(): if blacklist_fail: Logger.error("Source package %s is blacklisted.", src_pkg.source) elif blacklisted == 'ALWAYS': - Logger.normal("Source package %s is blacklisted.", src_pkg.source) + Logger.info(u"Source package %s is blacklisted.", src_pkg.source) if messages: for message in messages: for line in textwrap.wrap(message): - Logger.normal(line) + Logger.info(line) if comments: - Logger.normal("Blacklist Comments:") + Logger.info("Blacklist Comments:") for comment in comments: for line in textwrap.wrap(comment): - Logger.normal(" " + line) + Logger.info(" " + line) if blacklist_fail: sys.exit(1) @@ -717,4 +721,4 @@ if __name__ == "__main__": try: main() except KeyboardInterrupt: - Logger.normal('User abort.') + Logger.info('User abort.') diff --git a/ubuntu-build b/ubuntu-build index 660f895..2083366 100755 --- a/ubuntu-build +++ b/ubuntu-build @@ -32,6 +32,9 @@ from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, from ubuntutools.lp.lpapicache import Distribution, PersonTeam from ubuntutools.misc import split_release_pocket +from ubuntutools import getLogger +Logger = getLogger(__name__) + def main(): # Usage. @@ -108,15 +111,15 @@ def main(): # Check our operation. if op not in ("rescore", "retry", "status"): - print("Invalid operation: %s." % op, file=sys.stderr) + Logger.error("Invalid operation: %s." % op) sys.exit(1) # If the user has specified an architecture to build, we only wish to # rebuild it and nothing else. if options.architecture: if options.architecture[0] not in valid_archs: - print("Invalid architecture specified: %s." - % options.architecture[0], file=sys.stderr) + Logger.error("Invalid architecture specified: %s." + % options.architecture[0]) sys.exit(1) else: one_arch = True @@ -127,7 +130,7 @@ def main(): try: (release, pocket) = split_release_pocket(release) except PocketDoesNotExistError as error: - print('E: %s' % error) + Logger.error(error) sys.exit(1) # Get the ubuntu archive @@ -141,7 +144,7 @@ def main(): sources = ubuntu_archive.getSourcePackage(package, release, pocket) distroseries = Distribution('ubuntu').getSeries(release) except (SeriesNotFoundException, PackageNotFoundException) as error: - print(error) + Logger.error(error) sys.exit(1) # Get list of builds for that package. builds = sources.getBuilds() @@ -163,16 +166,16 @@ def main(): pocket=pocket) if op in ('rescore', 'retry') and not necessary_privs: - print(("You cannot perform the %s operation on a %s package as " - "you do not have the permissions to do this action.") - % (op, component), file=sys.stderr) + Logger.error("You cannot perform the %s operation on a %s " + "package as you do not have the permissions " + "to do this action." % (op, component)) sys.exit(1) # Output details. - print("The source version for '%s' in %s (%s) is at %s." - % (package, release.capitalize(), component, version)) + Logger.info("The source version for '%s' in %s (%s) is at %s." % + (package, release.capitalize(), component, version)) - print("Current build status for this package:") + Logger.info("Current build status for this package:") # Output list of arches for package and their status. done = False @@ -182,29 +185,28 @@ def main(): continue done = True - print("%s: %s." % (build.arch_tag, build.buildstate)) + Logger.info("%s: %s." % (build.arch_tag, build.buildstate)) if op == 'rescore': if build.can_be_rescored: # FIXME: make priority an option priority = 5000 - print('Rescoring build %s to %d...' - % (build.arch_tag, priority)) + Logger.info('Rescoring build %s to %d...' % (build.arch_tag, priority)) build.rescore(score=priority) else: - print('Cannot rescore build on %s.' % build.arch_tag) + Logger.info('Cannot rescore build on %s.' % build.arch_tag) if op == 'retry': if build.can_be_retried: - print('Retrying build on %s...' % build.arch_tag) + Logger.info('Retrying build on %s...' % build.arch_tag) build.retry() else: - print('Cannot retry build on %s.' % build.arch_tag) + Logger.info('Cannot retry build on %s.' % build.arch_tag) # We are done if done: sys.exit(0) - print(("No builds for '%s' found in the %s release - it may have been " - "built in a former release.") % (package, release.capitalize())) + Logger.info("No builds for '%s' found in the %s release" % (package, release.capitalize())) + Logger.info("It may have been built in a former release.") sys.exit(0) # Batch mode @@ -225,14 +227,14 @@ def main(): try: (release, pocket) = split_release_pocket(release) except PocketDoesNotExistError as error: - print('E: %s' % error) + Logger.error(error) sys.exit(1) ubuntu_archive = Distribution('ubuntu').getArchive() try: distroseries = Distribution('ubuntu').getSeries(release) except SeriesNotFoundException as error: - print(error) + Logger.error(error) sys.exit(1) me = PersonTeam.me @@ -241,14 +243,14 @@ def main(): and me.isLpTeamMember('launchpad-buildd-admins')) or False) if options.priority and not can_rescore: - print("You don't have the permissions to rescore builds. " - "Ignoring your rescore request.", file=sys.stderr) + Logger.error("You don't have the permissions to rescore " + "builds. Ignoring your rescore request.") for pkg in args: try: pkg = ubuntu_archive.getSourcePackage(pkg, release, pocket) except PackageNotFoundException as error: - print(error) + Logger.error(error) continue # Check permissions (part 2): check upload permissions for the source @@ -258,20 +260,20 @@ def main(): pkg.getPackageName(), pkg.getComponent()) if options.retry and not can_retry: - print(("You don't have the permissions to retry the build of " - "'%s'. Ignoring your request.") - % pkg.getPackageName(), file=sys.stderr) + Logger.error("You don't have the permissions to retry the " + "build of '%s'. Ignoring your request." + % pkg.getPackageName()) - print("The source version for '%s' in '%s' (%s) is: %s" - % (pkg.getPackageName(), release, pocket, pkg.getVersion())) + Logger.info("The source version for '%s' in '%s' (%s) is: %s" % + (pkg.getPackageName(), release, pocket, pkg.getVersion())) - print(pkg.getBuildStates(archs)) + Logger.info(pkg.getBuildStates(archs)) if can_retry: - print(pkg.retryBuilds(archs)) + Logger.info(pkg.retryBuilds(archs)) if options.priority and can_rescore: - print(pkg.rescoreBuilds(archs, options.priority)) + Logger.info(pkg.rescoreBuilds(archs, options.priority)) - print() + Logger.info('') if __name__ == '__main__': diff --git a/ubuntu-iso b/ubuntu-iso index fd56a66..c07b420 100755 --- a/ubuntu-iso +++ b/ubuntu-iso @@ -24,6 +24,9 @@ import optparse import subprocess import sys +from ubuntutools import getLogger +Logger = getLogger(__name__) + def extract(iso, path): command = ['isoinfo', '-R', '-i', iso, '-x', path] @@ -54,12 +57,11 @@ def main(): version = extract(iso, '/.disk/info') if len(version) == 0: - print('%s does not appear to be an Ubuntu ISO' % iso, - file=sys.stderr) + Logger.error('%s does not appear to be an Ubuntu ISO' % iso) err = True continue - print(prefix + version) + Logger.info(prefix + version) if err: sys.exit(1) diff --git a/ubuntu-upload-permission b/ubuntu-upload-permission index 2099134..1debf86 100755 --- a/ubuntu-upload-permission +++ b/ubuntu-upload-permission @@ -20,9 +20,11 @@ import sys from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, Packageset, PackageNotFoundException, SeriesNotFoundException) -from ubuntutools.logger import Logger from ubuntutools.misc import split_release_pocket +from ubuntutools import getLogger +Logger = getLogger(__name__) + def parse_arguments(): '''Parse arguments and return (options, package)''' @@ -77,52 +79,50 @@ def main(): component_uploader = archive.getUploadersForComponent( component_name=component)[0] - print("All upload permissions for %s:" % package) - print() - print("Component (%s)" % component) - print("============" + ("=" * len(component))) + Logger.info("All upload permissions for %s:" % package) + Logger.info("") + Logger.info("Component (%s)" % component) + Logger.info("============" + ("=" * len(component))) print_uploaders([component_uploader], options.list_team_members) packagesets = sorted(Packageset.setsIncludingSource( distroseries=series, sourcepackagename=package)) if packagesets: - print() - print("Packagesets") - print("===========") + Logger.info("") + Logger.info("Packagesets") + Logger.info("===========") for packageset in packagesets: - print() - print("%s:" % packageset.name) + Logger.info("") + Logger.info("%s:" % packageset.name) print_uploaders(archive.getUploadersForPackageset( packageset=packageset), options.list_team_members) ppu_uploaders = archive.getUploadersForPackage( source_package_name=package) if ppu_uploaders: - print() - print("Per-Package-Uploaders") - print("=====================") - print() + Logger.info("") + Logger.info("Per-Package-Uploaders") + Logger.info("=====================") + Logger.info("") print_uploaders(ppu_uploaders, options.list_team_members) - print() + Logger.info("") if PersonTeam.me.canUploadPackage(archive, series, package, component, pocket): - print("You can upload %s to %s." % (package, options.release)) + Logger.info("You can upload %s to %s." % (package, options.release)) else: - print("You can not upload %s to %s, yourself." - % (package, options.release)) + Logger.info("You can not upload %s to %s, yourself." % (package, options.release)) if (series.status in ('Current Stable Release', 'Supported', 'Obsolete') and pocket == 'Release'): - print(("%s is in the '%s' state. You may want to query the " - "%s-proposed pocket.") - % (release, series.status, release)) + Logger.info("%s is in the '%s' state. You may want to query the %s-proposed pocket." % + (release, series.status, release)) else: - print("But you can still contribute to it via the sponsorship " - "process: https://wiki.ubuntu.com/SponsorshipProcess") + Logger.info("But you can still contribute to it via the sponsorship " + "process: https://wiki.ubuntu.com/SponsorshipProcess") if not options.list_uploaders: - print("To see who has the necessary upload rights, " - "use the --list-uploaders option.") + Logger.info("To see who has the necessary upload rights, " + "use the --list-uploaders option.") sys.exit(1) @@ -133,9 +133,9 @@ def print_uploaders(uploaders, expand_teams=False, prefix=''): recursion. """ for uploader in sorted(uploaders, key=lambda p: p.display_name): - print(("%s* %s (%s)%s" % - (prefix, uploader.display_name, uploader.name, - ' [team]' if uploader.is_team else ''))) + Logger.info("%s* %s (%s)%s" % + (prefix, uploader.display_name, uploader.name, + ' [team]' if uploader.is_team else '')) if expand_teams and uploader.is_team: print_uploaders(uploader.participants, True, prefix=prefix + ' ') diff --git a/ubuntutools/__init__.py b/ubuntutools/__init__.py index 40a7c42..f886333 100644 --- a/ubuntutools/__init__.py +++ b/ubuntutools/__init__.py @@ -2,3 +2,20 @@ # # Ubuntu Development Tools # https://launchpad.net/ubuntu-dev-tools + +import logging + + +def _loggingBasicConfig(**kwargs): + '''Set log level to INFO and define log format to use.''' + if 'level' not in kwargs: + kwargs['level'] = logging.INFO + if 'format' not in kwargs: + kwargs['format'] = '%(message)s' + logging.basicConfig(**kwargs) + + +def getLogger(name=None): + '''Get standard Python logging.Logger with some ubuntutools defaults.''' + _loggingBasicConfig() + return logging.getLogger(name) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 96ed316..43f4034 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -35,6 +35,7 @@ import hashlib import json import os.path import re +import shutil import subprocess import sys @@ -50,9 +51,11 @@ from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, from ubuntutools.lp.udtexceptions import (PackageNotFoundException, SeriesNotFoundException, InvalidDistroValueError) -from ubuntutools.logger import Logger from ubuntutools.version import Version +import logging +Logger = logging.getLogger(__name__) + class DownloadError(Exception): "Unable to pull a source package" @@ -139,7 +142,6 @@ class SourcePackage(object): lp = kwargs.get('lp') mirrors = kwargs.get('mirrors', ()) workdir = kwargs.get('workdir', '.') - quiet = kwargs.get('quiet', False) series = kwargs.get('series') pocket = kwargs.get('pocket') status = kwargs.get('status') @@ -152,7 +154,6 @@ class SourcePackage(object): self.binary = None self.try_binary = True self.workdir = workdir - self.quiet = quiet self._series = series self._pocket = pocket self._status = status @@ -216,21 +217,21 @@ class SourcePackage(object): # or we've already tried raise pnfe - Logger.normal('Source package lookup failed, ' - 'trying lookup of binary package %s' % self.source) + Logger.info('Source package lookup failed, ' + 'trying lookup of binary package %s' % self.source) try: bpph = archive.getBinaryPackage(self.source, **params) except PackageNotFoundException as bpnfe: # log binary lookup failure, in case it provides hints - Logger.normal(str(bpnfe)) + Logger.info(str(bpnfe)) # raise the original exception for the source lookup raise pnfe self.binary = self.source self.source = bpph.getSourcePackageName() - Logger.normal("Using source package '{}' for binary package '{}'" - .format(self.source, self.binary)) + Logger.info("Using source package '{}' for binary package '{}'" + .format(self.source, self.binary)) spph = bpph.getBuild().getSourcePackagePublishingHistory() if spph: @@ -394,14 +395,14 @@ class SourcePackage(object): message = 'Public key not found, could not verify signature' if self._verify_signature: if valid: - Logger.normal(message) + Logger.info(message) elif no_pub_key: - Logger.warn(message) + Logger.warning(message) else: Logger.error(message) raise DownloadError(message) else: - Logger.info(message) + Logger.debug(message) def _write_dsc(self): "Write dsc file to workdir" @@ -500,9 +501,9 @@ class SourcePackage(object): if self._download_file(url, name): break except HTTPError as e: - Logger.normal('HTTP Error %i: %s', e.code, str(e)) + Logger.info('HTTP Error %i: %s', e.code, str(e)) except URLError as e: - Logger.normal('URL Error: %s', e.reason) + Logger.info('URL Error: %s', e.reason) else: raise DownloadError('File %s could not be found' % name) @@ -525,13 +526,13 @@ class SourcePackage(object): found = True break except HTTPError as e: - Logger.normal('HTTP Error %i: %s', e.code, str(e)) + Logger.info('HTTP Error %i: %s', e.code, str(e)) except URLError as e: - Logger.normal('URL Error: %s', e.reason) + Logger.info('URL Error: %s', e.reason) if found: total += 1 else: - Logger.normal("Could not download from any location: %s", fname) + Logger.info("Could not download from any location: %s", fname) return total def verify(self): @@ -557,7 +558,7 @@ class SourcePackage(object): cmd = ['dpkg-source', '-x', self.dsc_name] if destdir: cmd.append(destdir) - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd, cwd=self.workdir): Logger.error('Source unpack failed.') sys.exit(1) @@ -569,14 +570,14 @@ class SourcePackage(object): """ cmd = ['debdiff', self.dsc_name, newpkg.dsc_name] difffn = newpkg.dsc_name[:-3] + 'debdiff' - Logger.command(cmd + ['> %s' % difffn]) + Logger.debug(' '.join(cmd) + ('> %s' % difffn)) with open(difffn, 'w') as f: if subprocess.call(cmd, stdout=f, cwd=self.workdir) > 2: Logger.error('Debdiff failed.') sys.exit(1) if diffstat: cmd = ('diffstat', '-p1', difffn) - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd): Logger.error('diffstat failed.') sys.exit(1) @@ -590,7 +591,7 @@ class DebianSPPH(SourcePackagePublishingHistory): resource_type = 'source_package_publishing_history' def getBinaries(self, arch, name=None): - Logger.normal('Using Snapshot to find binary packages') + Logger.info('Using Snapshot to find binary packages') srcpkg = Snapshot.getSourcePackage(self.getPackageName(), version=self.getVersion()) return srcpkg.getSPPH().getBinaries(arch=arch, name=name) @@ -631,7 +632,7 @@ class DebianSourcePackage(SourcePackage): except SeriesNotFoundException: pass - Logger.normal('Package not found in Launchpad, using Snapshot') + Logger.info('Package not found in Launchpad, using Snapshot') self._spph = self.snapshot_package.getSPPH() return self._spph @@ -692,7 +693,7 @@ class DebianSourcePackage(SourcePackage): self._snapshot_package = srcpkg else: # we have neither version nor spph, so look up our version using madison - Logger.normal('Using madison to find latest version number') + Logger.info('Using madison to find latest version number') series = self._series params = {'series': series} if series else {} srcpkg = Madison(self.distribution).getSourcePackage(self.source, **params) @@ -899,10 +900,10 @@ class _Snapshot(_WebJSON): if s.startswith('pool'): found_pool = True if not component: - Logger.warn("could not determine component from path %s" % path) + Logger.warning("could not determine component from path %s" % path) return self.DEBIAN_COMPONENTS[0] if component not in self.DEBIAN_COMPONENTS: - Logger.warn("unexpected component %s" % component) + Logger.warning("unexpected component %s" % component) return component def _get_package(self, name, url, pkginit, version, sort_key): diff --git a/ubuntutools/builder.py b/ubuntutools/builder.py index 8ab478f..c082602 100644 --- a/ubuntutools/builder.py +++ b/ubuntutools/builder.py @@ -21,7 +21,8 @@ import os import subprocess -from ubuntutools.logger import Logger +import logging +Logger = logging.getLogger(__name__) def _build_preparation(result_directory): @@ -71,7 +72,7 @@ class Pbuilder(Builder): self.name, "--build", "--architecture", self.architecture, "--distribution", dist, "--buildresult", result_directory, dsc_file] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) returncode = subprocess.call(cmd) return self._build_failure(returncode, dsc_file) @@ -79,7 +80,7 @@ class Pbuilder(Builder): cmd = ["sudo", "-E", "ARCH=" + self.architecture, "DIST=" + dist, self.name, "--update", "--architecture", self.architecture, "--distribution", dist] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) returncode = subprocess.call(cmd) return self._update_failure(returncode, dist) @@ -92,13 +93,13 @@ class Pbuilderdist(Builder): _build_preparation(result_directory) cmd = [self.name, dist, self.architecture, "build", dsc_file, "--buildresult", result_directory] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) returncode = subprocess.call(cmd) return self._build_failure(returncode, dsc_file) def update(self, dist): cmd = [self.name, dist, self.architecture, "update"] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) returncode = subprocess.call(cmd) return self._update_failure(returncode, dist) @@ -110,19 +111,19 @@ class Sbuild(Builder): def build(self, dsc_file, dist, result_directory): _build_preparation(result_directory) workdir = os.getcwd() - Logger.command(["cd", result_directory]) + Logger.debug("cd " + result_directory) os.chdir(result_directory) cmd = ["sbuild", "--arch-all", "--dist=" + dist, "--arch=" + self.architecture, dsc_file] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) returncode = subprocess.call(cmd) - Logger.command(["cd", workdir]) + Logger.debug("cd " + workdir) os.chdir(workdir) return self._build_failure(returncode, dsc_file) def update(self, dist): cmd = ["schroot", "--list"] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) process = subprocess.run(cmd, stdout=subprocess.PIPE, encoding='utf-8') chroots, _ = process.stdout.strip().split() if process.returncode != 0: @@ -144,7 +145,7 @@ class Sbuild(Builder): ["sbuild-clean", "-a", "-c"]] for cmd in commands: # pylint: disable=W0631 - Logger.command(cmd + [chroot]) + Logger.debug(' '.join(cmd) + " " + chroot) ret = subprocess.call(cmd + [chroot]) # pylint: enable=W0631 if ret != 0: diff --git a/ubuntutools/config.py b/ubuntutools/config.py index dfa53a3..5cc14ce 100644 --- a/ubuntutools/config.py +++ b/ubuntutools/config.py @@ -23,7 +23,8 @@ import socket import sys import locale -from ubuntutools.logger import Logger +import logging +Logger = logging.getLogger(__name__) class UDTConfig(object): @@ -71,8 +72,8 @@ class UDTConfig(object): for line in f: parsed = shlex.split(line, comments=True) if len(parsed) > 1: - Logger.warn('Cannot parse variable assignment in %s: %s', - getattr(f, 'name', ''), line) + Logger.warning('Cannot parse variable assignment in %s: %s', + getattr(f, 'name', ''), line) if len(parsed) >= 1 and '=' in parsed[0]: key, value = parsed[0].split('=', 1) config[key] = value @@ -111,10 +112,8 @@ class UDTConfig(object): replacements = self.prefix + '_' + key if key in self.defaults: replacements += 'or UBUNTUTOOLS_' + key - Logger.warn( - 'Using deprecated configuration variable %s. ' - 'You should use %s.', - k, replacements) + Logger.warning('Using deprecated configuration variable %s. ' + 'You should use %s.', k, replacements) return value return default diff --git a/ubuntutools/logger.py b/ubuntutools/logger.py deleted file mode 100644 index 3c04ca4..0000000 --- a/ubuntutools/logger.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# logger.py - A simple logging helper class -# -# Copyright (C) 2010, Benjamin Drung -# -# Permission to use, copy, modify, and/or distribute this software -# for any purpose with or without fee is hereby granted, provided -# that the above copyright notice and this permission notice appear -# in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE -# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import os -import sys - - -def escape_arg(arg): - """Shell-escpae arg, if necessary. - Fairly simplistic, doesn't escape anything except whitespace. - """ - if ' ' not in arg: - return arg - return '"%s"' % arg.replace('\\', r'\\').replace('"', r'\"') - - -class Logger(object): - script_name = os.path.basename(sys.argv[0]) - verbose = False - - stdout = sys.stdout - stderr = sys.stderr - - @classmethod - def _print(cls, format_, message, args=None, stderr=False): - if args: - message = message % args - stream = cls.stderr if stderr else cls.stdout - stream.write((format_ + "\n") % (cls.script_name, message)) - - @classmethod - def command(cls, cmd): - if cls.verbose: - cls._print("%s: I: %s", " ".join(escape_arg(arg) for arg in cmd)) - - @classmethod - def debug(cls, message, *args): - if cls.verbose: - cls._print("%s: D: %s", message, args, stderr=True) - - @classmethod - def error(cls, message, *args): - cls._print("%s: Error: %s", message, args, stderr=True) - - @classmethod - def warn(cls, message, *args): - cls._print("%s: Warning: %s", message, args, stderr=True) - - @classmethod - def info(cls, message, *args): - if cls.verbose: - cls._print("%s: I: %s", message, args) - - @classmethod - def normal(cls, message, *args): - cls._print("%s: %s", message, args) - - @classmethod - def set_verbosity(cls, verbose): - cls.verbose = verbose diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index a821ebc..de465d2 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -27,7 +27,6 @@ import collections import re -import sys from debian.changelog import Changelog from httplib2 import Http, HttpLib2Error @@ -47,6 +46,9 @@ from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError, PocketDoesNotExistError, SeriesNotFoundException) +import logging +Logger = logging.getLogger(__name__) + __all__ = [ 'Archive', @@ -72,7 +74,7 @@ class _Launchpad(object): self.__lp = LP.login_with('ubuntu-dev-tools', service, version=api_version) except IOError as error: - print('E: %s' % error, file=sys.stderr) + Logger.error(str(error)) raise else: raise AlreadyLoggedInError('Already logged in to Launchpad.') @@ -153,6 +155,7 @@ class BaseWrapper(object, metaclass=MetaWrapper): cached._lpobject = data # and add it to our cache cls._cache[data.self_link] = cached + Logger.debug("%s: %s" % (cls.__name__, data.self_link)) # add additional class specific caching (if available) cache = getattr(cls, 'cache', None) if isinstance(cache, collections.Callable): @@ -765,17 +768,17 @@ class SourcePackagePublishingHistory(BaseWrapper): if self._changelog is None: url = self._lpobject.changelogUrl() if url is None: - print('E: No changelog available for %s %s' % - (self.getPackageName(), self.getVersion()), file=sys.stderr) + Logger.error('No changelog available for %s %s' % + (self.getPackageName(), self.getVersion())) return None try: response, changelog = Http().request(url) except HttpLib2Error as e: - print(str(e), file=sys.stderr) + Logger.error(str(e)) return None if response.status != 200: - print('%s: %s %s' % (url, response.status, response.reason), file=sys.stderr) + Logger.error('%s: %s %s' % (url, response.status, response.reason)) return None self._changelog = changelog @@ -824,7 +827,7 @@ class SourcePackagePublishingHistory(BaseWrapper): self._have_all_binaries = True else: # we have to go the long way :( - print("Please wait, this may take some time...") + Logger.info("Please wait, this may take some time...") archive = self.getArchive() urls = self.binaryFileUrls() for url in urls: @@ -854,7 +857,7 @@ class SourcePackagePublishingHistory(BaseWrapper): try: bpph = archive.getBinaryPackage(**params) except PackageNotFoundException: - print("Could not find pkg in archive: %s" % filename) + Logger.debug("Could not find pkg in archive: %s" % filename) continue if a not in self._binaries: self._binaries[a] = {} diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index 11f1bd8..2e7a4ac 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -39,9 +39,13 @@ from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, PackageNotFoundException, PocketDoesNotExistError, InvalidDistroValueError) -from ubuntutools.logger import Logger from ubuntutools.misc import (split_release_pocket, host_architecture, STATUSES) +from ubuntutools import _loggingBasicConfig + +import logging +Logger = logging.getLogger(__name__) + PULL_SOURCE = 'source' PULL_DEBS = 'debs' PULL_DDEBS = 'ddebs' @@ -76,18 +80,22 @@ class PullPkg(object): """For use by stand-alone cmdline scripts. This will handle catching certain exceptions or kbd interrupts, - and printing out (via Logger) the error message, instead of - allowing the exception to flow up to the script. This does - not catch unexpected exceptions, such as internal errors. + setting up the root logger level to INFO, and printing out + (via Logger) a caught error message, instead of allowing the + exception to flow up to the script. This does not catch + unexpected exceptions, such as internal errors. + On (expected) error, this will call sys.exit(error); unexpected errors will flow up to the caller. On success, this simply returns. """ + _loggingBasicConfig() + try: cls(*args, **kwargs).pull() return except KeyboardInterrupt: - Logger.normal('User abort.') + Logger.info('User abort.') except (PackageNotFoundException, SeriesNotFoundException, PocketDoesNotExistError, InvalidDistroValueError) as e: Logger.error(str(e)) @@ -120,7 +128,7 @@ class PullPkg(object): parser = ArgumentParser(epilog=epilog) parser.add_argument('-v', '--verbose', action='store_true', - help="Print verbose/debug messages") + help="Print debug messages") parser.add_argument('-d', '--download-only', action='store_true', help="Do not extract the source package") parser.add_argument('-m', '--mirror', action='append', @@ -231,7 +239,7 @@ class PullPkg(object): debian_info = DebianDistroInfo() codename = debian_info.codename(release) if codename: - Logger.normal("Using release '%s' for '%s'", codename, release) + Logger.info("Using release '%s' for '%s'", codename, release) release = codename if distro == DISTRO_PPA: @@ -335,8 +343,8 @@ class PullPkg(object): options = vars(self.argparser.parse_args(args)) assert 'verbose' in options - if options['verbose'] is not None: - Logger.set_verbosity(options['verbose']) + if options['verbose']: + Logger.setLevel(logging.DEBUG) Logger.debug("pullpkg options: %s", options) @@ -349,15 +357,15 @@ class PullPkg(object): srcpkg = DISTRO_PKG_CLASS[distro](**params) spph = srcpkg.lp_spph - Logger.normal('Found %s', spph.display_name) + Logger.info('Found %s', spph.display_name) if pull == PULL_LIST: - Logger.normal("Source files:") + Logger.info("Source files:") for f in srcpkg.dsc['Files']: - Logger.normal(" %s", f['name']) - Logger.normal("Binary files:") + Logger.info(" %s", f['name']) + Logger.info("Binary files:") for f in spph.getBinaries(options['arch']): - Logger.normal(" %s", f.getFileName()) + Logger.info(" %s", f.getFileName()) elif pull == PULL_SOURCE: # allow DownloadError to flow up to caller srcpkg.pull() @@ -368,9 +376,9 @@ class PullPkg(object): else: name = '.*' if params['package'] != spph.getPackageName(): - Logger.normal("Pulling only binary package '%s'", params['package']) - Logger.normal("Use package name '%s' to pull all binary packages", - spph.getPackageName()) + Logger.info("Pulling only binary package '%s'", params['package']) + Logger.info("Use package name '%s' to pull all binary packages", + spph.getPackageName()) name = params['package'] if pull == PULL_DEBS: name = r'{}(?", self._debdiff_filename]) + Logger.debug(' '.join(cmd) + " > " + self._debdiff_filename) debdiff = subprocess.check_output(cmd, encoding='utf-8') # write debdiff file @@ -417,7 +419,7 @@ class SourcePackage(object): lintian_filename = os.path.join(self._workdir, self._package + "_" + strip_epoch(self._version) + ".lintian") - Logger.command(cmd + [">", lintian_filename]) + Logger.debug(' '.join(cmd) + " > " + lintian_filename) report = subprocess.check_output(cmd, encoding='utf-8') # write lintian report file @@ -434,7 +436,7 @@ class SourcePackage(object): cmd = ["syncpackage", self._package, "-b", str(bug_number), "-f", "-s", requester, "-V", str(self._version), "-d", series] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd) != 0: Logger.error("Syncing of %s %s failed.", self._package, str(self._version)) diff --git a/ubuntutools/sponsor_patch/sponsor_patch.py b/ubuntutools/sponsor_patch/sponsor_patch.py index d1d1b62..d873a81 100644 --- a/ubuntutools/sponsor_patch/sponsor_patch.py +++ b/ubuntutools/sponsor_patch/sponsor_patch.py @@ -25,7 +25,6 @@ from distro_info import UbuntuDistroInfo from launchpadlib.launchpad import Launchpad -from ubuntutools.logger import Logger from ubuntutools.update_maintainer import (update_maintainer, MaintainerUpdateException) from ubuntutools.question import input_number @@ -35,6 +34,9 @@ from ubuntutools.sponsor_patch.patch import Patch from ubuntutools.sponsor_patch.question import ask_for_manual_fixing from ubuntutools.sponsor_patch.source_package import SourcePackage +import logging +Logger = logging.getLogger(__name__) + def is_command_available(command, check_sbin=False): "Is command in $PATH?" @@ -59,8 +61,8 @@ def check_dependencies(): missing.append('pbuilder/cowbuilder/sbuild') if missing: - Logger.warn("sponsor-patch requires %s to be installed for full " - "functionality", ', '.join(missing)) + Logger.warning("sponsor-patch requires %s to be installed for full " + "functionality", ', '.join(missing)) def get_source_package_name(bug_task): @@ -82,7 +84,7 @@ def get_user_shell(): def edit_source(): # Spawn shell to allow modifications cmd = [get_user_shell()] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) print("""An interactive shell was launched in file://%s Edit your files. When you are done, exit the shell. If you wish to abort the @@ -112,7 +114,7 @@ def ask_for_patch_or_branch(bug, attached_patches, linked_branches): patches += "es" msg = "https://launchpad.net/bugs/%i has %s linked and %s attached:" % \ (bug.id, branches, patches) - Logger.normal(msg) + Logger.info(msg) i = 0 for linked_branch in linked_branches: i += 1 @@ -160,7 +162,7 @@ def download_branch(branch): if os.path.isdir(dir_name): shutil.rmtree(dir_name) cmd = ["bzr", "branch", branch] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd) != 0: Logger.error("Failed to download branch %s." % (branch)) sys.exit(1) @@ -170,7 +172,7 @@ def download_branch(branch): def merge_branch(branch): edit = False cmd = ["bzr", "merge", branch] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd) != 0: Logger.error("Failed to merge branch %s." % (branch)) ask_for_manual_fixing() @@ -182,7 +184,7 @@ def extract_source(dsc_file, verbose=False): cmd = ["dpkg-source", "--no-preparation", "-x", dsc_file] if not verbose: cmd.insert(1, "-q") - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd) != 0: Logger.error("Extraction of %s failed." % (os.path.basename(dsc_file))) sys.exit(1) @@ -219,21 +221,21 @@ def get_open_ubuntu_bug_task(launchpad, bug, branch=None): task = tasks[0] elif len(ubuntu_tasks) > 1: task_list = [t.get_short_info() for t in ubuntu_tasks] - Logger.info("%i Ubuntu tasks exist for bug #%i.\n%s", len(ubuntu_tasks), - bug_id, "\n".join(task_list)) + Logger.debug("%i Ubuntu tasks exist for bug #%i.\n%s", len(ubuntu_tasks), + bug_id, "\n".join(task_list)) open_ubuntu_tasks = [x for x in ubuntu_tasks if not x.is_complete()] if len(open_ubuntu_tasks) == 1: task = open_ubuntu_tasks[0] else: - Logger.normal("https://launchpad.net/bugs/%i has %i Ubuntu tasks:" % - (bug_id, len(ubuntu_tasks))) + Logger.info("https://launchpad.net/bugs/%i has %i Ubuntu tasks:" % + (bug_id, len(ubuntu_tasks))) for i in range(len(ubuntu_tasks)): print("%i) %s" % (i + 1, ubuntu_tasks[i].get_package_and_series())) selected = input_number("To which Ubuntu task does the patch belong", 1, len(ubuntu_tasks)) task = ubuntu_tasks[selected - 1] - Logger.info("Selected Ubuntu task: %s" % (task.get_short_info())) + Logger.debug("Selected Ubuntu task: %s" % (task.get_short_info())) return task @@ -248,15 +250,15 @@ def _create_and_change_into(workdir): (workdir, error.errno, error.strerror)) sys.exit(1) if workdir != os.getcwd(): - Logger.command(["cd", workdir]) + Logger.debug("cd " + workdir) os.chdir(workdir) def _update_maintainer_field(): """Update the Maintainer field in debian/control.""" - Logger.command(["update-maintainer"]) + Logger.debug("update-maintainer") try: - update_maintainer("debian", Logger.verbose) + update_maintainer("debian", Logger.isEnabledFor(logging.DEBUG)) except MaintainerUpdateException as e: Logger.error("update-maintainer failed: %s", str(e)) sys.exit(1) @@ -265,9 +267,9 @@ def _update_maintainer_field(): def _update_timestamp(): """Run dch to update the timestamp of debian/changelog.""" cmd = ["dch", "--maintmaint", "--release", ""] - Logger.command(cmd) + Logger.debug(' '.join(cmd)) if subprocess.call(cmd) != 0: - Logger.info("Failed to update timestamp in debian/changelog.") + Logger.debug("Failed to update timestamp in debian/changelog.") def _download_and_change_into(task, dsc_file, patch, branch): @@ -277,23 +279,23 @@ def _download_and_change_into(task, dsc_file, patch, branch): branch_dir = download_branch(task.get_branch_link()) # change directory - Logger.command(["cd", branch_dir]) + Logger.debug("cd " + branch_dir) os.chdir(branch_dir) else: if patch: patch.download() - Logger.info("Ubuntu package: %s" % (task.package)) + Logger.debug("Ubuntu package: %s" % (task.package)) if task.is_merge(): - Logger.info("The task is a merge request.") + Logger.debug("The task is a merge request.") if task.is_sync(): - Logger.info("The task is a sync request.") + Logger.debug("The task is a sync request.") - extract_source(dsc_file, Logger.verbose) + extract_source(dsc_file, Logger.isEnabledFor(logging.DEBUG)) # change directory directory = task.package + '-' + task.get_version().upstream_version - Logger.command(["cd", directory]) + Logger.debug("cd " + directory) os.chdir(directory) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index 9483b8d..46f6c11 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -92,10 +92,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): self.url_opener = mock.MagicMock(spec=OpenerDirector) self.url_opener.open.side_effect = self.urlopen_proxy - # Silence the tests a little: - self._stubout('ubuntutools.logger.Logger.stdout') - self._stubout('ubuntutools.logger.Logger.stderr') - def _stubout(self, stub): patcher = mock.patch(stub) self.addCleanup(patcher.stop) @@ -151,7 +147,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): dscfile='test-data/example_1.0-1.dsc', workdir=self.workdir, verify_signature=False) - pkg.quiet = True pkg.pull() pkg.unpack() @@ -164,7 +159,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): 'example_1.0-1.dsc'), workdir=self.workdir, verify_signature=False) - pkg.quiet = True pkg.pull() pkg.unpack() @@ -179,7 +173,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): 'example_1.0-1.dsc'), workdir=self.workdir, verify_signature=False) - pkg.quiet = True pkg.pull() pkg.unpack() @@ -197,7 +190,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): dscfile='test-data/example_1.0-1.dsc', workdir=self.workdir, verify_signature=False) - pkg.quiet = True pkg.pull() def test_pull(self): @@ -208,7 +200,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): verify_signature=False) pkg.url_opener = self.url_opener - pkg.quiet = True pkg.pull() def test_mirrors(self): @@ -228,7 +219,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): mirrors=[mirror], verify_signature=False) pkg.url_opener = url_opener - pkg.quiet = True pkg.pull() def test_dsc_missing(self): @@ -237,7 +227,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): version='1.0-1', component='main', workdir=self.workdir) - pkg.quiet = True self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) @@ -267,7 +256,6 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): workdir=self.workdir, mirrors=[debian_mirror, debsec_mirror], verify_signature=False) - pkg.quiet = True pkg.url_opener = url_opener pkg.pull() pkg.unpack() diff --git a/ubuntutools/test/test_config.py b/ubuntutools/test/test_config.py index 08b37fb..f8653a6 100644 --- a/ubuntutools/test/test_config.py +++ b/ubuntutools/test/test_config.py @@ -18,11 +18,10 @@ import locale import mock import os -import sys +# import sys from io import StringIO from ubuntutools.config import UDTConfig, ubu_email -from ubuntutools.logger import Logger from ubuntutools.test import unittest @@ -51,16 +50,16 @@ class ConfigTestCase(unittest.TestCase): self.addCleanup(patcher.stop) patcher.start() - Logger.stdout = StringIO() - Logger.stderr = StringIO() + # Logger.stdout = StringIO() + # Logger.stderr = StringIO() self.clean_environment() def tearDown(self): - self.assertEqual(Logger.stdout.getvalue(), '') - self.assertEqual(Logger.stderr.getvalue(), '') - Logger.stdout = sys.stdout - Logger.stderr = sys.stderr + # self.assertEqual(Logger.stdout.getvalue(), '') + # self.assertEqual(Logger.stderr.getvalue(), '') + # Logger.stdout = sys.stdout + # Logger.stderr = sys.stderr self.clean_environment() @@ -98,11 +97,11 @@ REPEAT=yes 'INHERIT': 'user', 'REPEAT': 'yes', }) - errs = Logger.stderr.getvalue().strip() - Logger.stderr = StringIO() - self.assertEqual(len(errs.splitlines()), 1) - self.assertRegex(errs, - r'Warning: Cannot parse.*\bCOMMAND_EXECUTION=a') + # errs = Logger.stderr.getvalue().strip() + # Logger.stderr = StringIO() + # self.assertEqual(len(errs.splitlines()), 1) + # self.assertRegex(errs, + # r'Warning: Cannot parse.*\bCOMMAND_EXECUTION=a') def get_value(self, *args, **kwargs): config = UDTConfig(prefix='TEST') @@ -138,11 +137,11 @@ REPEAT=yes self._config_files['user'] = 'COMPATFOOBAR=bar' self.assertEqual(self.get_value('QUX', compat_keys=['COMPATFOOBAR']), 'bar') - errs = Logger.stderr.getvalue().strip() - Logger.stderr = StringIO() - self.assertEqual(len(errs.splitlines()), 1) - self.assertRegex(errs, - r'deprecated.*\bCOMPATFOOBAR\b.*\bTEST_QUX\b') + # errs = Logger.stderr.getvalue().strip() + # Logger.stderr = StringIO() + # self.assertEqual(len(errs.splitlines()), 1) + # self.assertRegex(errs, + # r'deprecated.*\bCOMPATFOOBAR\b.*\bTEST_QUX\b') def test_boolean(self): self._config_files['user'] = "TEST_BOOLEAN=yes" diff --git a/ubuntutools/test/test_logger.py b/ubuntutools/test/test_logger.py deleted file mode 100644 index 3f7fec3..0000000 --- a/ubuntutools/test/test_logger.py +++ /dev/null @@ -1,54 +0,0 @@ -# test_logger.py - Test ubuntutools.logger.Logger. -# -# Copyright (C) 2012, Stefano Rivera -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -from io import StringIO -import sys - -from ubuntutools.logger import Logger -from ubuntutools.test import unittest - - -class LoggerTestCase(unittest.TestCase): - def setUp(self): - Logger.stdout = StringIO() - Logger.stderr = StringIO() - self._script_name = Logger.script_name - Logger.script_name = 'test' - self._verbose = Logger.verbose - - def tearDown(self): - Logger.stdout = sys.stdout - Logger.stderr = sys.stderr - Logger.script_name = self._script_name - Logger.verbose = self._verbose - - def testCommand(self): - Logger.command(('ls', 'a b')) - self.assertEqual(Logger.stdout.getvalue(), '') - Logger.set_verbosity(True) - Logger.command(('ls', 'a b')) - self.assertEqual(Logger.stdout.getvalue(), 'test: I: ls "a b"\n') - self.assertEqual(Logger.stderr.getvalue(), '') - - def testNoArgs(self): - Logger.normal('hello %s') - self.assertEqual(Logger.stdout.getvalue(), 'test: hello %s\n') - self.assertEqual(Logger.stderr.getvalue(), '') - - def testArgs(self): - Logger.normal('hello %s', 'world') - self.assertEqual(Logger.stdout.getvalue(), 'test: hello world\n') - self.assertEqual(Logger.stderr.getvalue(), '') diff --git a/ubuntutools/test/test_update_maintainer.py b/ubuntutools/test/test_update_maintainer.py index bd5d567..b4ff824 100644 --- a/ubuntutools/test/test_update_maintainer.py +++ b/ubuntutools/test/test_update_maintainer.py @@ -18,10 +18,9 @@ import mock import os -import sys +# import sys from io import StringIO -from ubuntutools.logger import Logger from ubuntutools.test import unittest from ubuntutools.update_maintainer import update_maintainer @@ -236,18 +235,18 @@ class UpdateMaintainerTestCase(unittest.TestCase): self.addCleanup(patcher.stop) patcher.start() self._files["rules"] = StringIO(_SIMPLE_RULES) - Logger.stdout = StringIO() - Logger.stderr = StringIO() + # Logger.stdout = StringIO() + # Logger.stderr = StringIO() def tearDown(self): - self.assertEqual(Logger.stdout.getvalue(), '') - self.assertEqual(Logger.stderr.getvalue(), '') + # self.assertEqual(Logger.stdout.getvalue(), '') + # self.assertEqual(Logger.stderr.getvalue(), '') self._files["changelog"] = None self._files["control"] = None self._files["control.in"] = None self._files["rules"] = None - Logger.stdout = sys.stdout - Logger.stderr = sys.stderr + # Logger.stdout = sys.stdout + # Logger.stderr = sys.stderr # pylint: enable=C0103 def test_debian_package(self): @@ -266,11 +265,11 @@ class UpdateMaintainerTestCase(unittest.TestCase): self._files["control"] = StringIO(_AXIS2C_CONTROL) update_maintainer(self._directory) self.assertEqual(self._files["control"].getvalue(), _AXIS2C_UPDATED) - warnings = Logger.stderr.getvalue().strip() - Logger.stderr = StringIO() - self.assertEqual(len(warnings.splitlines()), 1) - self.assertRegex(warnings, "Warning: Overwriting original maintainer: " - "Soren Hansen ") + # warnings = Logger.stderr.getvalue().strip() + # Logger.stderr = StringIO() + # self.assertEqual(len(warnings.splitlines()), 1) + # self.assertRegex(warnings, "Warning: Overwriting original maintainer: " + # "Soren Hansen ") def test_update_maintainer(self): """Test: Update Maintainer field.""" diff --git a/ubuntutools/update_maintainer.py b/ubuntutools/update_maintainer.py index 2c5de57..89a5e3a 100644 --- a/ubuntutools/update_maintainer.py +++ b/ubuntutools/update_maintainer.py @@ -20,7 +20,9 @@ import os import re import debian.changelog -from ubuntutools.logger import Logger + +import logging +Logger = logging.getLogger(__name__) # Prior May 2009 these Maintainers were used: _PREVIOUS_UBUNTU_MAINTAINER = ( @@ -176,8 +178,8 @@ def update_maintainer(debian_directory, verbose=False): return if control.get_original_maintainer() is not None: - Logger.warn("Overwriting original maintainer: %s", - control.get_original_maintainer()) + Logger.warning("Overwriting original maintainer: %s", + control.get_original_maintainer()) if verbose: print("The original maintainer is: %s" % original_maintainer) From 87a09640ba83af9438aa52917dc57c901546bf8d Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Mon, 8 Jul 2019 18:27:14 -0400 Subject: [PATCH 27/30] ubuntutools/pullpkg.py: convert -v to count, enable package-wide debug for -vv --- ubuntutools/pullpkg.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index 2e7a4ac..c88f58e 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -127,8 +127,8 @@ class PullPkg(object): "'Pending' and 'Published' status will be searched.") parser = ArgumentParser(epilog=epilog) - parser.add_argument('-v', '--verbose', action='store_true', - help="Print debug messages") + parser.add_argument('-v', '--verbose', action='count', default=0, + help="Increase verbosity/debug") parser.add_argument('-d', '--download-only', action='store_true', help="Do not extract the source package") parser.add_argument('-m', '--mirror', action='append', @@ -345,6 +345,8 @@ class PullPkg(object): assert 'verbose' in options if options['verbose']: Logger.setLevel(logging.DEBUG) + if options['verbose'] > 1: + logging.getLogger(__package__).setLevel(logging.DEBUG) Logger.debug("pullpkg options: %s", options) From e37e9db7eb8f0837eb60644da7dc07f84f6fc663 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 13 Feb 2019 21:31:52 -0500 Subject: [PATCH 28/30] pullpkg: add --security param to query ubuntu sec team proposed ppa --- ubuntutools/archive.py | 3 +- ubuntutools/pullpkg.py | 121 +++++++++++++++++++++++------------------ 2 files changed, 69 insertions(+), 55 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 43f4034..35b14bc 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -146,13 +146,14 @@ class SourcePackage(object): pocket = kwargs.get('pocket') status = kwargs.get('status') verify_signature = kwargs.get('verify_signature', False) + try_binary = kwargs.get('try_binary', True) assert (package is not None or dscfile is not None) self.source = package self._lp = lp self.binary = None - self.try_binary = True + self.try_binary = try_binary self.workdir = workdir self._series = series self._pocket = pocket diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index c88f58e..f6982be 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -105,13 +105,9 @@ class PullPkg(object): self._default_pull = kwargs.get('pull') self._default_distro = kwargs.get('distro') self._default_arch = kwargs.get('arch', host_architecture()) - self._parser = None - self._ppa_parser = None - @property - def argparser(self): - if self._parser: - return self._parser + def parse_args(self, args): + args = args[:] help_default_pull = "What to pull: " + ", ".join(VALID_PULLS) if self._default_pull: @@ -122,11 +118,9 @@ class PullPkg(object): help_default_arch = ("Get binary packages for arch") help_default_arch += ("(default: %s)" % self._default_arch) - epilog = ("Note on --status: if a version is provided, all status types " - "will be searched; if no version is provided, by default only " - "'Pending' and 'Published' status will be searched.") - - parser = ArgumentParser(epilog=epilog) + # use add_help=False because we do parse_known_args() below, and if + # that sees --help then it exits immediately + parser = ArgumentParser(add_help=False) parser.add_argument('-v', '--verbose', action='count', default=0, help="Increase verbosity/debug") parser.add_argument('-d', '--download-only', action='store_true', @@ -145,40 +139,42 @@ class PullPkg(object): help=help_default_pull) parser.add_argument('-D', '--distro', default=self._default_distro, help=help_default_distro) - parser.add_argument('--ppa', help='PPA to pull from') + + # add distro-specific params + try: + distro = self.parse_distro(parser.parse_known_args(args)[0].distro) + except InvalidDistroValueError: + # don't fail at this point, finish setting up parser help/usage + distro = None + + if distro == DISTRO_UBUNTU: + parser.add_argument('--security', action='store_true', + help='Check the Ubuntu Security Team PPA') + if distro == DISTRO_PPA: + parser.add_argument('--ppa', help='PPA to pull from') + if parser.parse_known_args(args)[0].ppa is None: + # check for any param starting with "ppa:" + # if found, move it to a --ppa param + for param in args: + if param.startswith('ppa:'): + args.remove(param) + args.insert(0, param) + args.insert(0, '--ppa') + break + + # add the positional params parser.add_argument('package', help="Package name to pull") parser.add_argument('release', nargs='?', help="Release to pull from") parser.add_argument('version', nargs='?', help="Package version to pull") - self._parser = parser - return self._parser - def parse_ppa_args(self, args): - """When pulling from PPA, convert from bare ppa:USER/NAME to --ppa option""" - if not args: - myargs = sys.argv[1:] + epilog = ("Note on --status: if a version is provided, all status types " + "will be searched; if no version is provided, by default only " + "'Pending' and 'Published' status will be searched.") - options = vars(self.argparser.parse_known_args(myargs)[0]) - # we use these, which all should be always provided by the parser, - # even if their value is None - assert 'distro' in options - assert 'ppa' in options - assert 'release' in options - assert 'version' in options + # since parser has no --help handler, create a new parser that does + newparser = ArgumentParser(parents=[parser], epilog=epilog) - # if we're not pulling from a PPA, or if we are but --ppa was given, - # then no change to the args is needed - if options['distro'] != DISTRO_PPA or options['ppa'] is not None: - return args - - # check if release or version is a ppa: - # if it is, move it to a --ppa param - for param in ['release', 'version']: - if str(options[param]).startswith('ppa:'): - myargs.remove(options[param]) - myargs.insert(0, options[param]) - myargs.insert(0, '--ppa') - - return myargs + return self.parse_options(vars(newparser.parse_args(args))) def parse_pull(self, pull): if not pull: @@ -276,6 +272,7 @@ class PullPkg(object): # they should all be provided, though the optional ones may be None # type bool + assert 'verbose' in options assert 'download_only' in options assert 'no_conf' in options assert 'no_verify_signature' in options @@ -286,14 +283,23 @@ class PullPkg(object): assert 'arch' in options assert 'package' in options # type string, optional - assert 'ppa' in options assert 'release' in options assert 'version' in options # type list of strings, optional assert 'mirror' in options - pull = self.parse_pull(options['pull']) - distro = self.parse_distro(options['distro']) + options['pull'] = self.parse_pull(options['pull']) + options['distro'] = self.parse_distro(options['distro']) + + # ensure these are always included so we can just check for None/False later + options['ppa'] = options.get('ppa', None) + options['security'] = options.get('security', False) + + return options + + def _get_params(self, options): + distro = options['distro'] + pull = options['pull'] params = {} params['package'] = options['package'] @@ -309,6 +315,13 @@ class PullPkg(object): params['dscfile'] = params['package'] params.pop('package') + if options['security']: + if options['ppa']: + Logger.warning('Both --security and --ppa specified, ignoring --ppa') + Logger.debug('Checking Ubuntu Security PPA') + # --security is just a shortcut for --ppa ppa:ubuntu-security-proposed/ppa + options['ppa'] = 'ubuntu-security-proposed/ppa' + if options['ppa']: if options['ppa'].startswith('ppa:'): params['ppa'] = options['ppa'][4:] @@ -333,16 +346,12 @@ class PullPkg(object): params['status'] = STATUSES if 'all' in options['status'] else options['status'] - return (pull, distro, params) + return params - def pull(self, args=None): + def pull(self, args=sys.argv[1:]): """Pull (download) specified package file(s)""" - # pull-ppa-* may need conversion from ppa:USER/NAME to --ppa USER/NAME - args = self.parse_ppa_args(args) + options = self.parse_args(args) - options = vars(self.argparser.parse_args(args)) - - assert 'verbose' in options if options['verbose']: Logger.setLevel(logging.DEBUG) if options['verbose'] > 1: @@ -353,7 +362,11 @@ class PullPkg(object): # Login anonymously to LP Launchpad.login_anonymously() - (pull, distro, params) = self.parse_options(options) + pull = options['pull'] + distro = options['distro'] + + params = self._get_params(options) + package = params['package'] # call implementation, and allow exceptions to flow up to caller srcpkg = DISTRO_PKG_CLASS[distro](**params) @@ -377,11 +390,11 @@ class PullPkg(object): srcpkg.unpack() else: name = '.*' - if params['package'] != spph.getPackageName(): - Logger.info("Pulling only binary package '%s'", params['package']) + if package != spph.getPackageName(): + Logger.info("Pulling only binary package '%s'", package) Logger.info("Use package name '%s' to pull all binary packages", spph.getPackageName()) - name = params['package'] + name = package if pull == PULL_DEBS: name = r'{}(? Date: Fri, 8 Feb 2019 17:22:31 -0500 Subject: [PATCH 29/30] test: remove test_archive obsolete tests some tests mocked into the implementation of archive.py, which has now changed dramatically, and the tests are no longer valid. --- ubuntutools/test/test_archive.py | 103 +------------------------------ 1 file changed, 1 insertion(+), 102 deletions(-) diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index 46f6c11..804ee1f 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -20,10 +20,9 @@ import os.path import shutil import tempfile from io import BytesIO -from urllib.error import HTTPError, URLError +from urllib.error import HTTPError from urllib.request import OpenerDirector, urlopen -import debian.deb822 import httplib2 import ubuntutools.archive @@ -192,35 +191,6 @@ class LocalSourcePackageTestCase(unittest.TestCase): verify_signature=False) pkg.pull() - def test_pull(self): - pkg = self.SourcePackage(package='example', - version='1.0-1', - component='main', - workdir=self.workdir, - verify_signature=False) - - pkg.url_opener = self.url_opener - pkg.pull() - - def test_mirrors(self): - mirror = 'http://mirror' - sequence = [self.urlopen_null, self.urlopen_404, self.urlopen_proxy, - self.urlopen_proxy] - - def _callable_iter(*args, **kwargs): - return sequence.pop(0)(*args, **kwargs) - url_opener = mock.MagicMock(spec=OpenerDirector) - url_opener.open.side_effect = _callable_iter - - pkg = self.SourcePackage(package='example', - version='1.0-1', - component='main', - workdir=self.workdir, - mirrors=[mirror], - verify_signature=False) - pkg.url_opener = url_opener - pkg.pull() - def test_dsc_missing(self): self.mock_http.side_effect = self.request_404 pkg = self.SourcePackage(package='example', @@ -228,74 +198,3 @@ class LocalSourcePackageTestCase(unittest.TestCase): component='main', workdir=self.workdir) self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) - - -class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): - SourcePackage = ubuntutools.archive.DebianSourcePackage - - def test_mirrors(self): - debian_mirror = 'http://mirror/debian' - debsec_mirror = 'http://mirror/debsec' - - sequence = [self.urlopen_null, - self.urlopen_404, - self.urlopen_404, - lambda x: BytesIO( - b'{"fileinfo": {"hashabc": [{"name": "example_1.0.orig.tar.gz"}]}}'), - self.urlopen_file('example_1.0.orig.tar.gz'), - self.urlopen_proxy] - - def _callable_iter(*args, **kwargs): - return sequence.pop(0)(*args, **kwargs) - url_opener = mock.MagicMock(spec=OpenerDirector) - url_opener.open.side_effect = _callable_iter - - pkg = self.SourcePackage(package='example', - version='1.0-1', - component='main', - workdir=self.workdir, - mirrors=[debian_mirror, debsec_mirror], - verify_signature=False) - pkg.url_opener = url_opener - pkg.pull() - pkg.unpack() - - def test_dsc_missing(self): - mirror = 'http://mirror' - self.mock_http.side_effect = self.request_404_then_proxy - - patcher = mock.patch.object(debian.deb822.GpgInfo, 'from_sequence') - self.addCleanup(patcher.stop) - mock_gpg_info = patcher.start() - mock_gpg_info.return_value = debian.deb822.GpgInfo.from_output( - '[GNUPG:] GOODSIG DEADBEEF Joe Developer ' - '') - - pkg = self.SourcePackage(package='example', - version='1.0-1', - component='main', - workdir=self.workdir, - mirrors=[mirror], - verify_signature=False) - pkg.url_opener = self.url_opener - pkg.pull() - - def test_dsc_badsig(self): - mirror = 'http://mirror' - self.mock_http.side_effect = self.request_404_then_proxy - - patcher = mock.patch.object(debian.deb822.GpgInfo, 'from_sequence') - self.addCleanup(patcher.stop) - mock_gpg_info = patcher.start() - mock_gpg_info.return_value = debian.deb822.GpgInfo.from_output( - '[GNUPG:] ERRSIG DEADBEEF') - - pkg = self.SourcePackage(package='example', - version='1.0-1', - component='main', - workdir=self.workdir, - mirrors=[mirror]) - try: - self.assertRaises(ubuntutools.archive.DownloadError, pkg.pull) - except URLError: - raise unittest.SkipTest('Test needs addr resolution to work') From 82c8c438f7f09d756df95cf45fe377e4e902e0c4 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Mon, 25 Nov 2019 16:40:18 -0500 Subject: [PATCH 30/30] ubuntutools: use file extension when possible to determine deb/ddeb/udeb --- ubuntutools/archive.py | 64 +++++++++++++++++++++++++----------- ubuntutools/lp/__init__.py | 2 +- ubuntutools/lp/lpapicache.py | 55 +++++++++++++++++++++++++------ ubuntutools/pullpkg.py | 31 ++++++++++------- 4 files changed, 111 insertions(+), 41 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 35b14bc..ac8a2d2 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -314,7 +314,7 @@ class SourcePackage(object): yield self._mirror_url(mirror, name) yield self._lp_url(name) - def _binary_urls(self, name, default_url): + def _binary_urls(self, name, default_urls): "Generator of URLs for name" for mirror in self.mirrors: yield self._mirror_url(mirror, name) @@ -322,11 +322,13 @@ class SourcePackage(object): if mirror not in self.mirrors: yield self._mirror_url(mirror, name) yield self._lp_url(name) - yield default_url + for url in default_urls: + yield url - def _binary_files_info(self, arch, name): - for bpph in self.lp_spph.getBinaries(arch=arch, name=name): - yield (bpph.getFileName(), bpph.getUrl(), 0) + def _binary_files_info(self, arch, name, ext): + for bpph in self.lp_spph.getBinaries(arch=arch, name=name, ext=ext): + urls = bpph.binaryFileUrls() + [bpph.getUrl()] + yield (bpph.getFileName(), urls, 0) def pull_dsc(self): "Retrieve dscfile and parse" @@ -508,20 +510,26 @@ class SourcePackage(object): else: raise DownloadError('File %s could not be found' % name) - def pull_binaries(self, arch, name=None): + def pull_binaries(self, arch, name=None, ext=None): """Pull binary debs into workdir. If name is specified, only binary packages matching the regex are included. + + If ext is specified, only binary packages with that ext are included; for + example to only download dbgsym ddebs, specify ext='ddeb'. + Must specify arch, or use 'all' to pull all archs. Returns the number of files downloaded. """ total = 0 + Logger.debug("pull_binaries(arch=%s, name=%s, ext=%s)" % (arch, name, ext)) + if not arch: raise RuntimeError("Must specify arch") - for (fname, furl, fsize) in self._binary_files_info(arch, name): + for (fname, furls, fsize) in self._binary_files_info(arch, name, ext): found = False - for url in self._binary_urls(fname, furl): + for url in self._binary_urls(fname, furls): try: if self._download_file(url, fname, False, fsize): found = True @@ -591,11 +599,11 @@ class DebianSPPH(SourcePackagePublishingHistory): """ resource_type = 'source_package_publishing_history' - def getBinaries(self, arch, name=None): + def getBinaries(self, arch, name=None, ext=None): Logger.info('Using Snapshot to find binary packages') srcpkg = Snapshot.getSourcePackage(self.getPackageName(), version=self.getVersion()) - return srcpkg.getSPPH().getBinaries(arch=arch, name=name) + return srcpkg.getSPPH().getBinaries(arch=arch, name=name, ext=ext) class DebianSourcePackage(SourcePackage): @@ -655,9 +663,9 @@ class DebianSourcePackage(SourcePackage): break yield self.snapshot_files[name] - def _binary_files_info(self, arch, name): - for f in self.snapshot_package.getBinaryFiles(arch=arch, name=name): - yield (f.name, f.getUrl(), f.size) + def _binary_files_info(self, arch, name, ext): + for f in self.snapshot_package.getBinaryFiles(arch=arch, name=name, ext=ext): + yield (f.name, [f.getUrl()], f.size) def pull_dsc(self): "Retrieve dscfile and parse" @@ -973,7 +981,7 @@ class SnapshotSourcePackage(SnapshotPackage): def getAllFiles(self): return self.getFiles() + self.getBinaryFiles() - def getBinaryFiles(self, arch=None, name=None): + def getBinaryFiles(self, arch=None, name=None, ext=None): if not self._binary_files: url = "/mr/package/{}/{}/allfiles".format(self.name, self.version) response = Snapshot.load("{}?fileinfo=1".format(url)) @@ -985,9 +993,11 @@ class SnapshotSourcePackage(SnapshotPackage): self._binary_files = files bins = list(self._binary_files) if arch: - bins = filter(lambda b: b.isArch(arch), bins) + bins = [b for b in bins if b.isArch(arch)] if name: - bins = filter(lambda b: re.match(name, b.name), bins) + bins = [b for b in bins if re.match(name, b.package_name)] + if ext: + bins = [b for b in bins if re.match(ext, b.ext)] return bins def getFiles(self): @@ -1038,7 +1048,7 @@ class SnapshotBinaryPackage(SnapshotPackage): for r in response['result']] if not arch: return list(self._files) - return filter(lambda f: f.isArch(arch), self._files) + return [f for f in self._files if f.isArch(arch)] class SnapshotFile(object): @@ -1061,6 +1071,10 @@ class SnapshotFile(object): def name(self): return self._obj['name'] + @property + def ext(self): + return self.name.rpartition('.')[2] + @property def path(self): return self._obj['path'] @@ -1199,9 +1213,9 @@ class SnapshotSPPH(object): new_entries.append(str(block)) return ''.join(new_entries) - def getBinaries(self, arch, name=None): + def getBinaries(self, arch, name=None, ext=None): return [b.getBPPH() - for b in self._pkg.getBinaryFiles(arch=arch, name=name)] + for b in self._pkg.getBinaryFiles(arch=arch, name=name, ext=ext)] class SnapshotBPPH(object): @@ -1255,11 +1269,23 @@ class SnapshotBPPH(object): def getComponent(self): return self._file.component + def binaryFileUrls(self): + return [self.getUrl()] + def getBuild(self): return None def getUrl(self): return self._file.getUrl() + def getFileVersion(self): + return self.getVersion() + + def getFileArch(self): + return self.arch + + def getFileExt(self): + return self._file.ext + def getFileName(self): return self._file.name diff --git a/ubuntutools/lp/__init__.py b/ubuntutools/lp/__init__.py index 02c9fcd..17e3542 100644 --- a/ubuntutools/lp/__init__.py +++ b/ubuntutools/lp/__init__.py @@ -3,4 +3,4 @@ # service = 'production' -api_version = '1.0' +api_version = 'devel' diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index de465d2..dd35bec 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -795,11 +795,14 @@ class SourcePackagePublishingHistory(BaseWrapper): new_entries.append(str(block)) return ''.join(new_entries) - def getBinaries(self, arch, name=None): + def getBinaries(self, arch, name=None, ext=None): ''' Returns the resulting BinaryPackagePublishingHistorys. Must specify arch, or use 'all' to get all archs. + If name is specified, only returns BPPH matching that (regex) name. + + If ext is specified, only returns BPPH matching that (regex) ext. ''' if not arch: raise RuntimeError("Must specify arch") @@ -834,7 +837,7 @@ class SourcePackagePublishingHistory(BaseWrapper): # strip out the URL leading text. filename = url.rsplit('/', 1)[1] # strip the file suffix - pkgname = filename.rsplit('.', 1)[0] + (pkgname, _, e) = filename.rpartition('.') # split into name, version, arch (n, v, a) = pkgname.rsplit('_', 2) if a == 'all': @@ -845,6 +848,9 @@ class SourcePackagePublishingHistory(BaseWrapper): # Only check the name requested - saves time if name and not re.match(name, n): continue + # Only check the ext requested - saves time + if ext and not re.match(ext, e): + continue # If we already have this BPPH, keep going if a in self._binaries and n in self._binaries[a]: continue @@ -862,7 +868,7 @@ class SourcePackagePublishingHistory(BaseWrapper): if a not in self._binaries: self._binaries[a] = {} self._binaries[a][n] = bpph - if not name and arch == 'all': + if not name and not ext and arch == 'all': # We must have got them all self._have_all_binaries = True @@ -874,7 +880,10 @@ class SourcePackagePublishingHistory(BaseWrapper): bpphs = self._binaries[arch].copy().values() if name: - bpphs = filter(lambda b: re.match(name, b.binary_package_name), bpphs) + bpphs = [b for b in bpphs if re.match(name, b.binary_package_name)] + + if ext: + bpphs = [b for b in bpphs if re.match(ext, b.getFileExt())] return bpphs @@ -938,6 +947,7 @@ class BinaryPackagePublishingHistory(BaseWrapper): def __init__(self, *args): self._arch = None + self._ext = None @property def arch(self): @@ -976,7 +986,11 @@ class BinaryPackagePublishingHistory(BaseWrapper): Only available in the devel API, not 1.0 ''' try: - return self._lpobject.binaryFileUrls() + urls = self._lpobject.binaryFileUrls() + if not urls: + Logger.warning('BPPH %s_%s has no binaryFileUrls' % + (self.getPackageName(), self.getVersion())) + return urls except AttributeError: raise AttributeError("binaryFileUrls can only be found in lpapi " "devel, not 1.0. Login using devel to have it.") @@ -1013,12 +1027,33 @@ class BinaryPackagePublishingHistory(BaseWrapper): ''' Returns the file extension; "deb", "ddeb", or "udeb". ''' - if self.getPackageName().endswith("-dbgsym"): - return "ddeb" - elif self.getPackageName().endswith("-di"): + if not self._ext: + self._ext = self._getFileExt() + + return self._ext + + def _getFileExt(self): + try: + # this is the best way, from the actual URL filename + return self.binaryFileUrls()[0].rpartition('.')[2] + except (AttributeError, IndexError): + Logger.debug('Could not get file ext from url, trying to guess...') + + # is_debug should be reliable way of detecting ddeb...? + try: + if self.is_debug: + return "ddeb" + except AttributeError: + # is_debug only available with api version 'devel' + if self.getPackageName().endswith("-dbgsym"): + return "ddeb" + + # is this reliable? + if self.getPackageName().endswith("-di") or self.getPackageName().endswith("-udeb"): return "udeb" - else: - return "deb" + + # everything else - assume regular deb + return "deb" def getFileName(self): ''' diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index f6982be..32bc013 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -53,6 +53,7 @@ PULL_UDEBS = 'udebs' PULL_LIST = 'list' VALID_PULLS = [PULL_SOURCE, PULL_DEBS, PULL_DDEBS, PULL_UDEBS, PULL_LIST] +VALID_BINARY_PULLS = [PULL_DEBS, PULL_DDEBS, PULL_UDEBS] DISTRO_DEBIAN = 'debian' DISTRO_UBUNTU = 'ubuntu' @@ -388,24 +389,32 @@ class PullPkg(object): Logger.debug("--download-only specified, not extracting") else: srcpkg.unpack() - else: - name = '.*' + elif pull in VALID_BINARY_PULLS: + name = None if package != spph.getPackageName(): Logger.info("Pulling only binary package '%s'", package) Logger.info("Use package name '%s' to pull all binary packages", spph.getPackageName()) name = package - if pull == PULL_DEBS: - name = r'{}(? 'deb' + ext = pull.rstrip('s') + + if distro == DISTRO_DEBIAN: + # Debian ddebs don't use .ddeb extension, unfortunately :( + if pull in [PULL_DEBS, PULL_DDEBS]: + name = name or '.*' + ext = 'deb' + if pull == PULL_DEBS: + name += r'(?