Lubuntu/ubuntu-dev-tools
3
0
Fork 0
mirror of https://git.launchpad.net/ubuntu-dev-tools synced 2025-07-23 12:51:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

ubuntutools: update archive/lpapicache to optionally search all series

Browse Source 
For PPA and UCA repos, the latest build is not necessarily in the
'development' release; so for those SourcePackage classes, search
all the 'active' series (starting with latest devel) for any
matches to the provided package name.  This allows not having to
specify the series name when looking in PPA/UCA repos for the 'latest'
version of a specific package.
This commit is contained in:
Dan Streetman 2018-07-24 16:28:25 -04:00
parent b11b83f0e2
commit 0f61836b10
2 changed files with 242 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
ubuntutools/archive.py
View File

@ -46,8 +46,7 @@ from contextlib import closing
from ubuntutools.config import UDTConfig from ubuntutools.config import UDTConfig
from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam,
SourcePackagePublishingHistory, SourcePackagePublishingHistory)
BinaryPackagePublishingHistory)
from ubuntutools.lp.udtexceptions import (PackageNotFoundException, from ubuntutools.lp.udtexceptions import (PackageNotFoundException,
SeriesNotFoundException, SeriesNotFoundException,
InvalidDistroValueError) InvalidDistroValueError)
@ -154,7 +153,6 @@ class SourcePackage(object):
self.workdir = workdir self.workdir = workdir
self.quiet = quiet self.quiet = quiet
self._series = series self._series = series
self._use_series = True
self._pocket = pocket self._pocket = pocket
self._dsc_source = dscfile self._dsc_source = dscfile
self._verify_signature = verify_signature self._verify_signature = verify_signature
@ -188,65 +186,61 @@ class SourcePackage(object):
else: else:
Launchpad.login_anonymously() Launchpad.login_anonymously()
distro = self.getDistribution()
archive = self.getArchive() archive = self.getArchive()
series = None params = {}
params = {'exact_match': True, 'order_by_date': True}
if self._version: if self._version:
# if version was specified, use that # if version was specified, use that
params['version'] = self._version.full_version params['version'] = self._version.full_version
elif self._use_series: elif self._series:
if self._series: # if version not specified, get the latest from this series
# if version not specified, get the latest from this series params['series'] = self._series
series = distro.getSeries(self._series) # note that if not specified, pocket defaults to all EXCEPT -backports
else:
# if no version or series, get the latest from devel series
series = distro.getDevelopmentSeries()
params['distro_series'] = series()
if self._pocket: if self._pocket:
params['pocket'] = self._pocket params['pocket'] = self._pocket
spphs = archive.getPublishedSources(source_name=self.source, **params) else:
if spphs: # We always want to search all series, if not specified
self._spph = self.spph_class(spphs[0]) params['search_all_series'] = True
try:
self._spph = archive.getSourcePackage(self.source,
wrapper=self.spph_class,
**params)
return self._spph return self._spph
except PackageNotFoundException as pnfe:
if not self.try_binary or self.binary:
# either we don't need to bother trying binary name lookup,
# or we've already tried
raise pnfe
if self.try_binary and not self.binary: Logger.normal('Source package lookup failed, '
if series: 'trying lookup of binary package %s' % self.source)
arch_series = series.getArchSeries()
params['distro_arch_series'] = arch_series()
del params['distro_series']
bpphs = archive.getPublishedBinaries(binary_name=self.source, **params)
if bpphs:
bpph = BinaryPackagePublishingHistory(bpphs[0])
self.binary = self.source
self.source = bpph.getSourcePackageName()
Logger.normal("Using source package '{}' for binary package '{}'"
.format(self.source, self.binary))
try:
spph = bpph.getBuild().getSourcePackagePublishingHistory()
except Exception:
spph = None
if spph:
self._spph = spph
return self._spph
else:
# binary build didn't include source link, unfortunately
# so try again with the updated self.source name
if not self._version:
# Get version first if user didn't specify it, as some
# binaries have their version hardcoded in their name,
# such as the kernel package
self._version = Version(bpph.getVersion())
return self.lp_spph
msg = "No {} package found".format(self.source) try:
if self._version: bpph = archive.getBinaryPackage(self.source, **params)
msg += " for version {}".format(self._version.full_version) except PackageNotFoundException as bpnfe:
elif series: # log binary lookup failure, in case it provides hints
msg += " in series {}".format(series.name) Logger.normal(str(bpnfe))
if self._pocket: # raise the original exception for the source lookup
msg += " pocket {}".format(self._pocket) raise pnfe
raise PackageNotFoundException(msg)
self.binary = self.source
self.source = bpph.getSourcePackageName()
Logger.normal("Using source package '{}' for binary package '{}'"
.format(self.source, self.binary))
spph = bpph.getBuild().getSourcePackagePublishingHistory()
if spph:
self._spph = self.spph_class(spph.self_link)
return self._spph
# binary build didn't include source link, unfortunately
# so try again with the updated self.source name
if not self._version:
# Get version first if user didn't specify it, as some
# binaries have their version hardcoded in their name,
# such as the kernel package
self._version = Version(bpph.getVersion())
return self.lp_spph
@property @property
def version(self): def version(self):
@ -761,7 +755,6 @@ class UbuntuCloudArchiveSourcePackage(PersonalPackageArchiveSourcePackage):
kwargs['ppa'] = ('%s/%s-staging' % kwargs['ppa'] = ('%s/%s-staging' %
(UbuntuCloudArchiveSourcePackage._ppateam, series)) (UbuntuCloudArchiveSourcePackage._ppateam, series))
super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs) super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs)
self._use_series = False # each UCA series is for a single Ubuntu series
self.uca_release = series self.uca_release = series
self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"] self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"]
@ -948,7 +941,7 @@ class SnapshotSourcePackage(SnapshotPackage):
r['architecture'], self.name) r['architecture'], self.name)
for b in response['result']['binaries'] for r in b['files']] for b in response['result']['binaries'] for r in b['files']]
self._binary_files = files self._binary_files = files
bins = self._binary_files.copy() bins = list(self._binary_files)
if arch: if arch:
bins = filter(lambda b: b.isArch(arch), bins) bins = filter(lambda b: b.isArch(arch), bins)
if name: if name:
@ -1002,7 +995,7 @@ class SnapshotBinaryPackage(SnapshotPackage):
r['architecture'], self.source) r['architecture'], self.source)
for r in response['result']] for r in response['result']]
if not arch: if not arch:
return self._files.copy() return list(self._files)
return filter(lambda f: f.isArch(arch), self._files) return filter(lambda f: f.isArch(arch), self._files)

291
ubuntutools/lp/lpapicache.py
View File

@ -37,7 +37,8 @@ from lazr.restfulclient.resource import Entry
from ubuntutools.version import Version from ubuntutools.version import Version
from ubuntutools.lp import (service, api_version) from ubuntutools.lp import (service, api_version)
from ubuntutools.misc import host_architecture from ubuntutools.misc import (host_architecture,
DEFAULT_POCKETS, POCKETS)
from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError, from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError,
ArchiveNotFoundException, ArchiveNotFoundException,
ArchSeriesNotFoundException, ArchSeriesNotFoundException,
@ -59,8 +60,6 @@ __all__ = [
'SourcePackagePublishingHistory', 'SourcePackagePublishingHistory',
] ]
_POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports')
class _Launchpad(object): class _Launchpad(object):
'''Singleton for LP API access.''' '''Singleton for LP API access.'''
@ -191,15 +190,23 @@ class Distribution(BaseWrapper):
resource_type = 'distribution' resource_type = 'distribution'
def __init__(self, *args): def __init__(self, *args):
# Don't share _series and _archives between different Distributions self._archives = dict()
if '_series' not in self.__dict__: self._series_by_name = dict()
self._series = dict() self._series = dict()
if '_archives' not in self.__dict__: self._dev_series = None
self._archives = dict() self._have_all_series = False
def cache(self): def cache(self):
self._cache[self.name] = self self._cache[self.name] = self
def _cache_series(self, series):
'''
Add the DistroSeries to the cache if needed.
'''
if series.version not in self._series:
self._series_by_name[series.name] = series
self._series[series.version] = series
@classmethod @classmethod
def fetch(cls, dist): def fetch(cls, dist):
''' '''
@ -246,28 +253,49 @@ class Distribution(BaseWrapper):
(e.g. 'karmic') or version (e.g. '9.10'). (e.g. 'karmic') or version (e.g. '9.10').
If the series is not found: raise SeriesNotFoundException If the series is not found: raise SeriesNotFoundException
''' '''
if name_or_version not in self._series: if name_or_version in self._series:
try: return self._series[name_or_version]
series = DistroSeries(self().getSeries(name_or_version=name_or_version)) if name_or_version in self._series_by_name:
# Cache with name and version return self._series_by_name[name_or_version]
self._series[series.name] = series
self._series[series.version] = series try:
except HTTPError: series = DistroSeries(self().getSeries(name_or_version=name_or_version))
message = "Release '%s' is unknown in '%s'." % \ except HTTPError:
(name_or_version, self.display_name) message = "Release '%s' is unknown in '%s'." % \
raise SeriesNotFoundException(message) (name_or_version, self.display_name)
return self._series[name_or_version] raise SeriesNotFoundException(message)
self._cache_series(series)
return series
def getDevelopmentSeries(self): def getDevelopmentSeries(self):
''' '''
Returns a DistroSeries object of the current development series. Returns a DistroSeries object of the current development series.
''' '''
dev = DistroSeries(self.current_series_link) if not self._dev_series:
# Cache it in _series if not already done series = DistroSeries(self.current_series_link)
if dev.name not in self._series: self._cache_series(series)
self._series[dev.name] = dev self._dev_series = series
self._series[dev.version] = dev return self._dev_series
return dev
def getAllSeries(self, active=True):
'''
Returns a list of all DistroSeries objects.
'''
if not self._have_all_series:
for s in Launchpad.load(self.series_collection_link).entries:
series = DistroSeries(s['self_link'])
self._cache_series(series)
self._have_all_series = True
allseries = filter(lambda s: s.active, self._series.values())
allseries = sorted(allseries,
key=lambda s: float(s.version),
reverse=True)
Logger.debug("Found series: %s" % ", ".join(map(lambda s: "%s (%s)" %
(s.name, s.version),
allseries)))
return collections.OrderedDict((s.name, s) for s in allseries)
class DistroArchSeries(BaseWrapper): class DistroArchSeries(BaseWrapper):
@ -329,7 +357,7 @@ class Archive(BaseWrapper):
self._component_uploaders = {} self._component_uploaders = {}
def getSourcePackage(self, name, series=None, pocket=None, version=None, def getSourcePackage(self, name, series=None, pocket=None, version=None,
status=None): status=None, wrapper=None, search_all_series=False):
''' '''
Returns a SourcePackagePublishingHistory object for the most Returns a SourcePackagePublishingHistory object for the most
recent source package in the distribution 'dist', series and recent source package in the distribution 'dist', series and
@ -338,11 +366,20 @@ class Archive(BaseWrapper):
series defaults to the current development series if not specified. series defaults to the current development series if not specified.
series must be either a series name string, or DistroSeries object. series must be either a series name string, or DistroSeries object.
pocket may be a string or a list. If a list, the highest version
will be returned. It defaults to all pockets except backports.
version may be specified to get only the exact version requested. version may be specified to get only the exact version requested.
pocket may be a string or a list. If no version is provided, it
defaults to all pockets except 'Backports'; if searching for a
specific version, it defaults to all pockets. Pocket strings must
be capitalized.
wrapper is the class to return an instance of; defaults to
SourcePackagePublishingHistory.
search_all_series is used if series is None. If False, this will
search only the latest devel series, and if True all series
will be searched, in reverse order, starting with the latest
devel series. Defaults to False.
status is optional, to restrict search to a given status only. status is optional, to restrict search to a given status only.
If the requested source package doesn't exist a If the requested source package doesn't exist a
@ -351,13 +388,15 @@ class Archive(BaseWrapper):
return self._getPublishedItem(name, series, pocket, cache=self._srcpkgs, return self._getPublishedItem(name, series, pocket, cache=self._srcpkgs,
function='getPublishedSources', function='getPublishedSources',
name_key='source_name', name_key='source_name',
wrapper=SourcePackagePublishingHistory, wrapper=wrapper or SourcePackagePublishingHistory,
version=version, version=version,
status=status, status=status,
search_all_series=search_all_series,
binary=False) binary=False)
def getBinaryPackage(self, name, archtag=None, series=None, pocket=None, def getBinaryPackage(self, name, archtag=None, series=None, pocket=None,
version=None, status=None): version=None, status=None, wrapper=None,
search_all_series=False):
''' '''
Returns a BinaryPackagePublishingHistory object for the most Returns a BinaryPackagePublishingHistory object for the most
recent source package in the distribution 'dist', architecture recent source package in the distribution 'dist', architecture
@ -367,11 +406,20 @@ class Archive(BaseWrapper):
series must be either a series name string, or DistroArchSeries object. series must be either a series name string, or DistroArchSeries object.
series may be omitted if version is specified. series may be omitted if version is specified.
pocket may be a string or a list. If a list, the highest version
will be returned. It defaults to all pockets except backports.
version may be specified to get only the exact version requested. version may be specified to get only the exact version requested.
pocket may be a string or a list. If no version is provided, it
defaults to all pockets except 'Backports'; if searching for a
specific version, it defaults to all pockets. Pocket strings must
be capitalized.
wrapper is the class to return an instance of; defaults to
BinaryPackagePublishingHistory.
search_all_series is used if series is None. If False, this will
search only the latest devel series, and if True all series
will be searched, in reverse order, starting with the latest
devel series. Defaults to False.
status is optional, to restrict search to a given status only. status is optional, to restrict search to a given status only.
If the requested binary package doesn't exist a If the requested binary package doesn't exist a
@ -381,36 +429,40 @@ class Archive(BaseWrapper):
cache=self._binpkgs, cache=self._binpkgs,
function='getPublishedBinaries', function='getPublishedBinaries',
name_key='binary_name', name_key='binary_name',
wrapper=BinaryPackagePublishingHistory, wrapper=wrapper or BinaryPackagePublishingHistory,
version=version, version=version,
status=status, status=status,
search_all_series=search_all_series,
binary=True) binary=True)
def _getPublishedItem(self, name, series, pocket, cache, def _getPublishedItem(self, name, series, pocket, cache,
function, name_key, wrapper, archtag=None, function, name_key, wrapper, archtag=None,
version=None, status=None, version=None, status=None, search_all_series=False,
binary=False): binary=False):
''' '''
Common code between getSourcePackage and getBinaryPackage. Common code between getSourcePackage and getBinaryPackage.
Don't use this directly. Don't use this directly.
''' '''
if pocket is None: if not pocket:
pockets = frozenset(('Proposed', 'Updates', 'Security', 'Release')) if version and not series:
# check ALL pockets if specific version in any series
pockets = POCKETS
else:
# otherwise, check all pockets EXCEPT 'Backports'
pockets = DEFAULT_POCKETS
elif isinstance(pocket, str): elif isinstance(pocket, str):
pockets = frozenset((pocket,)) pockets = (pocket,)
else: else:
pockets = frozenset(pocket) pockets = tuple(pocket)
for p in pockets:
if p not in POCKETS:
raise PocketDoesNotExistError("Pocket '%s' does not exist." % p)
for pocket in pockets:
if pocket not in _POCKETS:
raise PocketDoesNotExistError("Pocket '%s' does not exist." %
pocket)
dist = Distribution(self.distribution_link) dist = Distribution(self.distribution_link)
arch_series = None
# please don't pass DistroArchSeries as archtag! # please don't pass DistroArchSeries as archtag!
# but, the code was like that before so keep # but, the code was like that before so keep
# backwards compatibility. # backwards compatibility.
@ -418,83 +470,126 @@ class Archive(BaseWrapper):
series = archtag series = archtag
archtag = None archtag = None
if isinstance(series, DistroArchSeries): series_to_check = [series]
arch_series = series if not version and not series:
series = series.getSeries() # if neither version or series are specified, use either the
elif isinstance(series, DistroSeries): # devel series or search all series
pass if search_all_series:
elif series: series_to_check = dist.getAllSeries().values()
series = dist.getSeries(series) else:
elif not version: series_to_check = [dist.getDevelopmentSeries()]
series = dist.getDevelopmentSeries()
if binary: # check each series - if only version was provided, series will be None
if arch_series is None and series: for series in series_to_check:
arch_series = series.getArchSeries(archtag=archtag) arch_series = None
if archtag is None and arch_series:
archtag = arch_series.architecture_tag
if archtag is None:
archtag = host_architecture()
index = (name, getattr(series, 'name', None), archtag, pockets, status, version) if isinstance(series, DistroArchSeries):
arch_series = series
series = series.getSeries()
elif isinstance(series, DistroSeries):
pass
elif series:
series = dist.getSeries(series)
if binary:
if arch_series is None and series:
arch_series = series.getArchSeries(archtag=archtag)
if archtag is None and arch_series:
archtag = arch_series.architecture_tag
if archtag is None:
archtag = host_architecture()
index = (name, getattr(series, 'name', None), archtag, pockets, status, version)
if index in cache:
return cache[index]
if index not in cache:
params = { params = {
name_key: name, name_key: name,
'exact_match': True, 'exact_match': True,
} }
if status:
params['status'] = status
if arch_series: if arch_series:
params['distro_arch_series'] = arch_series() params['distro_arch_series'] = arch_series()
elif series: elif series:
params['distro_series'] = series() params['distro_series'] = series()
if len(pockets) == 1: if len(pockets) == 1:
params['pocket'] = list(pockets)[0] params['pocket'] = pockets[0]
if version: if version:
params['version'] = version params['version'] = version
Logger.debug('Calling %s(%s)' % (function,
', '.join(['%s=%s' % (k, v)
for (k, v) in params.items()])))
records = getattr(self, function)(**params) records = getattr(self, function)(**params)
latest = None err_msg = ("does not exist in the %s %s archive" %
(dist.display_name, self.name))
for record in records: for record in records:
if binary:
rversion = getattr(record, 'binary_package_version', None)
else:
rversion = getattr(record, 'source_package_version', None)
skipmsg = ('Skipping version %s: ' % rversion)
if record.pocket not in pockets: if record.pocket not in pockets:
err_msg = 'pocket %s not in (%s)' % (record.pocket, ','.join(pockets))
Logger.debug(skipmsg + err_msg)
continue
continue continue
r = wrapper(record) r = wrapper(record)
if binary and archtag and archtag != r.arch: if binary and archtag and archtag != r.arch:
err_msg = 'arch %s does not match requested arch %s' % (r.arch, archtag)
Logger.debug(skipmsg + err_msg)
continue continue
if latest is None or latest.getVersion() < r.getVersion(): # results are ordered so first is latest
latest = r cache[index] = r
return r
if latest is None: version_with_epoch = None
if name_key == 'binary_name': if version and version == Version(version).strip_epoch() and len(records) == 0:
package_type = "binary package" # a specific version was asked for, but we found none;
elif name_key == 'source_name': # check if one exists with an epoch to give a hint in error msg
package_type = "source package" for epoch in range(1, 9):
else: v = Version(version)
package_type = "package" v.epoch = epoch
msg = "The %s '%s' " % (package_type, name) params['version'] = v.full_version
if version: if len(getattr(self, function)(**params)) > 0:
msg += "version %s " % version version_with_epoch = v.full_version
msg += ("does not exist in the %s %s archive" % Logger.debug('Found version with epoch %s' % version_with_epoch)
(dist.display_name, self.name)) break
if binary:
msg += " for architecture %s" % archtag
if series:
pockets = [series.name if pocket == 'Release'
else '%s-%s' % (series.name, pocket.lower())
for pocket in pockets]
if len(pockets) > 1:
pockets[-2:] = [' or '.join(pockets[-2:])]
msg += " in " + ', '.join(pockets)
raise PackageNotFoundException(msg)
cache[index] = latest if name_key == 'binary_name':
return cache[index] package_type = "binary package"
elif name_key == 'source_name':
package_type = "source package"
else:
package_type = "package"
msg = "The %s '%s' " % (package_type, name)
if version:
msg += "version %s " % version
msg += err_msg
if binary and archtag:
msg += " for architecture %s" % archtag
if len(series_to_check) > 1:
msg += " in any release"
if len(pockets) == 1:
msg += " for pocket %s" % pockets[0]
elif len(pockets) != len(POCKETS):
msg += " for pockets " + ', '.join(pockets)
elif series:
msg += " in %s" % series.name
if len(pockets) == 1:
msg += "-%s" % pockets[0]
elif len(pockets) != len(POCKETS):
msg += " for pockets " + ', '.join(pockets)
if version_with_epoch:
msg += " (did you forget the epoch? try %s)" % version_with_epoch
raise PackageNotFoundException(msg)
def copyPackage(self, source_name, version, from_archive, to_pocket, def copyPackage(self, source_name, version, from_archive, to_pocket,
to_series=None, sponsored=None, include_binaries=False): to_series=None, sponsored=None, include_binaries=False):
@ -695,7 +790,7 @@ class SourcePackagePublishingHistory(BaseWrapper):
self._binaries[a][b.binary_package_name] = b self._binaries[a][b.binary_package_name] = b
self._have_all_binaries = True self._have_all_binaries = True
else: else:
# Older version, so we have to go the long way :( # we have to go the long way :(
print("Please wait, this may take some time...") print("Please wait, this may take some time...")
archive = self.getArchive() archive = self.getArchive()
urls = self.binaryFileUrls() urls = self.binaryFileUrls()
@ -976,7 +1071,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
if package is None and component is None: if package is None and component is None:
raise ValueError('Either a source package name or a component has ' raise ValueError('Either a source package name or a component has '
'to be specified.') 'to be specified.')
if pocket not in _POCKETS: if pocket not in POCKETS:
raise PocketDoesNotExistError("Pocket '%s' does not exist." % raise PocketDoesNotExistError("Pocket '%s' does not exist." %
pocket) pocket)
@ -1012,7 +1107,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
return self._ppas return self._ppas
def getPPAByName(self, name): def getPPAByName(self, name):
return self._lpobject.getPPAByName(name=name) return Archive(self._lpobject.getPPAByName(name=name))
class Build(BaseWrapper): class Build(BaseWrapper):