Lubuntu/ubuntu-dev-tools
3
0
Fork 0
mirror of https://git.launchpad.net/ubuntu-dev-tools synced 2025-03-12 23:51:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

ubuntutools: update archive/lpapicache to optionally search all series

Browse Source 
For PPA and UCA repos, the latest build is not necessarily in the
'development' release; so for those SourcePackage classes, search
all the 'active' series (starting with latest devel) for any
matches to the provided package name.  This allows not having to
specify the series name when looking in PPA/UCA repos for the 'latest'
version of a specific package.
This commit is contained in:
Dan Streetman 2018-07-24 16:28:25 -04:00
parent b11b83f0e2
commit 0f61836b10
2 changed files with 242 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
ubuntutools/archive.py
View File

@ -46,8 +46,7 @@ from contextlib import closing
from ubuntutools.config import UDTConfig
from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam,
SourcePackagePublishingHistory,
BinaryPackagePublishingHistory)
SourcePackagePublishingHistory)
from ubuntutools.lp.udtexceptions import (PackageNotFoundException,
SeriesNotFoundException,
InvalidDistroValueError)
@ -154,7 +153,6 @@ class SourcePackage(object):
self.workdir = workdir
self.quiet = quiet
self._series = series
self._use_series = True
self._pocket = pocket
self._dsc_source = dscfile
self._verify_signature = verify_signature
@ -188,65 +186,61 @@ class SourcePackage(object):
else:
Launchpad.login_anonymously()
distro = self.getDistribution()
archive = self.getArchive()
series = None
params = {'exact_match': True, 'order_by_date': True}
params = {}
if self._version:
# if version was specified, use that
params['version'] = self._version.full_version
elif self._use_series:
if self._series:
# if version not specified, get the latest from this series
series = distro.getSeries(self._series)
else:
# if no version or series, get the latest from devel series
series = distro.getDevelopmentSeries()
params['distro_series'] = series()
elif self._series:
# if version not specified, get the latest from this series
params['series'] = self._series
# note that if not specified, pocket defaults to all EXCEPT -backports
if self._pocket:
params['pocket'] = self._pocket
spphs = archive.getPublishedSources(source_name=self.source, **params)
if spphs:
self._spph = self.spph_class(spphs[0])
else:
# We always want to search all series, if not specified
params['search_all_series'] = True
try:
self._spph = archive.getSourcePackage(self.source,
wrapper=self.spph_class,
**params)
return self._spph
except PackageNotFoundException as pnfe:
if not self.try_binary or self.binary:
# either we don't need to bother trying binary name lookup,
# or we've already tried
raise pnfe
if self.try_binary and not self.binary:
if series:
arch_series = series.getArchSeries()
params['distro_arch_series'] = arch_series()
del params['distro_series']
bpphs = archive.getPublishedBinaries(binary_name=self.source, **params)
if bpphs:
bpph = BinaryPackagePublishingHistory(bpphs[0])
self.binary = self.source
self.source = bpph.getSourcePackageName()
Logger.normal("Using source package '{}' for binary package '{}'"
.format(self.source, self.binary))
try:
spph = bpph.getBuild().getSourcePackagePublishingHistory()
except Exception:
spph = None
if spph:
self._spph = spph
return self._spph
else:
# binary build didn't include source link, unfortunately
# so try again with the updated self.source name
if not self._version:
# Get version first if user didn't specify it, as some
# binaries have their version hardcoded in their name,
# such as the kernel package
self._version = Version(bpph.getVersion())
return self.lp_spph
Logger.normal('Source package lookup failed, '
'trying lookup of binary package %s' % self.source)
msg = "No {} package found".format(self.source)
if self._version:
msg += " for version {}".format(self._version.full_version)
elif series:
msg += " in series {}".format(series.name)
if self._pocket:
msg += " pocket {}".format(self._pocket)
raise PackageNotFoundException(msg)
try:
bpph = archive.getBinaryPackage(self.source, **params)
except PackageNotFoundException as bpnfe:
# log binary lookup failure, in case it provides hints
Logger.normal(str(bpnfe))
# raise the original exception for the source lookup
raise pnfe
self.binary = self.source
self.source = bpph.getSourcePackageName()
Logger.normal("Using source package '{}' for binary package '{}'"
.format(self.source, self.binary))
spph = bpph.getBuild().getSourcePackagePublishingHistory()
if spph:
self._spph = self.spph_class(spph.self_link)
return self._spph
# binary build didn't include source link, unfortunately
# so try again with the updated self.source name
if not self._version:
# Get version first if user didn't specify it, as some
# binaries have their version hardcoded in their name,
# such as the kernel package
self._version = Version(bpph.getVersion())
return self.lp_spph
@property
def version(self):
@ -761,7 +755,6 @@ class UbuntuCloudArchiveSourcePackage(PersonalPackageArchiveSourcePackage):
kwargs['ppa'] = ('%s/%s-staging' %
(UbuntuCloudArchiveSourcePackage._ppateam, series))
super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs)
self._use_series = False # each UCA series is for a single Ubuntu series
self.uca_release = series
self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"]
@ -948,7 +941,7 @@ class SnapshotSourcePackage(SnapshotPackage):
r['architecture'], self.name)
for b in response['result']['binaries'] for r in b['files']]
self._binary_files = files
bins = self._binary_files.copy()
bins = list(self._binary_files)
if arch:
bins = filter(lambda b: b.isArch(arch), bins)
if name:
@ -1002,7 +995,7 @@ class SnapshotBinaryPackage(SnapshotPackage):
r['architecture'], self.source)
for r in response['result']]
if not arch:
return self._files.copy()
return list(self._files)
return filter(lambda f: f.isArch(arch), self._files)

291
ubuntutools/lp/lpapicache.py
View File

@ -37,7 +37,8 @@ from lazr.restfulclient.resource import Entry
from ubuntutools.version import Version
from ubuntutools.lp import (service, api_version)
from ubuntutools.misc import host_architecture
from ubuntutools.misc import (host_architecture,
DEFAULT_POCKETS, POCKETS)
from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError,
ArchiveNotFoundException,
ArchSeriesNotFoundException,
@ -59,8 +60,6 @@ __all__ = [
'SourcePackagePublishingHistory',
]
_POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports')
class _Launchpad(object):
'''Singleton for LP API access.'''
@ -191,15 +190,23 @@ class Distribution(BaseWrapper):
resource_type = 'distribution'
def __init__(self, *args):
# Don't share _series and _archives between different Distributions
if '_series' not in self.__dict__:
self._series = dict()
if '_archives' not in self.__dict__:
self._archives = dict()
self._archives = dict()
self._series_by_name = dict()
self._series = dict()
self._dev_series = None
self._have_all_series = False
def cache(self):
self._cache[self.name] = self
def _cache_series(self, series):
'''
Add the DistroSeries to the cache if needed.
'''
if series.version not in self._series:
self._series_by_name[series.name] = series
self._series[series.version] = series
@classmethod
def fetch(cls, dist):
'''
@ -246,28 +253,49 @@ class Distribution(BaseWrapper):
(e.g. 'karmic') or version (e.g. '9.10').
If the series is not found: raise SeriesNotFoundException
'''
if name_or_version not in self._series:
try:
series = DistroSeries(self().getSeries(name_or_version=name_or_version))
# Cache with name and version
self._series[series.name] = series
self._series[series.version] = series
except HTTPError:
message = "Release '%s' is unknown in '%s'." % \
(name_or_version, self.display_name)
raise SeriesNotFoundException(message)
return self._series[name_or_version]
if name_or_version in self._series:
return self._series[name_or_version]
if name_or_version in self._series_by_name:
return self._series_by_name[name_or_version]
try:
series = DistroSeries(self().getSeries(name_or_version=name_or_version))
except HTTPError:
message = "Release '%s' is unknown in '%s'." % \
(name_or_version, self.display_name)
raise SeriesNotFoundException(message)
self._cache_series(series)
return series
def getDevelopmentSeries(self):
'''
Returns a DistroSeries object of the current development series.
'''
dev = DistroSeries(self.current_series_link)
# Cache it in _series if not already done
if dev.name not in self._series:
self._series[dev.name] = dev
self._series[dev.version] = dev
return dev
if not self._dev_series:
series = DistroSeries(self.current_series_link)
self._cache_series(series)
self._dev_series = series
return self._dev_series
def getAllSeries(self, active=True):
'''
Returns a list of all DistroSeries objects.
'''
if not self._have_all_series:
for s in Launchpad.load(self.series_collection_link).entries:
series = DistroSeries(s['self_link'])
self._cache_series(series)
self._have_all_series = True
allseries = filter(lambda s: s.active, self._series.values())
allseries = sorted(allseries,
key=lambda s: float(s.version),
reverse=True)
Logger.debug("Found series: %s" % ", ".join(map(lambda s: "%s (%s)" %
(s.name, s.version),
allseries)))
return collections.OrderedDict((s.name, s) for s in allseries)
class DistroArchSeries(BaseWrapper):
@ -329,7 +357,7 @@ class Archive(BaseWrapper):
self._component_uploaders = {}
def getSourcePackage(self, name, series=None, pocket=None, version=None,
status=None):
status=None, wrapper=None, search_all_series=False):
'''
Returns a SourcePackagePublishingHistory object for the most
recent source package in the distribution 'dist', series and
@ -338,11 +366,20 @@ class Archive(BaseWrapper):
series defaults to the current development series if not specified.
series must be either a series name string, or DistroSeries object.
pocket may be a string or a list. If a list, the highest version
will be returned. It defaults to all pockets except backports.
version may be specified to get only the exact version requested.
pocket may be a string or a list. If no version is provided, it
defaults to all pockets except 'Backports'; if searching for a
specific version, it defaults to all pockets. Pocket strings must
be capitalized.
wrapper is the class to return an instance of; defaults to
SourcePackagePublishingHistory.
search_all_series is used if series is None. If False, this will
search only the latest devel series, and if True all series
will be searched, in reverse order, starting with the latest
devel series. Defaults to False.
status is optional, to restrict search to a given status only.
If the requested source package doesn't exist a
@ -351,13 +388,15 @@ class Archive(BaseWrapper):
return self._getPublishedItem(name, series, pocket, cache=self._srcpkgs,
function='getPublishedSources',
name_key='source_name',
wrapper=SourcePackagePublishingHistory,
wrapper=wrapper or SourcePackagePublishingHistory,
version=version,
status=status,
search_all_series=search_all_series,
binary=False)
def getBinaryPackage(self, name, archtag=None, series=None, pocket=None,
version=None, status=None):
version=None, status=None, wrapper=None,
search_all_series=False):
'''
Returns a BinaryPackagePublishingHistory object for the most
recent source package in the distribution 'dist', architecture
@ -367,11 +406,20 @@ class Archive(BaseWrapper):
series must be either a series name string, or DistroArchSeries object.
series may be omitted if version is specified.
pocket may be a string or a list. If a list, the highest version
will be returned. It defaults to all pockets except backports.
version may be specified to get only the exact version requested.
pocket may be a string or a list. If no version is provided, it
defaults to all pockets except 'Backports'; if searching for a
specific version, it defaults to all pockets. Pocket strings must
be capitalized.
wrapper is the class to return an instance of; defaults to
BinaryPackagePublishingHistory.
search_all_series is used if series is None. If False, this will
search only the latest devel series, and if True all series
will be searched, in reverse order, starting with the latest
devel series. Defaults to False.
status is optional, to restrict search to a given status only.
If the requested binary package doesn't exist a
@ -381,36 +429,40 @@ class Archive(BaseWrapper):
cache=self._binpkgs,
function='getPublishedBinaries',
name_key='binary_name',
wrapper=BinaryPackagePublishingHistory,
wrapper=wrapper or BinaryPackagePublishingHistory,
version=version,
status=status,
search_all_series=search_all_series,
binary=True)
def _getPublishedItem(self, name, series, pocket, cache,
function, name_key, wrapper, archtag=None,
version=None, status=None,
version=None, status=None, search_all_series=False,
binary=False):
'''
Common code between getSourcePackage and getBinaryPackage.
Don't use this directly.
'''
if pocket is None:
pockets = frozenset(('Proposed', 'Updates', 'Security', 'Release'))
if not pocket:
if version and not series:
# check ALL pockets if specific version in any series
pockets = POCKETS
else:
# otherwise, check all pockets EXCEPT 'Backports'
pockets = DEFAULT_POCKETS
elif isinstance(pocket, str):
pockets = frozenset((pocket,))
pockets = (pocket,)
else:
pockets = frozenset(pocket)
pockets = tuple(pocket)
for p in pockets:
if p not in POCKETS:
raise PocketDoesNotExistError("Pocket '%s' does not exist." % p)
for pocket in pockets:
if pocket not in _POCKETS:
raise PocketDoesNotExistError("Pocket '%s' does not exist." %
pocket)
dist = Distribution(self.distribution_link)
arch_series = None
# please don't pass DistroArchSeries as archtag!
# but, the code was like that before so keep
# backwards compatibility.
@ -418,83 +470,126 @@ class Archive(BaseWrapper):
series = archtag
archtag = None
if isinstance(series, DistroArchSeries):
arch_series = series
series = series.getSeries()
elif isinstance(series, DistroSeries):
pass
elif series:
series = dist.getSeries(series)
elif not version:
series = dist.getDevelopmentSeries()
series_to_check = [series]
if not version and not series:
# if neither version or series are specified, use either the
# devel series or search all series
if search_all_series:
series_to_check = dist.getAllSeries().values()
else:
series_to_check = [dist.getDevelopmentSeries()]
if binary:
if arch_series is None and series:
arch_series = series.getArchSeries(archtag=archtag)
if archtag is None and arch_series:
archtag = arch_series.architecture_tag
if archtag is None:
archtag = host_architecture()
# check each series - if only version was provided, series will be None
for series in series_to_check:
arch_series = None
index = (name, getattr(series, 'name', None), archtag, pockets, status, version)
if isinstance(series, DistroArchSeries):
arch_series = series
series = series.getSeries()
elif isinstance(series, DistroSeries):
pass
elif series:
series = dist.getSeries(series)
if binary:
if arch_series is None and series:
arch_series = series.getArchSeries(archtag=archtag)
if archtag is None and arch_series:
archtag = arch_series.architecture_tag
if archtag is None:
archtag = host_architecture()
index = (name, getattr(series, 'name', None), archtag, pockets, status, version)
if index in cache:
return cache[index]
if index not in cache:
params = {
name_key: name,
'exact_match': True,
}
if status:
params['status'] = status
if arch_series:
params['distro_arch_series'] = arch_series()
elif series:
params['distro_series'] = series()
if len(pockets) == 1:
params['pocket'] = list(pockets)[0]
params['pocket'] = pockets[0]
if version:
params['version'] = version
Logger.debug('Calling %s(%s)' % (function,
', '.join(['%s=%s' % (k, v)
for (k, v) in params.items()])))
records = getattr(self, function)(**params)
latest = None
err_msg = ("does not exist in the %s %s archive" %
(dist.display_name, self.name))
for record in records:
if binary:
rversion = getattr(record, 'binary_package_version', None)
else:
rversion = getattr(record, 'source_package_version', None)
skipmsg = ('Skipping version %s: ' % rversion)
if record.pocket not in pockets:
err_msg = 'pocket %s not in (%s)' % (record.pocket, ','.join(pockets))
Logger.debug(skipmsg + err_msg)
continue
continue
r = wrapper(record)
if binary and archtag and archtag != r.arch:
err_msg = 'arch %s does not match requested arch %s' % (r.arch, archtag)
Logger.debug(skipmsg + err_msg)
continue
if latest is None or latest.getVersion() < r.getVersion():
latest = r
# results are ordered so first is latest
cache[index] = r
return r
if latest is None:
if name_key == 'binary_name':
package_type = "binary package"
elif name_key == 'source_name':
package_type = "source package"
else:
package_type = "package"
msg = "The %s '%s' " % (package_type, name)
if version:
msg += "version %s " % version
msg += ("does not exist in the %s %s archive" %
(dist.display_name, self.name))
if binary:
msg += " for architecture %s" % archtag
if series:
pockets = [series.name if pocket == 'Release'
else '%s-%s' % (series.name, pocket.lower())
for pocket in pockets]
if len(pockets) > 1:
pockets[-2:] = [' or '.join(pockets[-2:])]
msg += " in " + ', '.join(pockets)
raise PackageNotFoundException(msg)
version_with_epoch = None
if version and version == Version(version).strip_epoch() and len(records) == 0:
# a specific version was asked for, but we found none;
# check if one exists with an epoch to give a hint in error msg
for epoch in range(1, 9):
v = Version(version)
v.epoch = epoch
params['version'] = v.full_version
if len(getattr(self, function)(**params)) > 0:
version_with_epoch = v.full_version
Logger.debug('Found version with epoch %s' % version_with_epoch)
break
cache[index] = latest
return cache[index]
if name_key == 'binary_name':
package_type = "binary package"
elif name_key == 'source_name':
package_type = "source package"
else:
package_type = "package"
msg = "The %s '%s' " % (package_type, name)
if version:
msg += "version %s " % version
msg += err_msg
if binary and archtag:
msg += " for architecture %s" % archtag
if len(series_to_check) > 1:
msg += " in any release"
if len(pockets) == 1:
msg += " for pocket %s" % pockets[0]
elif len(pockets) != len(POCKETS):
msg += " for pockets " + ', '.join(pockets)
elif series:
msg += " in %s" % series.name
if len(pockets) == 1:
msg += "-%s" % pockets[0]
elif len(pockets) != len(POCKETS):
msg += " for pockets " + ', '.join(pockets)
if version_with_epoch:
msg += " (did you forget the epoch? try %s)" % version_with_epoch
raise PackageNotFoundException(msg)
def copyPackage(self, source_name, version, from_archive, to_pocket,
to_series=None, sponsored=None, include_binaries=False):
@ -695,7 +790,7 @@ class SourcePackagePublishingHistory(BaseWrapper):
self._binaries[a][b.binary_package_name] = b
self._have_all_binaries = True
else:
# Older version, so we have to go the long way :(
# we have to go the long way :(
print("Please wait, this may take some time...")
archive = self.getArchive()
urls = self.binaryFileUrls()
@ -976,7 +1071,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
if package is None and component is None:
raise ValueError('Either a source package name or a component has '
'to be specified.')
if pocket not in _POCKETS:
if pocket not in POCKETS:
raise PocketDoesNotExistError("Pocket '%s' does not exist." %
pocket)
@ -1012,7 +1107,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
return self._ppas
def getPPAByName(self, name):
return self._lpobject.getPPAByName(name=name)
return Archive(self._lpobject.getPPAByName(name=name))
class Build(BaseWrapper):