ubuntutools: update archive/lpapicache to optionally search all series

For PPA and UCA repos, the latest build is not necessarily in the
'development' release; so for those SourcePackage classes, search
all the 'active' series (starting with latest devel) for any
matches to the provided package name.  This allows not having to
specify the series name when looking in PPA/UCA repos for the 'latest'
version of a specific package.

ubuntutools/archive.py

@@ -46,8 +46,7 @@ from contextlib import closing
 
 from ubuntutools.config import UDTConfig
 from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam,
-                                       SourcePackagePublishingHistory,
+                                       SourcePackagePublishingHistory)
-                                       BinaryPackagePublishingHistory)
 from ubuntutools.lp.udtexceptions import (PackageNotFoundException,
                                           SeriesNotFoundException,
                                           InvalidDistroValueError)
@@ -154,7 +153,6 @@ class SourcePackage(object):
         self.workdir = workdir
         self.quiet = quiet
         self._series = series
-        self._use_series = True
         self._pocket = pocket
         self._dsc_source = dscfile
         self._verify_signature = verify_signature
@@ -188,48 +186,53 @@ class SourcePackage(object):
             else:
                 Launchpad.login_anonymously()
 
-        distro = self.getDistribution()
         archive = self.getArchive()
-        series = None
+        params = {}
-        params = {'exact_match': True, 'order_by_date': True}
         if self._version:
             # if version was specified, use that
             params['version'] = self._version.full_version
-        elif self._use_series:
+        elif self._series:
-            if self._series:
             # if version not specified, get the latest from this series
-                series = distro.getSeries(self._series)
+            params['series'] = self._series
-            else:
+            # note that if not specified, pocket defaults to all EXCEPT -backports
-                # if no version or series, get the latest from devel series
-                series = distro.getDevelopmentSeries()
-            params['distro_series'] = series()
             if self._pocket:
                 params['pocket'] = self._pocket
-        spphs = archive.getPublishedSources(source_name=self.source, **params)
+        else:
-        if spphs:
+            # We always want to search all series, if not specified
-            self._spph = self.spph_class(spphs[0])
+            params['search_all_series'] = True
-            return self._spph
+
+        try:
+            self._spph = archive.getSourcePackage(self.source,
+                                                  wrapper=self.spph_class,
+                                                  **params)
+            return self._spph
+        except PackageNotFoundException as pnfe:
+            if not self.try_binary or self.binary:
+                # either we don't need to bother trying binary name lookup,
+                # or we've already tried
+                raise pnfe
+
+            Logger.normal('Source package lookup failed, '
+                          'trying lookup of binary package %s' % self.source)
+
+            try:
+                bpph = archive.getBinaryPackage(self.source, **params)
+            except PackageNotFoundException as bpnfe:
+                # log binary lookup failure, in case it provides hints
+                Logger.normal(str(bpnfe))
+                # raise the original exception for the source lookup
+                raise pnfe
 
-        if self.try_binary and not self.binary:
-            if series:
-                arch_series = series.getArchSeries()
-                params['distro_arch_series'] = arch_series()
-                del params['distro_series']
-            bpphs = archive.getPublishedBinaries(binary_name=self.source, **params)
-            if bpphs:
-                bpph = BinaryPackagePublishingHistory(bpphs[0])
             self.binary = self.source
             self.source = bpph.getSourcePackageName()
             Logger.normal("Using source package '{}' for binary package '{}'"
                           .format(self.source, self.binary))
-                try:
+
             spph = bpph.getBuild().getSourcePackagePublishingHistory()
-                except Exception:
-                    spph = None
             if spph:
-                    self._spph = spph
+                self._spph = self.spph_class(spph.self_link)
                 return self._spph
-                else:
+
             # binary build didn't include source link, unfortunately
             # so try again with the updated self.source name
             if not self._version:
@@ -239,15 +242,6 @@ class SourcePackage(object):
                 self._version = Version(bpph.getVersion())
             return self.lp_spph
 
-        msg = "No {} package found".format(self.source)
-        if self._version:
-            msg += " for version {}".format(self._version.full_version)
-        elif series:
-            msg += " in series {}".format(series.name)
-            if self._pocket:
-                msg += " pocket {}".format(self._pocket)
-        raise PackageNotFoundException(msg)
-
     @property
     def version(self):
         "Return Package version"
@@ -761,7 +755,6 @@ class UbuntuCloudArchiveSourcePackage(PersonalPackageArchiveSourcePackage):
         kwargs['ppa'] = ('%s/%s-staging' %
                          (UbuntuCloudArchiveSourcePackage._ppateam, series))
         super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs)
-        self._use_series = False  # each UCA series is for a single Ubuntu series
         self.uca_release = series
         self.masters = ["http://ubuntu-cloud.archive.canonical.com/ubuntu/"]
 
@@ -948,7 +941,7 @@ class SnapshotSourcePackage(SnapshotPackage):
                                         r['architecture'], self.name)
                      for b in response['result']['binaries'] for r in b['files']]
             self._binary_files = files
-        bins = self._binary_files.copy()
+        bins = list(self._binary_files)
         if arch:
             bins = filter(lambda b: b.isArch(arch), bins)
         if name:
@@ -1002,7 +995,7 @@ class SnapshotBinaryPackage(SnapshotPackage):
                                               r['architecture'], self.source)
                            for r in response['result']]
         if not arch:
-            return self._files.copy()
+            return list(self._files)
         return filter(lambda f: f.isArch(arch), self._files)
 
 

ubuntutools/lp/lpapicache.py

@@ -37,7 +37,8 @@ from lazr.restfulclient.resource import Entry
 
 from ubuntutools.version import Version
 from ubuntutools.lp import (service, api_version)
-from ubuntutools.misc import host_architecture
+from ubuntutools.misc import (host_architecture,
+                              DEFAULT_POCKETS, POCKETS)
 from ubuntutools.lp.udtexceptions import (AlreadyLoggedInError,
                                           ArchiveNotFoundException,
                                           ArchSeriesNotFoundException,
@@ -59,8 +60,6 @@ __all__ = [
     'SourcePackagePublishingHistory',
     ]
 
-_POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports')
-
 
 class _Launchpad(object):
     '''Singleton for LP API access.'''
@@ -191,15 +190,23 @@ class Distribution(BaseWrapper):
     resource_type = 'distribution'
 
     def __init__(self, *args):
-        # Don't share _series and _archives between different Distributions
-        if '_series' not in self.__dict__:
-            self._series = dict()
-        if '_archives' not in self.__dict__:
         self._archives = dict()
+        self._series_by_name = dict()
+        self._series = dict()
+        self._dev_series = None
+        self._have_all_series = False
 
     def cache(self):
         self._cache[self.name] = self
 
+    def _cache_series(self, series):
+        '''
+        Add the DistroSeries to the cache if needed.
+        '''
+        if series.version not in self._series:
+            self._series_by_name[series.name] = series
+            self._series[series.version] = series
+
     @classmethod
     def fetch(cls, dist):
         '''
@@ -246,28 +253,49 @@ class Distribution(BaseWrapper):
         (e.g. 'karmic') or version (e.g. '9.10').
         If the series is not found: raise SeriesNotFoundException
         '''
-        if name_or_version not in self._series:
+        if name_or_version in self._series:
+            return self._series[name_or_version]
+        if name_or_version in self._series_by_name:
+            return self._series_by_name[name_or_version]
+
         try:
             series = DistroSeries(self().getSeries(name_or_version=name_or_version))
-                # Cache with name and version
-                self._series[series.name] = series
-                self._series[series.version] = series
         except HTTPError:
             message = "Release '%s' is unknown in '%s'." % \
                       (name_or_version, self.display_name)
             raise SeriesNotFoundException(message)
-        return self._series[name_or_version]
+
+        self._cache_series(series)
+        return series
 
     def getDevelopmentSeries(self):
         '''
         Returns a DistroSeries object of the current development series.
         '''
-        dev = DistroSeries(self.current_series_link)
+        if not self._dev_series:
-        # Cache it in _series if not already done
+            series = DistroSeries(self.current_series_link)
-        if dev.name not in self._series:
+            self._cache_series(series)
-            self._series[dev.name] = dev
+            self._dev_series = series
-            self._series[dev.version] = dev
+        return self._dev_series
-        return dev
+
+    def getAllSeries(self, active=True):
+        '''
+        Returns a list of all DistroSeries objects.
+        '''
+        if not self._have_all_series:
+            for s in Launchpad.load(self.series_collection_link).entries:
+                series = DistroSeries(s['self_link'])
+                self._cache_series(series)
+            self._have_all_series = True
+
+        allseries = filter(lambda s: s.active, self._series.values())
+        allseries = sorted(allseries,
+                           key=lambda s: float(s.version),
+                           reverse=True)
+        Logger.debug("Found series: %s" % ", ".join(map(lambda s: "%s (%s)" %
+                                                        (s.name, s.version),
+                                                        allseries)))
+        return collections.OrderedDict((s.name, s) for s in allseries)
 
 
 class DistroArchSeries(BaseWrapper):
@@ -329,7 +357,7 @@ class Archive(BaseWrapper):
         self._component_uploaders = {}
 
     def getSourcePackage(self, name, series=None, pocket=None, version=None,
-                         status=None):
+                         status=None, wrapper=None, search_all_series=False):
         '''
         Returns a SourcePackagePublishingHistory object for the most
         recent source package in the distribution 'dist', series and
@@ -338,11 +366,20 @@ class Archive(BaseWrapper):
         series defaults to the current development series if not specified.
         series must be either a series name string, or DistroSeries object.
 
-        pocket may be a string or a list. If a list, the highest version
-        will be returned.  It defaults to all pockets except backports.
-
         version may be specified to get only the exact version requested.
 
+        pocket may be a string or a list.  If no version is provided, it
+        defaults to all pockets except 'Backports'; if searching for a
+        specific version, it defaults to all pockets.  Pocket strings must
+        be capitalized.
+
+        wrapper is the class to return an instance of; defaults to
+        SourcePackagePublishingHistory.
+
+        search_all_series is used if series is None.  If False, this will
+        search only the latest devel series, and if True all series
+        will be searched, in reverse order, starting with the latest
+        devel series.  Defaults to False.
         status is optional, to restrict search to a given status only.
 
         If the requested source package doesn't exist a
@@ -351,13 +388,15 @@ class Archive(BaseWrapper):
         return self._getPublishedItem(name, series, pocket, cache=self._srcpkgs,
                                       function='getPublishedSources',
                                       name_key='source_name',
-                                      wrapper=SourcePackagePublishingHistory,
+                                      wrapper=wrapper or SourcePackagePublishingHistory,
                                       version=version,
                                       status=status,
+                                      search_all_series=search_all_series,
                                       binary=False)
 
     def getBinaryPackage(self, name, archtag=None, series=None, pocket=None,
-                         version=None, status=None):
+                         version=None, status=None, wrapper=None,
+                         search_all_series=False):
         '''
         Returns a BinaryPackagePublishingHistory object for the most
         recent source package in the distribution 'dist', architecture
@@ -367,11 +406,20 @@ class Archive(BaseWrapper):
         series must be either a series name string, or DistroArchSeries object.
         series may be omitted if version is specified.
 
-        pocket may be a string or a list. If a list, the highest version
-        will be returned.  It defaults to all pockets except backports.
-
         version may be specified to get only the exact version requested.
 
+        pocket may be a string or a list.  If no version is provided, it
+        defaults to all pockets except 'Backports'; if searching for a
+        specific version, it defaults to all pockets.  Pocket strings must
+        be capitalized.
+
+        wrapper is the class to return an instance of; defaults to
+        BinaryPackagePublishingHistory.
+
+        search_all_series is used if series is None.  If False, this will
+        search only the latest devel series, and if True all series
+        will be searched, in reverse order, starting with the latest
+        devel series.  Defaults to False.
         status is optional, to restrict search to a given status only.
 
         If the requested binary package doesn't exist a
@@ -381,36 +429,40 @@ class Archive(BaseWrapper):
                                       cache=self._binpkgs,
                                       function='getPublishedBinaries',
                                       name_key='binary_name',
-                                      wrapper=BinaryPackagePublishingHistory,
+                                      wrapper=wrapper or BinaryPackagePublishingHistory,
                                       version=version,
                                       status=status,
+                                      search_all_series=search_all_series,
                                       binary=True)
 
     def _getPublishedItem(self, name, series, pocket, cache,
                           function, name_key, wrapper, archtag=None,
-                          version=None, status=None,
+                          version=None, status=None, search_all_series=False,
                           binary=False):
         '''
         Common code between getSourcePackage and getBinaryPackage.
 
         Don't use this directly.
         '''
-        if pocket is None:
+        if not pocket:
-            pockets = frozenset(('Proposed', 'Updates', 'Security', 'Release'))
+            if version and not series:
-        elif isinstance(pocket, str):
+                # check ALL pockets if specific version in any series
-            pockets = frozenset((pocket,))
+                pockets = POCKETS
             else:
-            pockets = frozenset(pocket)
+                # otherwise, check all pockets EXCEPT 'Backports'
+                pockets = DEFAULT_POCKETS
+        elif isinstance(pocket, str):
+            pockets = (pocket,)
+        else:
+            pockets = tuple(pocket)
+
+        for p in pockets:
+            if p not in POCKETS:
+                raise PocketDoesNotExistError("Pocket '%s' does not exist." % p)
 
-        for pocket in pockets:
-            if pocket not in _POCKETS:
-                raise PocketDoesNotExistError("Pocket '%s' does not exist." %
-                                              pocket)
 
         dist = Distribution(self.distribution_link)
 
-        arch_series = None
-
         # please don't pass DistroArchSeries as archtag!
         # but, the code was like that before so keep
         # backwards compatibility.
@@ -418,6 +470,19 @@ class Archive(BaseWrapper):
             series = archtag
             archtag = None
 
+        series_to_check = [series]
+        if not version and not series:
+            # if neither version or series are specified, use either the
+            # devel series or search all series
+            if search_all_series:
+                series_to_check = dist.getAllSeries().values()
+            else:
+                series_to_check = [dist.getDevelopmentSeries()]
+
+        # check each series - if only version was provided, series will be None
+        for series in series_to_check:
+            arch_series = None
+
             if isinstance(series, DistroArchSeries):
                 arch_series = series
                 series = series.getSeries()
@@ -425,8 +490,6 @@ class Archive(BaseWrapper):
                 pass
             elif series:
                 series = dist.getSeries(series)
-        elif not version:
-            series = dist.getDevelopmentSeries()
 
             if binary:
                 if arch_series is None and series:
@@ -438,39 +501,68 @@ class Archive(BaseWrapper):
 
             index = (name, getattr(series, 'name', None), archtag, pockets, status, version)
 
-        if index not in cache:
+            if index in cache:
+                return cache[index]
+
             params = {
                 name_key: name,
                 'exact_match': True,
             }
 
-            if status:
-                params['status'] = status
-
             if arch_series:
                 params['distro_arch_series'] = arch_series()
             elif series:
                 params['distro_series'] = series()
 
             if len(pockets) == 1:
-                params['pocket'] = list(pockets)[0]
+                params['pocket'] = pockets[0]
+
 
             if version:
                 params['version'] = version
 
+            Logger.debug('Calling %s(%s)' % (function,
+                                             ', '.join(['%s=%s' % (k, v)
+                                                        for (k, v) in params.items()])))
             records = getattr(self, function)(**params)
 
-            latest = None
+            err_msg = ("does not exist in the %s %s archive" %
+                       (dist.display_name, self.name))
+
             for record in records:
+                if binary:
+                    rversion = getattr(record, 'binary_package_version', None)
+                else:
+                    rversion = getattr(record, 'source_package_version', None)
+                skipmsg = ('Skipping version %s: ' % rversion)
+
                 if record.pocket not in pockets:
+                    err_msg = 'pocket %s not in (%s)' % (record.pocket, ','.join(pockets))
+                    Logger.debug(skipmsg + err_msg)
+                    continue
                     continue
                 r = wrapper(record)
                 if binary and archtag and archtag != r.arch:
+                    err_msg = 'arch %s does not match requested arch %s' % (r.arch, archtag)
+                    Logger.debug(skipmsg + err_msg)
                     continue
-                if latest is None or latest.getVersion() < r.getVersion():
+                # results are ordered so first is latest
-                    latest = r
+                cache[index] = r
+                return r
+
+        version_with_epoch = None
+        if version and version == Version(version).strip_epoch() and len(records) == 0:
+            # a specific version was asked for, but we found none;
+            # check if one exists with an epoch to give a hint in error msg
+            for epoch in range(1, 9):
+                v = Version(version)
+                v.epoch = epoch
+                params['version'] = v.full_version
+                if len(getattr(self, function)(**params)) > 0:
+                    version_with_epoch = v.full_version
+                    Logger.debug('Found version with epoch %s' % version_with_epoch)
+                    break
 
-            if latest is None:
         if name_key == 'binary_name':
             package_type = "binary package"
         elif name_key == 'source_name':
@@ -480,22 +572,25 @@ class Archive(BaseWrapper):
         msg = "The %s '%s' " % (package_type, name)
         if version:
             msg += "version %s " % version
-                msg += ("does not exist in the %s %s archive" %
+        msg += err_msg
-                        (dist.display_name, self.name))
+        if binary and archtag:
-                if binary:
             msg += " for architecture %s" % archtag
-                if series:
+        if len(series_to_check) > 1:
-                    pockets = [series.name if pocket == 'Release'
+            msg += " in any release"
-                               else '%s-%s' % (series.name, pocket.lower())
+            if len(pockets) == 1:
-                               for pocket in pockets]
+                msg += " for pocket %s" % pockets[0]
-                    if len(pockets) > 1:
+            elif len(pockets) != len(POCKETS):
-                        pockets[-2:] = [' or '.join(pockets[-2:])]
+                msg += " for pockets " + ', '.join(pockets)
-                    msg += " in " + ', '.join(pockets)
+        elif series:
+            msg += " in %s" % series.name
+            if len(pockets) == 1:
+                msg += "-%s" % pockets[0]
+            elif len(pockets) != len(POCKETS):
+                msg += " for pockets " + ', '.join(pockets)
+        if version_with_epoch:
+            msg += " (did you forget the epoch? try %s)" % version_with_epoch
         raise PackageNotFoundException(msg)
 
-            cache[index] = latest
-        return cache[index]
-
     def copyPackage(self, source_name, version, from_archive, to_pocket,
                     to_series=None, sponsored=None, include_binaries=False):
         '''Copy a single named source into this archive.
@@ -695,7 +790,7 @@ class SourcePackagePublishingHistory(BaseWrapper):
                 self._binaries[a][b.binary_package_name] = b
             self._have_all_binaries = True
         else:
-            # Older version, so we have to go the long way :(
+            # we have to go the long way :(
             print("Please wait, this may take some time...")
             archive = self.getArchive()
             urls = self.binaryFileUrls()
@@ -976,7 +1071,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
         if package is None and component is None:
             raise ValueError('Either a source package name or a component has '
                              'to be specified.')
-        if pocket not in _POCKETS:
+        if pocket not in POCKETS:
             raise PocketDoesNotExistError("Pocket '%s' does not exist." %
                                           pocket)
 
@@ -1012,7 +1107,7 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
         return self._ppas
 
     def getPPAByName(self, name):
-        return self._lpobject.getPPAByName(name=name)
+        return Archive(self._lpobject.getPPAByName(name=name))
 
 
 class Build(BaseWrapper):