pull-uca-source: update to allow using binary pkg names, -pockets, and specific versions

pull-uca-source

@@ -1,11 +1,12 @@
 #!/usr/bin/python
 #
 # pull-uca-source -- pull a source package from Ubuntu Cloud Archive
-# Basic usage: pull-uca-source <source package> <openstack release>
+# Basic usage: pull-uca-source <source package> <openstack release> [version]
 #
 # Copyright (C) 2008,      Iain Lane <iain@orangesquash.org.uk>,
 #               2010-2011, Stefano Rivera <stefanor@ubuntu.com>
 #               2016,      Corey Bryant <corey.bryant@ubuntu.com>
+#               2016,      Dan Streetman <dan.streetman@canonical.com>
 #
 # ##################################################################
 #
@@ -24,56 +25,129 @@
 # ##################################################################
 
 
+import re
+import json
+import os
 import sys
+import urllib2
 from optparse import OptionParser
 
-from ubuntutools.archive import SourcePackage, DownloadError
+from distro_info import UbuntuDistroInfo, DistroDataOutdated
-from ubuntutools.lp.lpapicache import Launchpad
+
+from ubuntutools.archive import UbuntuCloudArchiveSourcePackage, DownloadError
+from ubuntutools.config import UDTConfig
+from ubuntutools.lp.lpapicache import Distribution, Launchpad
+from ubuntutools.lp.udtexceptions import (SeriesNotFoundException,
+                                          PackageNotFoundException,
+                                          PocketDoesNotExistError)
 from ubuntutools.logger import Logger
+from ubuntutools.misc import split_release_pocket
 
 from lazr.restfulclient.errors import NotFound
 
+from launchpadlib.launchpad import Launchpad as LP
+
+
+def showOpenstackReleases(uca):
+    releases = []
+    for p in uca.ppas:
+        if re.match("\w*-staging", p.name):
+            releases.append(re.sub("-staging", "", p.name))
+    Logger.error("Openstack releases are:\n\t%s", ", ".join(releases))
+
+
+def getSPPH(lp, archive, package, version=None, series=None, pocket=None, try_binary=True):
+    params = { 'exact_match': True, 'order_by_date': True, }
+    if pocket:
+        params['pocket'] = pocket
+    if series:
+        params['distro_series'] = series()
+    elif version:
+        params['version'] = version
+    Logger.normal("checking %s version %s pocket %s", package, version, pocket)
+    spphs = archive.getPublishedSources(source_name=package, **params)
+    if spphs:
+        return spphs[0]
+    if not try_binary:
+        return None
+
+    # Didn't find any, maybe the package is a binary package name
+    if series:
+        del params['distro_series']
+        archs = lp.load(series().architectures_collection_link).entries
+        params['distro_arch_series'] = archs[0]['self_link']
+    bpphs = archive.getPublishedBinaries(binary_name=package, **params)
+    if bpphs:
+        bpph_build = lp.load(bpphs[0].build_link)
+        source_package = bpph_build.source_package_name
+        return getSPPH(lp, archive, source_package, version, series, pocket,
+                       try_binary=False)
+
+    return None
+
 
 def main():
-    usage = "Usage: %prog <package> <openstack release>"
+    usage = "Usage: %prog <package> <openstack release> [version]"
     opt_parser = OptionParser(usage)
     opt_parser.add_option('-d', '--download-only',
                           dest='download_only', default=False,
                           action='store_true',
                           help="Do not extract the source package")
+    opt_parser.add_option('-m', '--mirror', metavar='OPENSTACK_MIRROR',
+                          dest='openstack_mirror',
+                          help='Preferred Openstack mirror (default: Launchpad)')
+    opt_parser.add_option('--no-conf',
+                          dest='no_conf', default=False, action='store_true',
+                          help="Don't read config files or environment "
+                               "variables")
     (options, args) = opt_parser.parse_args()
-    if len(sys.argv) != 3:
+    if len(args) < 2:
         opt_parser.error("Must specify package name and openstack release")
 
+    config = UDTConfig(options.no_conf)
+    if options.openstack_mirror is None:
+        options.openstack_mirror = config.get_value('OPENSTACK_MIRROR')
+    mirrors = []
+    if options.openstack_mirror:
+        mirrors.append(options.openstack_mirror)
+
     # Login anonymously to LP
     Launchpad.login_anonymously()
+    lp = LP.login_anonymously("pull-uca-source", "production")
+    uca = lp.people("ubuntu-cloud-archive")
 
     package = str(args[0]).lower()
     release = str(args[1]).lower()
     version = None
+    if len(args) > 2:
+        version = str(args[2])
 
-    # Downloads are from Ubuntu Cloud Archive staging PPAs
+    pocket = None
-    uca = Launchpad.distributions["~ubuntu-cloud-archive"]
-    ppa_name = ''.join([release, '-staging'])
     try:
-        ppa = uca.getPPAByName(name=ppa_name)
+        (release, pocket) = split_release_pocket(release, default=None)
+    except PocketDoesNotExistError, e:
+        pass
+
+    try:
+        archive = uca.getPPAByName(name="%s-staging" % release)
     except NotFound, e:
-        Logger.error('Archive does not exist for OpenStack release: %s',
+        Logger.error('Archive does not exist for Openstack release: %s',
-                     str(release))
+                     release)
+        showOpenstackReleases(uca)
         sys.exit(1)
 
-    srcpkg = None
+    spph = getSPPH(lp, archive, package, version, pocket=pocket)
-    for source in ppa.getPublishedSources(status='Published'):
+    if not spph:
-        if source.source_package_name == package:
+        Logger.error("Package %s in %s not found.", package, release)
-             dsc_file = source.sourceFileUrls()[0]
-             srcpkg = SourcePackage(dscfile=dsc_file)
-             version = srcpkg.dsc['Version']
-
-    if not srcpkg:
-        Logger.error('Package not found: %s', str(package))
         sys.exit(1)
 
-    Logger.normal('Downloading %s version %s', package, version)
+    package = spph.source_package_name
+    version = spph.source_package_version
+    component = spph.component_name
+    Logger.normal('Downloading %s version %s component %s', package, version, component)
+    srcpkg = UbuntuCloudArchiveSourcePackage(release, package, version, component=component,
+                                             mirrors=mirrors)
+
     try:
         srcpkg.pull()
     except DownloadError, e:

setup.py

@@ -38,6 +38,7 @@ else:
            'pull-debian-source',
            'pull-lp-source',
            'pull-revu-source',
+           'pull-uca-source',
            'requestbackport',
            'requestsync',
            'reverse-build-depends',

ubuntutools/archive.py

@@ -538,6 +538,20 @@ class UbuntuSourcePackage(SourcePackage):
     distribution = 'ubuntu'
 
 
+class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage):
+    "Download / unpack an Ubuntu Cloud Archive source package"
+    def __init__(self, uca_release, *args, **kwargs):
+        super(UbuntuCloudArchiveSourcePackage, self).__init__(*args, **kwargs)
+        self._uca_release = uca_release
+        self.masters = [ "http://ubuntu-cloud.archive.canonical.com/ubuntu/" ]
+
+    def _lp_url(self, filename):
+        "Build a source package URL on Launchpad"
+        return os.path.join('https://launchpad.net', "~ubuntu-cloud-archive",
+                            '+archive', ("%s-staging" % self._uca_release),
+                            '+files', filename)
+
+
 class FakeSPPH(object):
     """Provide the same interface as
     ubuntutools.lpapicache.SourcePackagePublishingHistory