pull-pkg: add pull-ppa-* functionality

Add functionality, and frontend pull-ppa-* scripts, to be able to pull
from PPA archives.

doc/pull-pkg.1

@@ -1,7 +1,7 @@
 .TH PULL\-PKG "1" "28 August 2017" "ubuntu-dev-tools"
 
 .SH NAME
-pull\-pkg \- download a package for Debian, Ubuntu, or UCA
+pull\-pkg \- download a package for Debian, Ubuntu, UCA, or a PPA
 
 .SH SYNOPSIS
 .B pull\-pkg \fR[\fIoptions\fR]\fR <\fIpackage name\fR>
@@ -22,9 +22,11 @@ appropriately: these are
 and \fBpull\-lp\-udebs\fR, which all pull Ubuntu packages;
 \fBpull\-debian\-source\fR, \fBpull\-debian\-debs\fR, \fBpull\-debian\-ddebs\fR,
 and \fBpull\-debian\-udebs\fR, which all pull Debian packages;
-and \fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR,
+\fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR,
-and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages.
+and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages;
-Each script pulls the file type in its name, i.e.
+and \fBpull\-ppa\-source\fR, \fBpull\-ppa\-debs\fR, \fBpull\-ppa\-ddebs\fR,
+and \fBpull\-ppa\-udebs\fR, which all pull from a specified Personal Package
+Archive on Launchpad.  Each script pulls the file type in its name, i.e.
 \fIsource\fR, \fIdebs\fR, \fIddebs\fR, or \fIudebs\fR.
 
 .SH OPTIONS
@@ -46,8 +48,7 @@ For ubuntu, you can use either the release name like \fBxenial\fR
 or the release-pocket like \fBxenial-proposed\fR.
 For ubuntu cloud archive (uca) you can use either the uca release
 name like \fBmitaka\fR or the ubuntu and uca release names like
-\fBtrusty-mitaka\fR.
+\fBtrusty-mitaka\fR.  Defaults to the current development release.
-Defaults to the current development release.
 .TP
 .BR \-h ", " \-\-help
 Display a help message and exit.
@@ -82,10 +83,17 @@ source and binary files, but does not actually download any.
 Defaults to \fBsource\fR.
 .TP
 .B \-D \fIDISTRO\fR, \fB\-\-distro\fR=\fIDISTRO\fR
-Pull from: \fBdebian\fR, \fBuca\fR, or \fBubuntu\fR.
+Pull from: \fBdebian\fR, \fBuca\fR, \fBubuntu\fR, or a \fBppa\fR.
 \fBlp\fR can be used instead of \fBubuntu\fR.
 Any string containing \fBcloud\fR can be used instead of \fBuca\fR.
-Deafults to \fBubuntu\fR.
+If pulling from a ppa, you must specify the PPA.  Deafults to \fBubuntu\fR.
+.TP
+.B \-\-ppa\fR=ppa:\fIUSER/NAME\fR
+Applies only when \fBdistro\fR is \fIppa\fR.  Can be provided either as
+a value to the \fB\-\-ppa\fR option parameter, or as a plain option
+(like \fIrelease\fR or \fIversion\fR).  When specified as a plain option,
+the form must be \fBppa:USER/NAME\fR; when specified as a value to the
+\fB\-\-ppa\fR option parameter, the leading \fBppa:\fR is optional.
 
 .SH ENVIRONMENT
 All of the \fBCONFIGURATION VARIABLES\fR below are also supported as
@@ -106,7 +114,7 @@ The default mirror.
 .BR PULL_PKG_UBUNTU_MIRROR
 The default mirror when using the \fBpull\-pkg\fR script.
 .TP
-.BR PULL_[LP|DEBIAN|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR
+.BR PULL_[LP|DEBIAN|PPA|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR
 The default mirror when using the associated script.
 
 .SH SEE ALSO
@@ -119,6 +127,10 @@ The default mirror when using the associated script.
 .BR pull\-debian\-debs (1),
 .BR pull\-debian\-ddebs (1),
 .BR pull\-debian\-udebs (1),
+.BR pull\-ppa\-source (1),
+.BR pull\-ppa\-debs (1),
+.BR pull\-ppa\-ddebs (1),
+.BR pull\-ppa\-udebs (1),
 .BR pull\-uca\-source (1),
 .BR pull\-uca\-debs (1),
 .BR pull\-uca\-ddebs (1),
@@ -128,7 +140,7 @@ The default mirror when using the associated script.
 
 .SH AUTHOR
 .PP
-\fBpull\-pkg\fR was written by Dan Streetman <dan.streetman@canonical.com>,
+\fBpull\-pkg\fR was written by Dan Streetman <ddstreet@canonical.com>,
 based on the original \fBpull\-lp\-source\fR; it and this manual page
 were written by Iain Lane <iain@orangesquash.org.uk>.
 All are released under the GNU General Public License, version 3 or later.

doc/pull-ppa-ddebs.1

@@ -0,0 +1 @@
+pull-pkg.1

doc/pull-ppa-debs.1

@@ -0,0 +1 @@
+pull-pkg.1

doc/pull-ppa-source.1

@@ -0,0 +1 @@
+pull-pkg.1

doc/pull-ppa-udebs.1

@@ -0,0 +1 @@
+pull-pkg.1

pull-ppa-ddebs

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+#
+# pull-ppa-ddebs -- pull ddeb package files for a Launchpad Personal Package Archive
+# Basic usage: pull-ppa-ddebs <package name> <ppa:USER/NAME> [version|release]
+#              pull-ppa-ddebs --ppa USER/NAME <package name> [version|release]
+#
+# See pull-pkg
+
+from ubuntutools.pullpkg import PullPkg
+
+if __name__ == '__main__':
+    PullPkg.main(distro='ppa', pull='ddebs')

pull-ppa-debs

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+#
+# pull-ppa-debs -- pull deb package files for a Launchpad Personal Package Archive
+# Basic usage: pull-ppa-debs <package name> <ppa:USER/NAME> [version|release]
+#              pull-ppa-debs --ppa USER/NAME <package name> [version|release]
+#
+# See pull-pkg
+
+from ubuntutools.pullpkg import PullPkg
+
+if __name__ == '__main__':
+    PullPkg.main(distro='ppa', pull='debs')

pull-ppa-source

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+#
+# pull-ppa-source -- pull source package files for a Launchpad Personal Package Archive
+# Basic usage: pull-ppa-source <package name> <ppa:USER/NAME> [version|release]
+#              pull-ppa-source --ppa USER/NAME <package name> [version|release]
+#
+# See pull-pkg
+
+from ubuntutools.pullpkg import PullPkg
+
+if __name__ == '__main__':
+    PullPkg.main(distro='ppa', pull='source')

pull-ppa-udebs

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+#
+# pull-ppa-udebs -- pull udeb package files for a Launchpad Personal Package Archive
+# Basic usage: pull-ppa-udebs <package name> <ppa:USER/NAME> [version|release]
+#              pull-ppa-udebs --ppa USER/NAME <package name> [version|release]
+#
+# See pull-pkg
+
+from ubuntutools.pullpkg import PullPkg
+
+if __name__ == '__main__':
+    PullPkg.main(distro='ppa', pull='udebs')

setup.py

@@ -37,6 +37,10 @@ scripts = [
     'pull-lp-debs',
     'pull-lp-ddebs',
     'pull-lp-udebs',
+    'pull-ppa-source',
+    'pull-ppa-debs',
+    'pull-ppa-ddebs',
+    'pull-ppa-udebs',
     'pull-revu-source',
     'pull-uca-source',
     'pull-uca-debs',

ubuntutools/archive.py

@@ -719,6 +719,38 @@ class UbuntuSourcePackage(SourcePackage):
     distribution = 'ubuntu'
 
 
+class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage):
+    "Download / unpack an Ubuntu Personal Package Archive source package"
+    def __init__(self, *args, **kwargs):
+        super(PersonalPackageArchiveSourcePackage, self).__init__(*args, **kwargs)
+        assert 'ppa' in kwargs
+        ppa = kwargs['ppa'].split('/')
+        if len(ppa) != 2:
+            raise ValueError('Invalid PPA value "%s",'
+                             'must be "<USER>/<PPA>"' % kwargs['ppa'])
+        self._ppateam = ppa[0]
+        self._ppaname = ppa[1]
+        self.masters = []
+        self._team = None
+        self._ppa = None
+
+    def getArchive(self):
+        if not self._ppa:
+            try:
+                self._team = PersonTeam.fetch(self._ppateam)
+            except KeyError:
+                raise ValueError('No user/team "%s" found on Launchpad' % self._ppateam)
+            self._ppa = self._team.getPPAByName(self._ppaname)
+            Logger.debug('Using PPA %s' % self._ppa.web_link)
+        return self._ppa
+
+    def _lp_url(self, filename):
+        "Build a source package URL on Launchpad"
+        return os.path.join('https://launchpad.net', '~' + self._ppateam,
+                            '+archive', self.distribution, self._ppaname,
+                            '+files', filename)
+
+
 class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage):
     "Download / unpack an Ubuntu Cloud Archive source package"
     _ppas = None
@@ -733,10 +765,12 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage):
 
     @classmethod
     def getArchives(cls):
-        if not cls._ppas:
+        if cls._ppas is None:
-            ppas = filter(lambda p: p.name.endswith('-staging'),
+            cls._ppas = []
-                          PersonTeam.fetch('ubuntu-cloud-archive').getPPAs())
+            ppas = PersonTeam.fetch('ubuntu-cloud-archive').getPPAs()
-            cls._ppas = sorted(ppas, key=lambda p: p.name, reverse=True)
+            for key in ppas.keys():
+                if key.endswith('-staging'):
+                    cls._ppas.append(ppas[key])
         return cls._ppas
 
     @classmethod
@@ -747,7 +781,7 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage):
 
     @classmethod
     def getDevelopmentRelease(cls):
-        return cls.getReleaseNames()[0]
+        return sorted(cls.getReleaseNames(), reverse=True)[0]
 
     @property
     def uca_release(self):

ubuntutools/lp/lpapicache.py

@@ -1005,11 +1005,15 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam):
         return canUpload
 
     def getPPAs(self):
-        if not self._ppas:
+        if self._ppas is None:
-            ppas = Launchpad.load(self._lpobject.ppas_collection_link).entries
+            ppas = [Archive(ppa['self_link']) for ppa in
-            self._ppas = [Archive(a['self_link']) for a in ppas]
+                    Launchpad.load(self._lpobject.ppas_collection_link).entries]
+            self._ppas = {ppa.name: ppa for ppa in ppas}
         return self._ppas
 
+    def getPPAByName(self, name):
+        return self._lpobject.getPPAByName(name=name)
+
 
 class Build(BaseWrapper):
     '''

ubuntutools/pullpkg.py

@@ -23,6 +23,7 @@
 
 
 import re
+import sys
 import errno
 
 from argparse import ArgumentParser
@@ -30,7 +31,8 @@ from argparse import ArgumentParser
 from distro_info import DebianDistroInfo
 
 from ubuntutools.archive import (UbuntuSourcePackage, DebianSourcePackage,
-                                 UbuntuCloudArchiveSourcePackage)
+                                 UbuntuCloudArchiveSourcePackage,
+                                 PersonalPackageArchiveSourcePackage)
 from ubuntutools.config import UDTConfig
 from ubuntutools.lp.lpapicache import (Distribution, Launchpad)
 from ubuntutools.lp.udtexceptions import (SeriesNotFoundException,
@@ -51,11 +53,13 @@ VALID_PULLS = [PULL_SOURCE, PULL_DEBS, PULL_DDEBS, PULL_UDEBS, PULL_LIST]
 DISTRO_DEBIAN = 'debian'
 DISTRO_UBUNTU = 'ubuntu'
 DISTRO_UCA = 'uca'
+DISTRO_PPA = 'ppa'
 
 DISTRO_PKG_CLASS = {
     DISTRO_DEBIAN: DebianSourcePackage,
     DISTRO_UBUNTU: UbuntuSourcePackage,
     DISTRO_UCA: UbuntuCloudArchiveSourcePackage,
+    DISTRO_PPA: PersonalPackageArchiveSourcePackage,
 }
 VALID_DISTROS = DISTRO_PKG_CLASS.keys()
 
@@ -94,6 +98,7 @@ class PullPkg(object):
         self._default_distro = kwargs.get('distro')
         self._default_arch = kwargs.get('arch', host_architecture())
         self._parser = None
+        self._ppa_parser = None
 
     @property
     def argparser(self):
@@ -126,12 +131,41 @@ class PullPkg(object):
                             help=help_default_pull)
         parser.add_argument('-D', '--distro', default=self._default_distro,
                             help=help_default_distro)
+        parser.add_argument('--ppa', help='PPA to pull from')
         parser.add_argument('package', help="Package name to pull")
         parser.add_argument('release', nargs='?', help="Release to pull from")
         parser.add_argument('version', nargs='?', help="Package version to pull")
         self._parser = parser
         return self._parser
 
+    def parse_ppa_args(self, args):
+        """When pulling from PPA, convert from bare ppa:USER/NAME to --ppa option"""
+        if not args:
+            myargs = sys.argv[1:]
+
+        options = vars(self.argparser.parse_known_args(myargs)[0])
+        # we use these, which all should be always provided by the parser,
+        # even if their value is None
+        assert 'distro' in options
+        assert 'ppa' in options
+        assert 'release' in options
+        assert 'version' in options
+
+        # if we're not pulling from a PPA, or if we are but --ppa was given,
+        # then no change to the args is needed
+        if options['distro'] != DISTRO_PPA or options['ppa'] is not None:
+            return args
+
+        # check if release or version is a ppa:
+        # if it is, move it to a --ppa param
+        for param in ['release', 'version']:
+            if str(options[param]).startswith('ppa:'):
+                myargs.remove(options[param])
+                myargs.insert(0, options[param])
+                myargs.insert(0, '--ppa')
+
+        return myargs
+
     def parse_pull(self, pull):
         if not pull:
             raise InvalidPullValueError("Must specify --pull")
@@ -194,7 +228,11 @@ class PullPkg(object):
                 Logger.normal("Using release '%s' for '%s'", codename, release)
                 release = codename
 
-        d = Distribution(distro)
+        if distro == DISTRO_PPA:
+            # PPAs are part of Ubuntu distribution
+            d = Distribution(DISTRO_UBUNTU)
+        else:
+            d = Distribution(distro)
 
         # let SeriesNotFoundException flow up
         d.getSeries(release)
@@ -233,6 +271,7 @@ class PullPkg(object):
         assert 'arch' in options
         assert 'package' in options
         # type string, optional
+        assert 'ppa' in options
         assert 'release' in options
         assert 'version' in options
         # type list of strings, optional
@@ -255,6 +294,14 @@ class PullPkg(object):
             params['dscfile'] = params['package']
             params.pop('package')
 
+        if options['ppa']:
+            if options['ppa'].startswith('ppa:'):
+                params['ppa'] = options['ppa'][4:]
+            else:
+                params['ppa'] = options['ppa']
+        elif distro == DISTRO_PPA:
+            raise ValueError('Must specify PPA to pull from')
+
         mirrors = []
         if options['mirror']:
             mirrors.append(options['mirror'])
@@ -273,6 +320,9 @@ class PullPkg(object):
 
     def pull(self, args=None):
         """Pull (download) specified package file(s)"""
+        # pull-ppa-* may need conversion from ppa:USER/NAME to --ppa USER/NAME
+        args = self.parse_ppa_args(args)
+
         options = vars(self.argparser.parse_args(args))
 
         assert 'verbose' in options