From 7084bfc8bc4bc292923a0b40935d14825256c7b2 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 10 Jul 2018 14:23:21 -0400 Subject: [PATCH] pull-pkg: add pull-ppa-* functionality Add functionality, and frontend pull-ppa-* scripts, to be able to pull from PPA archives. --- doc/pull-pkg.1 | 32 ++++++++++++++------- doc/pull-ppa-ddebs.1 | 1 + doc/pull-ppa-debs.1 | 1 + doc/pull-ppa-source.1 | 1 + doc/pull-ppa-udebs.1 | 1 + pull-ppa-ddebs | 12 ++++++++ pull-ppa-debs | 12 ++++++++ pull-ppa-source | 12 ++++++++ pull-ppa-udebs | 12 ++++++++ setup.py | 4 +++ ubuntutools/archive.py | 44 +++++++++++++++++++++++++---- ubuntutools/lp/lpapicache.py | 10 +++++-- ubuntutools/pullpkg.py | 54 ++++++++++++++++++++++++++++++++++-- 13 files changed, 176 insertions(+), 20 deletions(-) create mode 120000 doc/pull-ppa-ddebs.1 create mode 120000 doc/pull-ppa-debs.1 create mode 120000 doc/pull-ppa-source.1 create mode 120000 doc/pull-ppa-udebs.1 create mode 100755 pull-ppa-ddebs create mode 100755 pull-ppa-debs create mode 100755 pull-ppa-source create mode 100755 pull-ppa-udebs diff --git a/doc/pull-pkg.1 b/doc/pull-pkg.1 index f62aa11..08e4f68 100644 --- a/doc/pull-pkg.1 +++ b/doc/pull-pkg.1 @@ -1,7 +1,7 @@ .TH PULL\-PKG "1" "28 August 2017" "ubuntu-dev-tools" .SH NAME -pull\-pkg \- download a package for Debian, Ubuntu, or UCA +pull\-pkg \- download a package for Debian, Ubuntu, UCA, or a PPA .SH SYNOPSIS .B pull\-pkg \fR[\fIoptions\fR]\fR <\fIpackage name\fR> @@ -22,9 +22,11 @@ appropriately: these are and \fBpull\-lp\-udebs\fR, which all pull Ubuntu packages; \fBpull\-debian\-source\fR, \fBpull\-debian\-debs\fR, \fBpull\-debian\-ddebs\fR, and \fBpull\-debian\-udebs\fR, which all pull Debian packages; -and \fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR, -and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages. -Each script pulls the file type in its name, i.e. +\fBpull\-uca\-source\fR, \fBpull\-uca\-debs\fR, \fBpull\-uca\-ddebs\fR, +and \fBpull\-uca\-udebs\fR, which all pull Ubuntu Cloud Archive packages; +and \fBpull\-ppa\-source\fR, \fBpull\-ppa\-debs\fR, \fBpull\-ppa\-ddebs\fR, +and \fBpull\-ppa\-udebs\fR, which all pull from a specified Personal Package +Archive on Launchpad. Each script pulls the file type in its name, i.e. \fIsource\fR, \fIdebs\fR, \fIddebs\fR, or \fIudebs\fR. .SH OPTIONS @@ -46,8 +48,7 @@ For ubuntu, you can use either the release name like \fBxenial\fR or the release-pocket like \fBxenial-proposed\fR. For ubuntu cloud archive (uca) you can use either the uca release name like \fBmitaka\fR or the ubuntu and uca release names like -\fBtrusty-mitaka\fR. -Defaults to the current development release. +\fBtrusty-mitaka\fR. Defaults to the current development release. .TP .BR \-h ", " \-\-help Display a help message and exit. @@ -82,10 +83,17 @@ source and binary files, but does not actually download any. Defaults to \fBsource\fR. .TP .B \-D \fIDISTRO\fR, \fB\-\-distro\fR=\fIDISTRO\fR -Pull from: \fBdebian\fR, \fBuca\fR, or \fBubuntu\fR. +Pull from: \fBdebian\fR, \fBuca\fR, \fBubuntu\fR, or a \fBppa\fR. \fBlp\fR can be used instead of \fBubuntu\fR. Any string containing \fBcloud\fR can be used instead of \fBuca\fR. -Deafults to \fBubuntu\fR. +If pulling from a ppa, you must specify the PPA. Deafults to \fBubuntu\fR. +.TP +.B \-\-ppa\fR=ppa:\fIUSER/NAME\fR +Applies only when \fBdistro\fR is \fIppa\fR. Can be provided either as +a value to the \fB\-\-ppa\fR option parameter, or as a plain option +(like \fIrelease\fR or \fIversion\fR). When specified as a plain option, +the form must be \fBppa:USER/NAME\fR; when specified as a value to the +\fB\-\-ppa\fR option parameter, the leading \fBppa:\fR is optional. .SH ENVIRONMENT All of the \fBCONFIGURATION VARIABLES\fR below are also supported as @@ -106,7 +114,7 @@ The default mirror. .BR PULL_PKG_UBUNTU_MIRROR The default mirror when using the \fBpull\-pkg\fR script. .TP -.BR PULL_[LP|DEBIAN|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR +.BR PULL_[LP|DEBIAN|PPA|UCA]_[SOURCE|DEBS|DDEBS|UDEBS]_MIRROR The default mirror when using the associated script. .SH SEE ALSO @@ -119,6 +127,10 @@ The default mirror when using the associated script. .BR pull\-debian\-debs (1), .BR pull\-debian\-ddebs (1), .BR pull\-debian\-udebs (1), +.BR pull\-ppa\-source (1), +.BR pull\-ppa\-debs (1), +.BR pull\-ppa\-ddebs (1), +.BR pull\-ppa\-udebs (1), .BR pull\-uca\-source (1), .BR pull\-uca\-debs (1), .BR pull\-uca\-ddebs (1), @@ -128,7 +140,7 @@ The default mirror when using the associated script. .SH AUTHOR .PP -\fBpull\-pkg\fR was written by Dan Streetman , +\fBpull\-pkg\fR was written by Dan Streetman , based on the original \fBpull\-lp\-source\fR; it and this manual page were written by Iain Lane . All are released under the GNU General Public License, version 3 or later. diff --git a/doc/pull-ppa-ddebs.1 b/doc/pull-ppa-ddebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-ddebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-ppa-debs.1 b/doc/pull-ppa-debs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-debs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-ppa-source.1 b/doc/pull-ppa-source.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-source.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/doc/pull-ppa-udebs.1 b/doc/pull-ppa-udebs.1 new file mode 120000 index 0000000..a67bbc7 --- /dev/null +++ b/doc/pull-ppa-udebs.1 @@ -0,0 +1 @@ +pull-pkg.1 \ No newline at end of file diff --git a/pull-ppa-ddebs b/pull-ppa-ddebs new file mode 100755 index 0000000..8b9c3cf --- /dev/null +++ b/pull-ppa-ddebs @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-ddebs -- pull ddeb package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-ddebs [version|release] +# pull-ppa-ddebs --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='ddebs') diff --git a/pull-ppa-debs b/pull-ppa-debs new file mode 100755 index 0000000..0804e29 --- /dev/null +++ b/pull-ppa-debs @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-debs -- pull deb package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-debs [version|release] +# pull-ppa-debs --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='debs') diff --git a/pull-ppa-source b/pull-ppa-source new file mode 100755 index 0000000..065bb2b --- /dev/null +++ b/pull-ppa-source @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-source -- pull source package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-source [version|release] +# pull-ppa-source --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='source') diff --git a/pull-ppa-udebs b/pull-ppa-udebs new file mode 100755 index 0000000..4e59c3a --- /dev/null +++ b/pull-ppa-udebs @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# +# pull-ppa-udebs -- pull udeb package files for a Launchpad Personal Package Archive +# Basic usage: pull-ppa-udebs [version|release] +# pull-ppa-udebs --ppa USER/NAME [version|release] +# +# See pull-pkg + +from ubuntutools.pullpkg import PullPkg + +if __name__ == '__main__': + PullPkg.main(distro='ppa', pull='udebs') diff --git a/setup.py b/setup.py index 35fdddc..7078931 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,10 @@ scripts = [ 'pull-lp-debs', 'pull-lp-ddebs', 'pull-lp-udebs', + 'pull-ppa-source', + 'pull-ppa-debs', + 'pull-ppa-ddebs', + 'pull-ppa-udebs', 'pull-revu-source', 'pull-uca-source', 'pull-uca-debs', diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 47a7e6a..3c78d82 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -719,6 +719,38 @@ class UbuntuSourcePackage(SourcePackage): distribution = 'ubuntu' +class PersonalPackageArchiveSourcePackage(UbuntuSourcePackage): + "Download / unpack an Ubuntu Personal Package Archive source package" + def __init__(self, *args, **kwargs): + super(PersonalPackageArchiveSourcePackage, self).__init__(*args, **kwargs) + assert 'ppa' in kwargs + ppa = kwargs['ppa'].split('/') + if len(ppa) != 2: + raise ValueError('Invalid PPA value "%s",' + 'must be "/"' % kwargs['ppa']) + self._ppateam = ppa[0] + self._ppaname = ppa[1] + self.masters = [] + self._team = None + self._ppa = None + + def getArchive(self): + if not self._ppa: + try: + self._team = PersonTeam.fetch(self._ppateam) + except KeyError: + raise ValueError('No user/team "%s" found on Launchpad' % self._ppateam) + self._ppa = self._team.getPPAByName(self._ppaname) + Logger.debug('Using PPA %s' % self._ppa.web_link) + return self._ppa + + def _lp_url(self, filename): + "Build a source package URL on Launchpad" + return os.path.join('https://launchpad.net', '~' + self._ppateam, + '+archive', self.distribution, self._ppaname, + '+files', filename) + + class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): "Download / unpack an Ubuntu Cloud Archive source package" _ppas = None @@ -733,10 +765,12 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): @classmethod def getArchives(cls): - if not cls._ppas: - ppas = filter(lambda p: p.name.endswith('-staging'), - PersonTeam.fetch('ubuntu-cloud-archive').getPPAs()) - cls._ppas = sorted(ppas, key=lambda p: p.name, reverse=True) + if cls._ppas is None: + cls._ppas = [] + ppas = PersonTeam.fetch('ubuntu-cloud-archive').getPPAs() + for key in ppas.keys(): + if key.endswith('-staging'): + cls._ppas.append(ppas[key]) return cls._ppas @classmethod @@ -747,7 +781,7 @@ class UbuntuCloudArchiveSourcePackage(UbuntuSourcePackage): @classmethod def getDevelopmentRelease(cls): - return cls.getReleaseNames()[0] + return sorted(cls.getReleaseNames(), reverse=True)[0] @property def uca_release(self): diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index f69f456..5215108 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -1005,11 +1005,15 @@ class PersonTeam(BaseWrapper, metaclass=MetaPersonTeam): return canUpload def getPPAs(self): - if not self._ppas: - ppas = Launchpad.load(self._lpobject.ppas_collection_link).entries - self._ppas = [Archive(a['self_link']) for a in ppas] + if self._ppas is None: + ppas = [Archive(ppa['self_link']) for ppa in + Launchpad.load(self._lpobject.ppas_collection_link).entries] + self._ppas = {ppa.name: ppa for ppa in ppas} return self._ppas + def getPPAByName(self, name): + return self._lpobject.getPPAByName(name=name) + class Build(BaseWrapper): ''' diff --git a/ubuntutools/pullpkg.py b/ubuntutools/pullpkg.py index 9704a6e..ee7939e 100644 --- a/ubuntutools/pullpkg.py +++ b/ubuntutools/pullpkg.py @@ -23,6 +23,7 @@ import re +import sys import errno from argparse import ArgumentParser @@ -30,7 +31,8 @@ from argparse import ArgumentParser from distro_info import DebianDistroInfo from ubuntutools.archive import (UbuntuSourcePackage, DebianSourcePackage, - UbuntuCloudArchiveSourcePackage) + UbuntuCloudArchiveSourcePackage, + PersonalPackageArchiveSourcePackage) from ubuntutools.config import UDTConfig from ubuntutools.lp.lpapicache import (Distribution, Launchpad) from ubuntutools.lp.udtexceptions import (SeriesNotFoundException, @@ -51,11 +53,13 @@ VALID_PULLS = [PULL_SOURCE, PULL_DEBS, PULL_DDEBS, PULL_UDEBS, PULL_LIST] DISTRO_DEBIAN = 'debian' DISTRO_UBUNTU = 'ubuntu' DISTRO_UCA = 'uca' +DISTRO_PPA = 'ppa' DISTRO_PKG_CLASS = { DISTRO_DEBIAN: DebianSourcePackage, DISTRO_UBUNTU: UbuntuSourcePackage, DISTRO_UCA: UbuntuCloudArchiveSourcePackage, + DISTRO_PPA: PersonalPackageArchiveSourcePackage, } VALID_DISTROS = DISTRO_PKG_CLASS.keys() @@ -94,6 +98,7 @@ class PullPkg(object): self._default_distro = kwargs.get('distro') self._default_arch = kwargs.get('arch', host_architecture()) self._parser = None + self._ppa_parser = None @property def argparser(self): @@ -126,12 +131,41 @@ class PullPkg(object): help=help_default_pull) parser.add_argument('-D', '--distro', default=self._default_distro, help=help_default_distro) + parser.add_argument('--ppa', help='PPA to pull from') parser.add_argument('package', help="Package name to pull") parser.add_argument('release', nargs='?', help="Release to pull from") parser.add_argument('version', nargs='?', help="Package version to pull") self._parser = parser return self._parser + def parse_ppa_args(self, args): + """When pulling from PPA, convert from bare ppa:USER/NAME to --ppa option""" + if not args: + myargs = sys.argv[1:] + + options = vars(self.argparser.parse_known_args(myargs)[0]) + # we use these, which all should be always provided by the parser, + # even if their value is None + assert 'distro' in options + assert 'ppa' in options + assert 'release' in options + assert 'version' in options + + # if we're not pulling from a PPA, or if we are but --ppa was given, + # then no change to the args is needed + if options['distro'] != DISTRO_PPA or options['ppa'] is not None: + return args + + # check if release or version is a ppa: + # if it is, move it to a --ppa param + for param in ['release', 'version']: + if str(options[param]).startswith('ppa:'): + myargs.remove(options[param]) + myargs.insert(0, options[param]) + myargs.insert(0, '--ppa') + + return myargs + def parse_pull(self, pull): if not pull: raise InvalidPullValueError("Must specify --pull") @@ -194,7 +228,11 @@ class PullPkg(object): Logger.normal("Using release '%s' for '%s'", codename, release) release = codename - d = Distribution(distro) + if distro == DISTRO_PPA: + # PPAs are part of Ubuntu distribution + d = Distribution(DISTRO_UBUNTU) + else: + d = Distribution(distro) # let SeriesNotFoundException flow up d.getSeries(release) @@ -233,6 +271,7 @@ class PullPkg(object): assert 'arch' in options assert 'package' in options # type string, optional + assert 'ppa' in options assert 'release' in options assert 'version' in options # type list of strings, optional @@ -255,6 +294,14 @@ class PullPkg(object): params['dscfile'] = params['package'] params.pop('package') + if options['ppa']: + if options['ppa'].startswith('ppa:'): + params['ppa'] = options['ppa'][4:] + else: + params['ppa'] = options['ppa'] + elif distro == DISTRO_PPA: + raise ValueError('Must specify PPA to pull from') + mirrors = [] if options['mirror']: mirrors.append(options['mirror']) @@ -273,6 +320,9 @@ class PullPkg(object): def pull(self, args=None): """Pull (download) specified package file(s)""" + # pull-ppa-* may need conversion from ppa:USER/NAME to --ppa USER/NAME + args = self.parse_ppa_args(args) + options = vars(self.argparser.parse_args(args)) assert 'verbose' in options