- buildd: add a --batch mode for batch retrying/rescoring of packages

- lpapiwrapper.py: add the needed methods and classes for this
- udtexceptions.py: rename PocketDoesNotExist to PocketDoesNotExistException
  to be in line with the naming of the other exceptions
This commit is contained in:
Michael Bienia 2009-07-29 23:07:22 +02:00
commit 52f9b37eaa
3 changed files with 426 additions and 178 deletions

194
buildd
View File

@ -6,6 +6,7 @@
# Authors:
# - Martin Pitt <martin.pitt@canonical.com>
# - Jonathan Davies <jpds@ubuntu.com>
# - Michael Bienia <geser@ubuntu.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -34,8 +35,8 @@ usage += "Where operation may be one of: rescore, retry, or status.\n"
usage += "Only Launchpad Buildd Admins may rescore package builds."
# Valid architectures.
validArchs = ["armel", "amd64", "hppa", "i386",
"ia64", "lpia", "powerpc", "sparc"]
valid_archs = set(["armel", "amd64", "hppa", "i386",
"ia64", "lpia", "powerpc", "sparc"])
# Prepare our option parser.
optParser = OptionParser(usage)
@ -44,102 +45,121 @@ optParser = OptionParser(usage)
retryRescoreOptions = OptionGroup(optParser, "Retry and rescore options",
"These options may only be used with the 'retry' and 'rescore' operations.")
retryRescoreOptions.add_option("-a", "--arch", type = "string",
action = "store", dest = "architecture",
action = "append", dest = "architecture",
help = "Rebuild or rescore a specific architecture. " \
"Valid architectures include: " \
"%s." % ", ".join(validArchs))
"%s." % ", ".join(valid_archs))
# Batch processing options
batch_options = OptionGroup(
optParser, "Batch processing",
"These options and parameter ordering is only available in --batch mode.\n"
"Usage: buildd --batch [options] <package>...")
batch_options.add_option(
'--batch', action = 'store_true', dest = 'batch', default = False,
help = 'Enable batch mode')
batch_options.add_option(
'--series', action = 'store', dest = 'series', type = 'string',
help = 'Selects the Ubuntu series to operate on (default: current development series)')
batch_options.add_option(
'--retry', action = 'store_true', dest = 'retry', default = False,
help = 'Retry builds (give-back).')
batch_options.add_option(
'--rescore', action = 'store', dest = 'priority', type = 'int',
help = 'Rescore builds to <priority>.')
batch_options.add_option(
'--arch2', action = 'append', dest = 'architecture', type = 'string',
help = "Affect only 'architecture' (can be used several times). "
"Valid architectures are: %s." % ', '.join(valid_archs))
# Add the retry options to the main group.
optParser.add_option_group(retryRescoreOptions)
# Add the batch mode to the main group.
optParser.add_option_group(batch_options)
# Parse our options.
(options, args) = optParser.parse_args()
# 'help' called by itself - show our help.
try:
if args[0].lower() in ("help") and len(args) == 1:
if not len(args):
optParser.print_help()
sys.exit(0)
except IndexError:
optParser.print_help()
sys.exit(0)
sys.exit(1)
# Check we have the correct number of arguments.
if len(args) < 3:
if not options.batch:
# Check we have the correct number of arguments.
if len(args) < 3:
optParser.error("Incorrect number of arguments.")
try:
try:
package = str(args[0]).lower()
release = str(args[1]).lower()
op = str(args[2]).lower()
except IndexError:
except IndexError:
optParser.print_help()
sys.exit(0)
sys.exit(1)
# Check our operation.
if op not in ("rescore", "retry", "status"):
# Check our operation.
if op not in ("rescore", "retry", "status"):
print >> sys.stderr, "Invalid operation: %s." % op
sys.exit(1)
# If the user has specified an architecture to build, we only wish to rebuild it
# and nothing else.
if op not in ("retry", 'rescore') and options.architecture:
print >> sys.stderr, "Operation %s does not use the --arch option." % op
sys.exit(1)
elif op in ("retry", "rescore") and options.architecture:
if options.architecture not in validArchs:
print >> sys.stderr, "Invalid architecture specified: %s." % options.architecture
# If the user has specified an architecture to build, we only wish to rebuild it
# and nothing else.
if options.architecture:
if options.architecture[0] not in valid_archs:
print >> sys.stderr, "Invalid architecture specified: %s." % options.architecture[0]
sys.exit(1)
else:
oneArch = True
else:
else:
oneArch = False
# split release and pocket
if '-' in release:
# split release and pocket
if '-' in release:
(release, pocket) = release.split('-')
else:
else:
pocket = 'Release'
pocket = pocket.capitalize()
if pocket not in ('Release', 'Security', 'Updates', 'Proposed', 'Backports'):
pocket = pocket.capitalize()
if pocket not in ('Release', 'Security', 'Updates', 'Proposed', 'Backports'):
print 'Unknown pocket: %s' % pocket
sys.exit(1)
# Get an instance of the LP API wrapper
lpapiwrapper = LpApiWrapper()
# Get list of published sources for package in question.
try:
sources = lpapiwrapper.getUbuntuSourcePackage(package, release, pocket)
except (SeriesNotFoundException, PackageNotFoundException), e:
# Get the ubuntu archive
ubuntu_archive = LpApiWrapper.getUbuntuDistribution().getArchive()
# Get list of published sources for package in question.
try:
sources = ubuntu_archive.getSourcePackage(package, release, pocket)
except (SeriesNotFoundException, PackageNotFoundException), e:
print e
sys.exit(1)
# Get list of builds for that package.
builds = sources.getBuilds()
# Get list of builds for that package.
builds = sources.getBuilds()
# Find out the version and component in given release.
version = sources.getVersion()
component = sources.getComponent()
# Find out the version and component in given release.
version = sources.getVersion()
component = sources.getComponent()
# Operations that are remaining may only be done by Ubuntu developers (retry)
# or buildd admins (rescore). Check if the proper permissions are in place.
if op == "rescore": necessaryPrivs = lpapiwrapper.getMe().isLpTeamMember('launchpad-buildd-admins')
if op == "retry": necessaryPrivs = lpapiwrapper.canUploadPackage(package, release)
# Operations that are remaining may only be done by Ubuntu developers (retry)
# or buildd admins (rescore). Check if the proper permissions are in place.
me = LpApiWrapper.getMe()
if op == "rescore": necessaryPrivs = me.isLpTeamMember('launchpad-buildd-admins')
if op == "retry": necessaryPrivs = me.canUploadPackage(
ubuntu_archive, sources.getPackageName(), sources.getComponent())
if op in ('rescore', 'retry') and not necessaryPrivs:
if op in ('rescore', 'retry') and not necessaryPrivs:
print >> sys.stderr, "You cannot perform the %s operation on a %s package " \
"as you do not have the permissions to do this action." % (op, component)
sys.exit(1)
# Output details.
print "The source version for '%s' in %s (%s) is at %s." % (package,
# Output details.
print "The source version for '%s' in %s (%s) is at %s." % (package,
release.capitalize(), component, version)
print "Current build status for this package:"
print "Current build status for this package:"
# Output list of arches for package and their status.
done = False
for build in builds:
if oneArch and build.arch_tag != options.architecture:
# Output list of arches for package and their status.
done = False
for build in builds:
if oneArch and build.arch_tag != options.architecture[0]:
# Skip this architecture.
continue
@ -161,9 +181,65 @@ for build in builds:
print 'Cannot retry build on %s.' % build.arch_tag
# We are done
if done: sys.exit(0)
# We are done
if done: sys.exit(0)
print "No builds for '%s' found in the %s release - it may have been " \
print "No builds for '%s' found in the %s release - it may have been " \
"built in a former release." % (package, release.capitalize())
sys.exit(0)
sys.exit(0)
# Batch mode
if not options.architecture:
# no specific architectures specified, assume all valid ones
archs = valid_archs
else:
archs = set(options.architecture)
# filter out duplicate and invalid architectures
archs.intersection_update(valid_archs)
release = options.series # if None it falls back to the current development series
pocket = 'Release'
if release and '-' in release:
# split release and pocket
(release, pocket) = options.series.split('-')
pocket = pocket.capitalize()
if pocket not in ('Release', 'Security', 'Updates', 'Proposed', 'Backports'):
print 'Unknown pocket: %s' % pocket
sys.exit(1)
ubuntu_archive = LpApiWrapper.getUbuntuDistribution().getArchive()
me = LpApiWrapper.getMe()
# Check permisions (part 1): Rescoring can only be done by buildd admins
can_rescore = options.priority and me.isLpTeamMember('launchpad-buildd-admins') or False
if options.priority and not can_rescore:
print >> sys.stderr, "You don't have the permissions to rescore builds. Ignoring your rescore request."
for pkg in args:
try:
pkg = ubuntu_archive.getSourcePackage(pkg, release, pocket)
except SeriesNotFoundException, e:
print e
sys.exit(1)
except PackageNotFoundException, e:
print e
continue
# Check permissions (part 2): check upload permissions for the source package
can_retry = options.retry and me.canUploadPackage(ubuntu_archive, pkg.getPackageName(), pkg.getComponent())
if options.retry and not can_retry:
print >> sys.stderr, "You don't have the permissions to retry the build of '%s'. Ignoring your request." % pkg.getPackageName()
print "The source version for '%s' in '%s' (%s) is: %s" % (
pkg.getPackageName(), release, pocket, pkg.getVersion())
print pkg.getBuildStates(archs)
if can_retry:
print pkg.retryBuilds(archs)
if options.priority and can_rescore:
print pkg.rescoreBuilds(archs, options.priority)
print ''

View File

@ -27,7 +27,7 @@
import libsupport
from launchpadlib.errors import HTTPError
from launchpadlib.resource import Entry
from udtexceptions import PackageNotFoundException, SeriesNotFoundException, PocketDoesNotExist
from udtexceptions import *
__all__ = ['LpApiWrapper']
@ -58,9 +58,6 @@ class LpApiWrapper(object):
ubuntu-dev-tools.
'''
_me = None
_src_pkg = dict()
_upload_comp = dict()
_upload_pkg = dict()
@classmethod
def getMe(cls):
@ -86,76 +83,36 @@ class LpApiWrapper(object):
Returns a wrapped LP representation of the source package.
If the package does not exist: raise PackageNotFoundException
'''
# Check if pocket has a valid value
if pocket not in ('Release', 'Security', 'Updates', 'Proposed', 'Backports'):
raise PocketDoesNotExist("Pocket '%s' does not exist." % pocket)
# Check if we have already a LP representation of an Ubuntu series or not
if not isinstance(series, DistroSeries):
series = cls.getUbuntuDistribution().getSeries(series)
if (name, series, pocket) not in cls._src_pkg:
try:
srcpkg = cls.getUbuntuDistribution().getMainArchive().getPublishedSources(
source_name = name, distro_series = series(), pocket = pocket,
status = 'Published', exact_match = True)[0]
cls._src_pkg[(name, series, pocket)] = SourcePackage(srcpkg)
except IndexError:
if pocket == 'Release':
msg = "The package '%s' does not exist in the Ubuntu main archive in '%s'" % \
(name, series.name)
else:
msg = "The package '%s' does not exist in the Ubuntu main archive in '%s-%s'" % \
(name, series.name, pocket.lower())
raise PackageNotFoundException(msg)
return cls._src_pkg[(name, series, pocket)]
return cls.getUbuntuDistribution().getArchive().getSourcePackage(name, series, pocket)
@classmethod
def canUploadPackage(cls, package, series = None):
def canUploadPackage(cls, srcpkg, series = None):
'''
Check if the currently authenticated LP user has upload rights
for package either through component upload rights or
per-package upload rights.
'package' can either be a wrapped LP representation of a source
package or a string and an Ubuntu series. If 'package' doesn't
exist yet in Ubuntu assume 'universe' for component.
'package' can either be a SourcePackage object or a string and
an Ubuntu series. If 'package' doesn't exist yet in Ubuntu
assume 'universe' for component.
'''
component = 'universe'
archive = cls.getUbuntuDistribution().getArchive()
if isinstance(package, SourcePackage):
component = package.getComponent()
package = package.getPackageName()
if isinstance(srcpkg, SourcePackage):
package = srcpkg.getPackageName()
component = srcpkg.getComponent()
else:
if not series:
# Fall-back to current Ubuntu development series
series = cls.getUbuntuDistribution().getDevelopmentSeries()
try:
component = cls.getUbuntuSourcePackage(package, series).getComponent()
srcpkg = archive.getSourcePackage(srcpkg, series)
package = srcpkg.getPackageName()
component = srcpkg.getComponent()
except PackageNotFoundException:
# Probably a new package, assume "universe" as component
component = 'universe'
package = None
if component not in cls._upload_comp and package not in cls._upload_pkg:
me = cls.getMe()
archive = cls.getUbuntuDistribution().getMainArchive()
for perm in archive.getPermissionsForPerson(person = me()):
if perm.permission != 'Archive Upload Rights':
continue
if perm.component_name == component:
cls._upload_comp[component] = True
return True
if perm.source_package_name == package:
cls._upload_pkg[package] = True
return True
return False
elif component in cls._upload_comp:
return cls._upload_comp[component]
else:
return cls._upload_pkg[package]
return cls.getMe().canUploadPackage(archive, package, component)
# TODO: check if this is still needed after ArchiveReorg (or at all)
@classmethod
@ -164,11 +121,11 @@ class LpApiWrapper(object):
Check if the user has PerPackageUpload rights for package.
'''
if isinstance(package, SourcePackage):
pkg = package.getPackageName()
else:
pkg = package
package = package.getPackageName()
return cls.canUploadPackage(package, series) and pkg in cls._upload_pkg
archive = cls.getUbuntuDistribution().getArchive()
return cls.getMe().canUploadPackage(archive, package, None)
class MetaWrapper(type):
@ -243,9 +200,11 @@ class Distribution(BaseWrapper):
resource_type = 'https://api.edge.launchpad.net/beta/#distribution'
def __init__(self, *args):
# Don't share _series between different Distributions
# Don't share _series and _archives between different Distributions
if '_series' not in self.__dict__:
self._series = dict()
if '_archives' not in self.__dict__:
self._archives = dict()
def cache(self):
self._cache[self.name] = self
@ -262,13 +221,31 @@ class Distribution(BaseWrapper):
cached = Distribution(Launchpad.distributions[dist])
return cached
def getMainArchive(self):
def getArchive(self, archive = None):
'''
Returns the LP representation for the Ubuntu main archive.
Returns an Archive object for the requested archive.
Raises a ArchiveNotFoundException if the archive doesn't exist.
If 'archive' is None, return the main archive.
'''
if not '_archive' in self.__dict__:
self._archive = Archive(self.main_archive_link)
return self._archive
if archive:
res = self._archives.get(archive)
if not res:
for a in self.archives:
if a.name == archive:
res = Archive(a)
self._archives[res.name] = res
break
if res:
return res
else:
raise ArchiveNotFoundException("The Archive '%s' doesn't exist in %s" % (archive, self.display_name))
else:
if not '_main_archive' in self.__dict__:
self._main_archive = Archive(self.main_archive_link)
return self._main_archive
def getSeries(self, name_or_version):
'''
@ -311,6 +288,59 @@ class Archive(BaseWrapper):
'''
resource_type = 'https://api.edge.launchpad.net/beta/#archive'
def __init__(self, *args):
# Don't share _srcpkgs between different Archives
if '_srcpkgs' not in self.__dict__:
self._srcpkgs = dict()
def getSourcePackage(self, name, series = None, pocket = 'Release'):
'''
Returns a SourcePackage object for the most recent source package
in the distribution 'dist', series and pocket.
series defaults to the current development series if not specified.
If the requested source package doesn't exist a
PackageNotFoundException is raised.
'''
# Check if pocket has a valid value
if pocket not in ('Release', 'Security', 'Updates', 'Proposed', 'Backports'):
raise PocketDoesNotExistException("Pocket '%s' does not exist." % pocket)
dist = Distribution(self.distribution_link)
# Check if series is already a DistoSeries object or not
if not isinstance(series, DistroSeries):
if series:
series = dist.getSeries(series)
else:
series = dist.getDevelopmentSeries()
# NOTE:
# For Debian all source publication are in the state 'Pending' so filter on this
# instead of 'Published'. As the result is sorted also by date the first result
# will be the most recent one (i.e. the one we are interested in).
if dist.name in ('debian',):
state = 'Pending'
else:
state = 'Published'
if (name, series.name, pocket) not in self._srcpkgs:
try:
srcpkg = self.getPublishedSources(
source_name = name, distro_series = series(), pocket = pocket,
status = state, exact_match = True)[0]
self._srcpkgs[(name, series.name, pocket)] = SourcePackage(srcpkg)
except IndexError:
if pocket == 'Release':
msg = "The package '%s' does not exist in the %s %s archive in '%s'" % \
(name, dist.display_name, self.name, series.name)
else:
msg = "The package '%s' does not exist in the %s %s archive in '%s-%s'" % \
(name, dist.display_name, self.name, series.name, pocket.lower())
raise PackageNotFoundException(msg)
return self._srcpkgs[(name, series.name, pocket)]
class SourcePackage(BaseWrapper):
'''
@ -318,6 +348,11 @@ class SourcePackage(BaseWrapper):
'''
resource_type = 'https://api.edge.launchpad.net/beta/#source_package_publishing_history'
def __init__(self, *args):
# Don't share _builds between different SourcePackages
if '_builds' not in self.__dict__:
self._builds = dict()
def getPackageName(self):
'''
Returns the source package name.
@ -336,6 +371,57 @@ class SourcePackage(BaseWrapper):
'''
return self._lpobject.component_name
def _fetch_builds(self):
'''Populate self._builds with the build records.'''
builds = self.getBuilds()
for build in builds:
self._builds[build.arch_tag] = Build(build)
def getBuildStates(self, archs):
res = list()
if not self._builds:
self._fetch_builds()
for arch in archs:
build = self._builds.get(arch)
if build:
res.append(' %s' % build)
return "Build state(s) for '%s':\n%s" % (
self.getPackageName(), '\n'.join(res))
def rescoreBuilds(self, archs, score):
res = list()
if not self._builds:
self._fetch_builds()
for arch in archs:
build = self._builds.get(arch)
if build:
if build.rescore(score):
res.append(' %s: done' % arch)
else:
res.append(' %s: failed' % arch)
return "Rescoring builds of '%s' to %i:\n%s" % (
self.getPackageName(), score, '\n'.join(res))
def retryBuilds(self, archs):
res = list()
if not self._builds:
self._fetch_builds()
for arch in archs:
build = self._builds.get(arch)
if build:
if build.retry():
res.append(' %s: done' % arch)
else:
res.append(' %s: failed' % arch)
return "Retrying builds of '%s':\n%s" % (
self.getPackageName(), '\n'.join(res))
class PersonTeam(BaseWrapper):
'''
@ -343,8 +429,15 @@ class PersonTeam(BaseWrapper):
'''
resource_type = ('https://api.edge.launchpad.net/beta/#person', 'https://api.edge.launchpad.net/beta/#team')
def __init__(self, *args):
# Don't share _upload_{pkg,comp} between different PersonTeams
if '_upload_pkg' not in self.__dict__:
self._upload_pkg = dict()
if '_upload_comp' not in self.__dict__:
self._upload_comp = dict()
def __str__(self):
return '%s (%s)' % (self.display_name, self.name)
return u'%s (%s)' % (self.display_name, self.name)
def cache(self):
self._cache[self.name] = self
@ -368,3 +461,78 @@ class PersonTeam(BaseWrapper):
Returns True if the user is a member of the team otherwise False.
'''
return any(t.name == team for t in self.super_teams)
def canUploadPackage(self, archive, package, component):
'''
Check if the person or team has upload rights for the source package
to the specified 'archive' either through component upload
rights or per-package upload rights.
Either a source package name or a component has the specified.
'archive' has to be a Archive object.
'''
if not isinstance(archive, Archive):
raise TypeError("'%r' is not an Archive object." % archive)
if not isinstance(package, (str, None)):
raise TypeError('A source package name expected.')
if not isinstance(component, (str, None)):
raise TypeError('A component name expected.')
if not package and not component:
raise ValueError('Either a source package name or a component has to be specified.')
upload_comp = self._upload_comp.get((archive, component))
upload_pkg = self._upload_pkg.get((archive, package))
if upload_comp == None and upload_pkg == None:
for perm in archive.getPermissionsForPerson(person = self()):
if perm.permission != 'Archive Upload Rights':
continue
if component and perm.component_name == component:
self._upload_comp[(archive, component)] = True
return True
if package and perm.source_package_name == package:
self._upload_pkg[(archive, package)] = True
return True
# don't have upload rights
if package:
self._upload_pkg[(archive, package)] = False
if component:
self._upload_comp[(archive, component)] = False
return False
else:
return upload_comp or upload_pkg
# TODO: check if this is still needed after ArchiveReorg (or at all)
def isPerPackageUploader(self, archive, package):
'''
Check if the user has PerPackageUpload rights for package.
'''
if isinstance(package, SourcePackage):
pkg = package.getPackageName()
comp = package.getComponent()
else:
pkg = package
compon
return self.canUploadPackage(archive, pkg, None)
class Build(BaseWrapper):
'''
Wrapper class around a build object.
'''
resource_type = 'https://api.edge.launchpad.net/beta/#build'
def __str__(self):
return u'%s: %s' % (self.arch_tag, self.buildstate)
def rescore(self, score):
if self.can_be_rescored:
self().rescore(score = score)
return True
return False
def retry(self):
if self.can_be_retried:
self().retry()
return True
return False

View File

@ -6,6 +6,10 @@ class SeriesNotFoundException(BaseException):
""" Thrown when a distroseries is not found """
pass
class PocketDoesNotExist(BaseException):
class PocketDoesNotExistException(BaseException):
""" Thrown when a invalid pocket is passed """
pass
class ArchiveNotFoundException(BaseException):
""" Thrown when an archive for a distibution is not found """
pass