#!/usr/bin/python
# Copyright (C) 2011, 2012 Canonical Ltd.
# Author: Martin Pitt <martin.pitt@canonical.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
'''Release a proposed stable release update.
Copy packages from -proposed to -updates, and optionally to -security and the
development release.
USAGE:
sru-release [-s] [-d] <release> <package> [<package> ...]
'''
from __future__ import print_function
from collections import defaultdict
from functools import partial
import optparse
import sys
import unittest
try:
from urllib.request import urlopen
except ImportError:
from urllib import urlopen
from launchpadlib.launchpad import Launchpad
# Each entry in this list is a list of source packages that are known
# to have inter-dependencies and must be released simultaneously.
# If possible, each list should be ordered such that earlier
# entries could be released slightly before subsequent entries.
RELEASE_TOGETHER_PACKAGE_GROUPS = [
['linux-hwe', 'linux-meta-hwe'],
['linux', 'linux-meta'],
['grub2', 'grub2-signed'],
['shim', 'shim-signed'],
]
MISSING_PACKAGES_FROM_GROUP = (
"The set of packages requested for release are listed as dangerous \n"
"to release without also releasing the following at the same time:\n"
" {missing}\n\n"
"For more information, see:\n"
" https://lists.ubuntu.com/archives/ubuntu-devel/2018-June/040380.html\n\n"
"To ignore this message, pass '--skip-package-group-check'.")
def check_package_sets(packages):
"""Return a re-ordered list of packages respecting the PACKAGE_SETS
defined above. If any packages are missing, raise error."""
# pkg2group is a dict where each key is a pkg in a group and value is the
# complete group.
pkg2group = {}
for pgroup in RELEASE_TOGETHER_PACKAGE_GROUPS:
for pkg in pgroup:
if pkg in pkg2group:
raise RuntimeError(
"Overlapping package groups. '%s' is in '%s' and '%s'." %
(pkg, pgroup, pkg2group[pkg]))
pkg2group[pkg] = pgroup
seen = set()
new_pkgs = []
for pkg in packages:
if pkg not in pkg2group:
add = [pkg]
else:
add = list(pkg2group[pkg])
new_pkgs.extend([a for a in add if a not in seen])
seen.update(add)
orig = set(packages)
new = set(new_pkgs)
if orig != new:
raise ValueError(
MISSING_PACKAGES_FROM_GROUP.format(
missing=' '.join(new.difference(orig))))
return new_pkgs
class CheckPackageSets(unittest.TestCase):
def test_expected_linux_order_fixed(self):
self.assertEqual(
['pkg1', 'linux', 'linux-meta', 'pkg2'],
check_package_sets(['pkg1', 'linux-meta', 'linux', 'pkg2']))
def test_raises_value_error_on_missing(self):
self.assertRaises(
ValueError, check_package_sets, ['pkg1', 'linux'])
def test_single_item_with_missing(self):
self.assertRaises(
ValueError, check_package_sets, ['linux'])
def test_single_item_without_missing(self):
self.assertEqual(
check_package_sets(['pkg1']), ['pkg1'])
def test_multiple_package_groups(self):
"""Just make sure that having multiple groups listed still errors."""
self.assertRaises(
ValueError, check_package_sets, ['pkg1', 'linux', 'grub2'])
def match_srubugs(options, changesfileurl):
'''match between bugs with verification- tag and bugs in changesfile'''
bugs = []
if changesfileurl is None:
return bugs
# Load changesfile
changelog = urlopen(changesfileurl)
bugnums = []
for l in changelog:
if l.startswith('Launchpad-Bugs-Fixed: '):
bugnums = l.split()[1:]
break
for b in bugnums:
if b in options.exclude_bug:
continue
try:
bugs.append(launchpad.bugs[int(b)])
except:
print('%s: bug %s does not exist or is not accessible' %
(changesfileurl, b))
return bugs
def update_sru_bug(bug, pkg):
'''Unsubscribe SRU team and comment on bug re: how to report regressions'''
m_subjects = [m.subject for m in bug.messages]
if 'Update Released' in m_subjects:
print('LP: #%s was not commented on' % bug.id)
return
sru_team = launchpad.people['ubuntu-sru']
bug.unsubscribe(person=sru_team)
text = ("The verification of the Stable Release Update for %s has "
"completed successfully and the package has now been released "
"to -updates. Subsequently, the Ubuntu Stable Release Updates "
"Team is being unsubscribed and will not receive messages "
"about this bug report. In the event that you encounter "
"a regression using the package from -updates please report "
"a new bug using ubuntu-bug and tag the bug report "
"regression-update so we can easily find any regressions." % pkg)
bug.newMessage(subject="Update Released", content=text)
bug.lp_save()
def get_versions(options, sourcename):
'''Get current package versions.
If options.pattern is True, return all versions for package names
matching options.pattern.
If options.pattern is False, only return one result.
Return map pkgname -> {'release': version, 'updates': version,
'proposed': version, 'changesfile': url_of_proposed_changes,
'published': proposed_date}
'''
versions = defaultdict(dict)
if options.esm:
pocket = 'Release'
else:
pocket = 'Proposed'
matches = src_archive.getPublishedSources(
source_name=sourcename, exact_match=not options.pattern,
status='Published', pocket=pocket, distro_series=series)
for match in matches:
# versions in all pockets
for pub in src_archive.getPublishedSources(
source_name=match.source_package_name, exact_match=True,
status='Published', distro_series=series):
key = pub.pocket.lower()
# special case for ESM ppas, which don't have pockets but need
# to be treated as -proposed
if options.esm and key == 'release':
key = 'proposed'
versions[pub.source_package_name][key] = (
pub.source_package_version)
if pocket in pub.pocket:
versions[pub.source_package_name]['changesfile'] = (
pub.changesFileUrl())
# When the destination archive differs from the source scan that too.
if dst_archive != src_archive:
for pub in dst_archive.getPublishedSources(
source_name=match.source_package_name, exact_match=True,
status='Published', distro_series=series):
key = 'security' # pub.pocket.lower()
versions[pub.source_package_name][key] = (
pub.source_package_version)
if pocket in pub.pocket:
versions[pub.source_package_name]['changesfile'] = (
pub.changesFileUrl())
# devel version
if devel_series:
for pub in src_archive.getPublishedSources(
source_name=match.source_package_name, exact_match=True,
status='Published', distro_series=devel_series):
if pub.pocket in ('Release', 'Proposed'):
versions[pub.source_package_name]['devel'] = (
pub.source_package_version)
else:
versions[match.source_package_name]['devel'] = None
return versions
def release_package(options, package):
'''Release a package.'''
pkg_versions_map = get_versions(options, package)
if not pkg_versions_map:
message = 'ERROR: No such package, ' + package + ', in -proposed, aborting\n'
sys.stderr.write(message)
sys.exit(1)
for pkg, versions in pkg_versions_map.iteritems():
print('--- Releasing %s ---' % pkg)
print('Proposed: %s' % versions['proposed'])
if 'security' in versions:
print('Security: %s' % versions['security'])
if 'updates' in versions:
print('Updates: %s' % versions['updates'])
else:
print('Release: %s' % versions.get('release'))
if options.devel and 'devel' in versions:
print('Devel: %s' % versions['devel'])
copy = partial(
dst_archive.copyPackage, from_archive=src_archive,
include_binaries=True, source_name=pkg,
version=versions['proposed'], auto_approve=True)
if options.devel:
if ('devel' not in versions or
versions['devel'] in (
versions.get('updates', 'notexisting'),
versions['release'])):
if not options.no_act:
copy(to_pocket='Proposed', to_series=devel_series.name)
print('Version in %s matches development series, '
'copied to %s-proposed' % (release, devel_series.name))
else:
print('ERROR: Version in %s does not match development '
'series, not copying' % release)
if options.no_act:
if options.release:
print('Would copy to %s' % release)
else:
print('Would copy to %s-updates' % release)
else:
if options.release:
# -proposed -> release
copy(to_pocket='Release', to_series=release)
print('Copied to %s' % release)
else:
# -proposed -> -updates
# only phasing updates for >=raring to start
if (release not in ('lucid', 'precise') and
package != 'linux' and
not package.startswith('linux-') and
not options.security):
copy(to_pocket='Updates', to_series=release,
phased_update_percentage=options.percentage)
else:
copy(to_pocket='Updates', to_series=release)
print('Copied to %s-updates' % release)
if not options.no_bugs:
sru_bugs = match_srubugs(options, versions['changesfile'])
tag = 'verification-needed-%s' % release
for sru_bug in sru_bugs:
if tag not in sru_bug.tags:
update_sru_bug(sru_bug, pkg)
# -proposed -> -security
if options.security:
if options.no_act:
print('Would copy to %s-security' % release)
else:
copy(to_pocket='Security', to_series=release)
print('Copied to %s-security' % release)
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == "run-tests":
sys.exit(unittest.main(argv=[sys.argv[0]] + sys.argv[2:]))
parser = optparse.OptionParser(
usage='usage: %prog [options] <release> <package> [<package> ...]')
parser.add_option(
'-l', '--launchpad', dest='launchpad_instance', default='production')
parser.add_option(
'--security', action='store_true', default=False,
help='Additionally copy to -security pocket')
parser.add_option(
'-d', '--devel', action='store_true', default=False,
help='Additionally copy to development release (only works if that '
'has the same version as <release>)')
parser.add_option(
'-r', '--release', action='store_true', default=False,
help='Copy to release pocket instead of -updates (useful for staging '
'uploads in development release)')
parser.add_option(
"-z", "--percentage", type="int", default=10,
metavar="PERCENTAGE", help="set phased update percentage")
parser.add_option(
'-n', '--no-act', action='store_true', default=False,
help='Only perform checks, but do not actually copy packages')
parser.add_option(
'-p', '--pattern', action='store_true', default=False,
help='Treat package names as patterns, not exact matches')
parser.add_option(
'--no-bugs', action='store_true', default=False,
help='Do not act on any bugs (helpful to avoid races).')
parser.add_option(
'--exclude-bug', action='append', default=[], metavar='BUG',
help='Do not update BUG.')
parser.add_option(
'-E', '--esm', action='store_true', default=False,
help='Copy from the kernel ESM proposed PPA to the ESM publication PPA')
parser.add_option(
'--skip-package-group-check', action='store_true', default=False,
help=('Skip the package set checks that require some packages '
'be released together'))
options, args = parser.parse_args()
if len(args) < 2:
parser.error(
'You must specify a release and source package(s), see --help')
if options.release and (options.security or options.devel):
parser.error('-r and -s/-d are mutually exclusive, see --help')
release = args.pop(0)
packages = args
if not options.skip_package_group_check:
try:
packages = check_package_sets(packages)
except ValueError as e:
sys.stderr.write(e.args[0] + '\n')
sys.exit(1)
launchpad = Launchpad.login_with(
'ubuntu-archive-tools', options.launchpad_instance, version='devel')
ubuntu = launchpad.distributions['ubuntu']
series = ubuntu.getSeries(name_or_version=release)
devel_series = ubuntu.current_series
if not devel_series:
sys.stderr.write(
'WARNING: No current development series, -d will not work\n')
devel_series = None
if release == 'precise':
sys.stdout.write(
'Called for precise; assuming kernel ESM publication\n')
options.esm = True
if options.esm:
# --security is meaningless for ESM everything is a security update.
options.security = False
options.release = True
src_archive = launchpad.archives.getByReference(
reference='~canonical-kernel-esm/ubuntu/proposed')
dst_archive = launchpad.archives.getByReference(
reference='~ubuntu-esm/ubuntu/esm')
else:
src_archive = dst_archive = ubuntu.getArchive(name='primary')
for package in packages:
release_package(options, package)