You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
529 lines
21 KiB
529 lines
21 KiB
#!/usr/bin/python3
|
|
|
|
# Copyright (C) 2011, 2012 Canonical Ltd.
|
|
# Author: Martin Pitt <martin.pitt@canonical.com>
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 3 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
'''Release a proposed stable release update.
|
|
|
|
Copy packages from -proposed to -updates, and optionally to -security and the
|
|
development release.
|
|
|
|
USAGE:
|
|
sru-release [-s] [-d] <release> <package> [<package> ...]
|
|
'''
|
|
|
|
from __future__ import print_function
|
|
|
|
from collections import defaultdict
|
|
from functools import partial
|
|
import datetime
|
|
import optparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import unittest
|
|
|
|
from six.moves.urllib.request import urlopen
|
|
from io import TextIOWrapper
|
|
|
|
from launchpadlib.launchpad import Launchpad
|
|
|
|
from kernel_series import KernelSeries
|
|
|
|
|
|
# Each entry in this list is a list of source packages that are known
|
|
# to have inter-dependencies and must be released simultaneously.
|
|
# If possible, each list should be ordered such that earlier
|
|
# entries could be released slightly before subsequent entries.
|
|
RELEASE_TOGETHER_PACKAGE_GROUPS = [
|
|
['linux-hwe', 'linux-signed-hwe', 'linux-meta-hwe'],
|
|
['linux', 'linux-signed', 'linux-meta'],
|
|
['grub2', 'grub2-signed'],
|
|
['shim', 'shim-signed'],
|
|
['libreoffice', 'libreoffice-l10n'],
|
|
]
|
|
|
|
MISSING_PACKAGES_FROM_GROUP = (
|
|
"The set of packages requested for release are listed as dangerous \n"
|
|
"to release without also releasing the following at the same time:\n"
|
|
" {missing}\n\n"
|
|
"For more information, see:\n"
|
|
" https://lists.ubuntu.com/archives/ubuntu-devel/2018-June/040380.html\n\n"
|
|
"To ignore this message, pass '--skip-package-group-check'.")
|
|
|
|
BZR_HINT_BRANCH = "lp:~ubuntu-sru/britney/hints-ubuntu-%s"
|
|
|
|
|
|
def check_package_sets(packages):
|
|
"""Return a re-ordered list of packages respecting the PACKAGE_SETS
|
|
defined above. If any packages are missing, raise error."""
|
|
|
|
# pkg2group is a dict where each key is a pkg in a group and value is the
|
|
# complete group.
|
|
pkg2group = {}
|
|
for pgroup in RELEASE_TOGETHER_PACKAGE_GROUPS:
|
|
for pkg in pgroup:
|
|
if pkg in pkg2group:
|
|
raise RuntimeError(
|
|
"Overlapping package groups. '%s' is in '%s' and '%s'." %
|
|
(pkg, pgroup, pkg2group[pkg]))
|
|
pkg2group[pkg] = pgroup
|
|
|
|
seen = set()
|
|
new_pkgs = []
|
|
for pkg in packages:
|
|
if pkg not in pkg2group:
|
|
add = [pkg]
|
|
else:
|
|
add = list(pkg2group[pkg])
|
|
new_pkgs.extend([a for a in add if a not in seen])
|
|
seen.update(add)
|
|
|
|
orig = set(packages)
|
|
new = set(new_pkgs)
|
|
if orig != new:
|
|
raise ValueError(
|
|
MISSING_PACKAGES_FROM_GROUP.format(
|
|
missing=' '.join(new.difference(orig))))
|
|
return new_pkgs
|
|
|
|
|
|
class CheckPackageSets(unittest.TestCase):
|
|
def test_expected_linux_order_fixed(self):
|
|
self.assertEqual(
|
|
['pkg1', 'linux', 'linux-signed', 'linux-meta', 'pkg2'],
|
|
check_package_sets(['pkg1', 'linux-meta', 'linux', 'linux-signed', 'pkg2']))
|
|
|
|
def test_raises_value_error_on_missing(self):
|
|
self.assertRaises(
|
|
ValueError, check_package_sets, ['pkg1', 'linux'])
|
|
|
|
def test_single_item_with_missing(self):
|
|
self.assertRaises(
|
|
ValueError, check_package_sets, ['linux'])
|
|
|
|
def test_single_item_without_missing(self):
|
|
self.assertEqual(
|
|
check_package_sets(['pkg1']), ['pkg1'])
|
|
|
|
def test_multiple_package_groups(self):
|
|
"""Just make sure that having multiple groups listed still errors."""
|
|
self.assertRaises(
|
|
ValueError, check_package_sets, ['pkg1', 'linux', 'grub2'])
|
|
|
|
|
|
def match_srubugs(options, changesfileurl):
|
|
'''match between bugs with verification- tag and bugs in changesfile'''
|
|
|
|
bugs = []
|
|
|
|
if changesfileurl is None:
|
|
return bugs
|
|
|
|
# Load changesfile
|
|
changelog = TextIOWrapper(urlopen(changesfileurl), encoding='utf-8')
|
|
bugnums = []
|
|
for l in changelog:
|
|
if l.startswith('Launchpad-Bugs-Fixed: '):
|
|
bugnums = l.split()[1:]
|
|
break
|
|
|
|
for b in bugnums:
|
|
if b in options.exclude_bug:
|
|
continue
|
|
try:
|
|
bugs.append(launchpad.bugs[int(b)])
|
|
except:
|
|
print('%s: bug %s does not exist or is not accessible' %
|
|
(changesfileurl, b))
|
|
|
|
return bugs
|
|
|
|
|
|
def update_sru_bug(bug, pkg):
|
|
'''Unsubscribe SRU team and comment on bug re: how to report regressions'''
|
|
m_subjects = [m.subject for m in bug.messages]
|
|
if 'Update Released' in m_subjects:
|
|
print('LP: #%s was not commented on' % bug.id)
|
|
return
|
|
sru_team = launchpad.people['ubuntu-sru']
|
|
bug.unsubscribe(person=sru_team)
|
|
text = ("The verification of the Stable Release Update for %s has "
|
|
"completed successfully and the package is now being released "
|
|
"to -updates. Subsequently, the Ubuntu Stable Release Updates "
|
|
"Team is being unsubscribed and will not receive messages "
|
|
"about this bug report. In the event that you encounter "
|
|
"a regression using the package from -updates please report "
|
|
"a new bug using ubuntu-bug and tag the bug report "
|
|
"regression-update so we can easily find any regressions." % pkg)
|
|
bug.newMessage(subject="Update Released", content=text)
|
|
bug.lp_save()
|
|
|
|
|
|
def get_versions(options, sourcename):
|
|
'''Get current package versions.
|
|
|
|
If options.pattern is True, return all versions for package names
|
|
matching options.pattern.
|
|
If options.pattern is False, only return one result.
|
|
|
|
Return map pkgname -> {'release': version, 'updates': version,
|
|
'proposed': version, 'changesfile': url_of_proposed_changes,
|
|
'published': proposed_date}
|
|
'''
|
|
versions = defaultdict(dict)
|
|
if src_archive.reference == 'ubuntu':
|
|
pocket = 'Proposed'
|
|
else:
|
|
pocket = 'Release'
|
|
|
|
matches = src_archive.getPublishedSources(
|
|
source_name=sourcename, exact_match=not options.pattern,
|
|
status='Published', pocket=pocket, distro_series=series)
|
|
for match in matches:
|
|
# versions in all pockets
|
|
for pub in src_archive.getPublishedSources(
|
|
source_name=match.source_package_name, exact_match=True,
|
|
status='Published', distro_series=series):
|
|
key = pub.pocket.lower()
|
|
# special case for ppas, which don't have pockets but need
|
|
# to be treated as -proposed
|
|
if pocket == 'Release' and key == 'release':
|
|
key = 'proposed'
|
|
versions[pub.source_package_name][key] = (
|
|
pub.source_package_version)
|
|
if pocket in pub.pocket:
|
|
versions[pub.source_package_name]['changesfile'] = (
|
|
pub.changesFileUrl())
|
|
# When the destination archive differs from the source scan that too.
|
|
if dst_archive != src_archive:
|
|
for pub in dst_archive.getPublishedSources(
|
|
source_name=match.source_package_name, exact_match=True,
|
|
status='Published', distro_series=series):
|
|
key = 'security' # pub.pocket.lower()
|
|
versions[pub.source_package_name][key] = (
|
|
pub.source_package_version)
|
|
if pocket in pub.pocket:
|
|
versions[pub.source_package_name]['changesfile'] = (
|
|
pub.changesFileUrl())
|
|
# devel version
|
|
if devel_series:
|
|
for pub in src_archive.getPublishedSources(
|
|
source_name=match.source_package_name, exact_match=True,
|
|
status='Published', distro_series=devel_series):
|
|
if pub.pocket in ('Release', 'Proposed'):
|
|
versions[pub.source_package_name]['devel'] = (
|
|
pub.source_package_version)
|
|
else:
|
|
versions[match.source_package_name]['devel'] = None
|
|
|
|
return versions
|
|
|
|
|
|
def release_packages(options, packages):
|
|
'''Release the packages listed in the packages argument.'''
|
|
|
|
pkg_versions_map = {}
|
|
# Dictionary of packages and their versions that need copying by britney.
|
|
# Those packages have unblock hints added.
|
|
packages_to_britney = {}
|
|
|
|
for package in packages:
|
|
pkg_versions_map[package] = get_versions(options, package)
|
|
if not pkg_versions_map[package]:
|
|
message = ('ERROR: No such package, ' + package + ', in '
|
|
'-proposed, aborting\n')
|
|
sys.stderr.write(message)
|
|
sys.exit(1)
|
|
|
|
for pkg, versions in pkg_versions_map[package].items():
|
|
print('--- Releasing %s ---' % pkg)
|
|
print('Proposed: %s' % versions['proposed'])
|
|
if 'security' in versions:
|
|
print('Security: %s' % versions['security'])
|
|
if 'updates' in versions:
|
|
print('Updates: %s' % versions['updates'])
|
|
else:
|
|
print('Release: %s' % versions.get('release'))
|
|
if options.devel and 'devel' in versions:
|
|
print('Devel: %s' % versions['devel'])
|
|
|
|
copy = partial(
|
|
dst_archive.copyPackage, from_archive=src_archive,
|
|
include_binaries=True, source_name=pkg,
|
|
version=versions['proposed'], auto_approve=True)
|
|
|
|
if options.devel and not options.britney:
|
|
if ('devel' not in versions or
|
|
versions['devel'] in (
|
|
versions.get('updates', 'notexisting'),
|
|
versions['release'])):
|
|
if not options.no_act:
|
|
copy(to_pocket='Proposed', to_series=devel_series.name)
|
|
print('Version in %s matches development series, '
|
|
'copied to %s-proposed' %
|
|
(release, devel_series.name))
|
|
else:
|
|
print('ERROR: Version in %s does not match development '
|
|
'series, not copying' % release)
|
|
|
|
if options.no_act:
|
|
if options.release:
|
|
print('Would copy to %s' % release)
|
|
else:
|
|
print('Would copy to %s-updates' % release)
|
|
else:
|
|
if options.release:
|
|
# -proposed -> release
|
|
copy(to_pocket='Release', to_series=release)
|
|
print('Copied to %s' % release)
|
|
else:
|
|
# -proposed -> -updates
|
|
if (package != 'linux' and
|
|
not package.startswith('linux-') and
|
|
not options.security):
|
|
if options.britney:
|
|
# We can opt in to use britney for the package copy
|
|
# instead of doing direct pocket copies.
|
|
packages_to_britney[pkg] = versions['proposed']
|
|
else:
|
|
copy(to_pocket='Updates', to_series=release,
|
|
phased_update_percentage=options.percentage)
|
|
print('Copied to %s-updates' % release)
|
|
else:
|
|
copy(to_pocket='Updates', to_series=release)
|
|
print('Copied to %s-updates' % release)
|
|
|
|
# -proposed -> -security
|
|
if options.security:
|
|
if options.no_act:
|
|
print('Would copy to %s-security' % release)
|
|
else:
|
|
copy(to_pocket='Security', to_series=release)
|
|
print('Copied to %s-security' % release)
|
|
|
|
# Write hints for britney to copy the selected packages
|
|
if options.britney and packages_to_britney:
|
|
release_package_via_britney(options, packages_to_britney)
|
|
# If everything went well, update the bugs
|
|
if not options.no_bugs:
|
|
for pkg_versions in pkg_versions_map.values():
|
|
for pkg, versions in pkg_versions.items():
|
|
sru_bugs = match_srubugs(options, versions['changesfile'])
|
|
tag = 'verification-needed-%s' % release
|
|
for sru_bug in sru_bugs:
|
|
if tag not in sru_bug.tags:
|
|
update_sru_bug(sru_bug, pkg)
|
|
|
|
|
|
def release_package_via_britney(options, packages):
|
|
'''Release selected packages via britney unblock hints.'''
|
|
|
|
hints_path = os.path.join(options.cache, 'hints-ubuntu-%s' % release)
|
|
hints_file = os.path.join(hints_path, 'sru-release')
|
|
# Checkout the hints branch
|
|
if not os.path.exists(hints_path):
|
|
cmd = ['bzr', 'checkout', '--lightweight',
|
|
BZR_HINT_BRANCH % release, hints_path]
|
|
else:
|
|
cmd = ['bzr', 'update', hints_path]
|
|
try:
|
|
subprocess.check_call(cmd)
|
|
except subprocess.CalledProcessError:
|
|
sys.stderr.write("Failed bzr %s for the hints branch at %s\n" %
|
|
(cmd[1], hints_path))
|
|
sys.exit(1)
|
|
# Create the hint with a timestamp comment
|
|
timestamp = time.time() # In python2 we can't use datetime.timestamp()
|
|
date = datetime.datetime.now().ctime()
|
|
unblock_string = '# %s %s\n' % (timestamp, date)
|
|
unblock_string += ''.join(['unblock %s/%s\n' % (pkg, ver)
|
|
for pkg, ver in packages.items()])
|
|
unblock_string += '\n'
|
|
# Update and commit the hint
|
|
with open(hints_file, 'a+') as f:
|
|
f.write(unblock_string)
|
|
cmd = ['bzr', 'commit', '-m', 'sru-release %s %s' %
|
|
(release, ' '.join(packages.keys()))]
|
|
try:
|
|
subprocess.check_call(cmd)
|
|
except subprocess.CalledProcessError:
|
|
sys.stderr.write('Failed to bzr commit to the hints file %s\n'
|
|
'Please investigate the local hint branch and '
|
|
'commit the required unblock entries as otherwise '
|
|
'your changes will be lost.\n' %
|
|
hints_file)
|
|
sys.exit(1)
|
|
print('Added hints for promotion in release %s of packages %s' %
|
|
(release, ' '.join(packages.keys())))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) > 1 and sys.argv[1] == "run-tests":
|
|
sys.exit(unittest.main(argv=[sys.argv[0]] + sys.argv[2:]))
|
|
|
|
parser = optparse.OptionParser(
|
|
usage='usage: %prog [options] <release> <package> [<package> ...]')
|
|
|
|
parser.add_option(
|
|
'-l', '--launchpad', dest='launchpad_instance', default='production')
|
|
parser.add_option(
|
|
'--security', action='store_true', default=False,
|
|
help='Additionally copy to -security pocket')
|
|
parser.add_option(
|
|
'-d', '--devel', action='store_true', default=False,
|
|
help='Additionally copy to development release (only works if that '
|
|
'has the same version as <release>)')
|
|
parser.add_option(
|
|
'-r', '--release', action='store_true', default=False,
|
|
help='Copy to release pocket instead of -updates (useful for staging '
|
|
'uploads in development release)')
|
|
parser.add_option(
|
|
"-z", "--percentage", type="int", default=10,
|
|
metavar="PERCENTAGE", help="set phased update percentage")
|
|
parser.add_option(
|
|
'-n', '--no-act', action='store_true', default=False,
|
|
help='Only perform checks, but do not actually copy packages')
|
|
parser.add_option(
|
|
'-p', '--pattern', action='store_true', default=False,
|
|
help='Treat package names as patterns, not exact matches')
|
|
parser.add_option(
|
|
'--no-bugs', action='store_true', default=False,
|
|
help='Do not act on any bugs (helpful to avoid races).')
|
|
parser.add_option(
|
|
'--exclude-bug', action='append', default=[], metavar='BUG',
|
|
help='Do not update BUG.')
|
|
parser.add_option(
|
|
'-E', '--esm', action='store_true', default=False,
|
|
help='Copy from the kernel ESM proposed PPA to the ESM publication PPA')
|
|
parser.add_option(
|
|
'--skip-package-group-check', action='store_true', default=False,
|
|
help=('Skip the package set checks that require some packages '
|
|
'be released together'))
|
|
parser.add_option(
|
|
'--britney', action='store_true', default=False,
|
|
help='Use britney for copying the packages over to -updates (only '
|
|
'works for regular package releases into updates)')
|
|
parser.add_option(
|
|
'-C', '--cache', default='~/.cache/sru-release',
|
|
help='Cache directory to be used for the britney hints checkout')
|
|
|
|
options, args = parser.parse_args()
|
|
|
|
if len(args) < 2:
|
|
parser.error(
|
|
'You must specify a release and source package(s), see --help')
|
|
|
|
if options.release and (options.security or options.devel):
|
|
parser.error('-r and -s/-d are mutually exclusive, see --help')
|
|
|
|
release = args.pop(0)
|
|
packages = args
|
|
|
|
# XXX: we only want to instantiate KernelSeries if we suspect this is
|
|
# a kernel package, this is necessarily dirty, dirty, dirty.
|
|
kernel_checks = False
|
|
for package in packages:
|
|
if package.startswith('linux-') or package == 'linux':
|
|
kernel_checks = True
|
|
|
|
if not options.skip_package_group_check:
|
|
try:
|
|
packages = check_package_sets(packages)
|
|
except ValueError as e:
|
|
sys.stderr.write(e.args[0] + '\n')
|
|
sys.exit(1)
|
|
|
|
options.cache = os.path.expanduser(options.cache)
|
|
if not os.path.isdir(options.cache):
|
|
if os.path.exists(options.cache):
|
|
print('Cache path %s already exists and is not a directory.'
|
|
% options.cache)
|
|
sys.exit(1)
|
|
os.makedirs(options.cache)
|
|
|
|
launchpad = Launchpad.login_with(
|
|
'ubuntu-archive-tools', options.launchpad_instance, version='devel')
|
|
ubuntu = launchpad.distributions['ubuntu']
|
|
series = ubuntu.getSeries(name_or_version=release)
|
|
devel_series = ubuntu.current_series
|
|
if not devel_series:
|
|
sys.stderr.write(
|
|
'WARNING: No current development series, -d will not work\n')
|
|
devel_series = None
|
|
|
|
ks_source = None
|
|
if kernel_checks:
|
|
kernel_series = KernelSeries()
|
|
|
|
# See if we have a kernel-series record for this package. If we do
|
|
# then we are going to pivot to the routing therein.
|
|
ks_series = kernel_series.lookup_series(codename=release)
|
|
for ks_source_find in ks_series.sources:
|
|
for ks_package in ks_source_find.packages:
|
|
if ks_package.name == packages[0]:
|
|
ks_source = ks_source_find
|
|
break
|
|
|
|
# First confirm everything in this set we are attempting to release
|
|
# are indeed listed as valid for this kernel.
|
|
if ks_source is not None:
|
|
for package in packages:
|
|
if ks_source.lookup_package(package) is None:
|
|
sys.stderr.write(
|
|
'WARNING: {} not found in packages for kernel {}\n'.format(
|
|
package, ks_source.name))
|
|
|
|
if ks_source is None and release in ('precise', 'trusty'):
|
|
sys.stdout.write(
|
|
'Called for {}; assuming kernel ESM publication\n'.format(release))
|
|
options.esm = True
|
|
|
|
# If we found a KernelSeries entry this has accurate routing information
|
|
# attached use that.
|
|
if ks_source is not None:
|
|
src_archive_ref, src_archive_pocket = ks_source.routing.lookup_destination('proposed', primary=True)
|
|
src_archive = launchpad.archives.getByReference(
|
|
reference=src_archive_ref)
|
|
dst_archive_ref, dst_archive_pocket = ks_source.routing.lookup_destination('updates', primary=True)
|
|
if dst_archive_ref == src_archive_ref:
|
|
dst_archive = src_archive
|
|
else:
|
|
dst_archive = launchpad.archives.getByReference(
|
|
reference=dst_archive_ref)
|
|
|
|
# Announce any non-standard archive routing.
|
|
if src_archive_ref != 'ubuntu':
|
|
print("Src Archive: {}".format(src_archive_ref))
|
|
if dst_archive_ref != 'ubuntu':
|
|
print("Dst Archive: {}".format(dst_archive_ref))
|
|
# --security is meaningless for private PPA publishing (XXX: currently true)
|
|
options.security = False
|
|
options.release = True
|
|
|
|
elif options.esm:
|
|
# --security is meaningless for ESM everything is a security update.
|
|
options.security = False
|
|
options.release = True
|
|
src_archive = launchpad.archives.getByReference(
|
|
reference='~canonical-kernel-esm/ubuntu/proposed')
|
|
dst_archive = launchpad.archives.getByReference(
|
|
reference='~ubuntu-esm/ubuntu/esm')
|
|
else:
|
|
src_archive = dst_archive = ubuntu.getArchive(name='primary')
|
|
|
|
release_packages(options, packages)
|