You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
339 lines
13 KiB
339 lines
13 KiB
6 years ago
|
#!/usr/bin/env python
|
||
|
|
|
||
|
|
# Synchronise package priorities with germinate output
|
||
|
|
# Copyright (C) 2005, 2009, 2010, 2011, 2012 Canonical Ltd.
|
||
|
|
# Author: Colin Watson <cjwatson@ubuntu.com>
|
||
|
|
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License as published by
|
||
|
|
# the Free Software Foundation; either version 2 of the License, or
|
||
|
|
# (at your option) any later version.
|
||
|
|
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
|
||
|
|
# You should have received a copy of the GNU General Public License
|
||
|
|
# along with this program; if not, write to the Free Software
|
||
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||
|
|
|
||
|
|
# <vorlon> elmo: grip_3.2.0-5/sparc seems to have gone missing, marked as
|
||
|
|
# Uploaded 2 1/2 hours ago and nowhere to be found on newraff
|
||
|
|
# <elmo> uh?
|
||
|
|
# <elmo> grip | 3.2.0-5 | unstable | source, alpha, arm, hppa,
|
||
|
|
# i386, ia64, m68k, mips, mipsel, powerpc, s390, sparc
|
||
|
|
# <elmo> I hid it in the pool, being the cunning cabalist that I am
|
||
|
|
|
||
|
|
from __future__ import print_function
|
||
|
|
|
||
|
|
import atexit
|
||
|
|
from collections import defaultdict
|
||
|
|
import csv
|
||
|
|
import gzip
|
||
|
|
try:
|
||
|
|
from html import escape
|
||
|
|
except ImportError:
|
||
|
|
from cgi import escape
|
||
|
|
from optparse import OptionParser
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
from textwrap import dedent
|
||
|
|
import time
|
||
|
|
|
||
|
|
import apt_pkg
|
||
|
|
from launchpadlib.launchpad import Launchpad
|
||
|
|
|
||
|
|
from charts import make_chart, make_chart_header
|
||
|
|
|
||
|
|
|
||
|
|
tempdir = None
|
||
|
|
|
||
|
|
# XXX unhardcode, or maybe adjust seeds?
|
||
|
|
# These packages are not really to be installed by debootstrap, despite
|
||
|
|
# germinate saying so
|
||
|
|
re_not_base = re.compile(r"^(linux-(image|restricted|386|generic|server|power|"
|
||
|
|
"cell|imx51).*|nvidia-kernel-common|grub|yaboot)$")
|
||
|
|
|
||
|
|
# tuples of (package, desired_priority, architecture) which are known to not
|
||
|
|
# be fixable and should be ignored; in particular we cannot set per-arch
|
||
|
|
# priorities
|
||
|
|
ignore = [
|
||
|
|
('hfsutils', 'standard', 'powerpc'), # optional on all other arches
|
||
|
|
('bc', 'important', 'powerpc'), # needed for powerpc-ibm-utils
|
||
|
|
('bc', 'important', 'ppc64el'), # needed for powerpc-ibm-utils
|
||
|
|
('libsgutils2-2', 'standard', 'powerpc'), # needed for lsvpd
|
||
|
|
('libsgutils2-2', 'standard', 'ppc64el'), # needed for lsvpd
|
||
|
|
('libdrm-intel1', 'required', 'amd64'), # needed for plymouth only on x86
|
||
|
|
('libdrm-intel1', 'required', 'i386'), # needed for plymouth only on x86
|
||
|
|
('libelf1', 'optional', 'arm64'), # ltrace not built on arm64
|
||
|
|
('libpciaccess0', 'required', 'amd64'), # needed for plymouth only on x86
|
||
|
|
('libpciaccess0', 'required', 'i386'), # needed for plymouth only on x86
|
||
|
|
('libnuma1', 'optional', 's390x'), # standard on all other arches
|
||
|
|
('libnuma1', 'optional', 'armhf'), # standard on all other arches
|
||
|
|
('libunwind8','standard','amd64'), # wanted by strace on only amd64
|
||
|
|
('multiarch-support','optional','s390x'), # eventually, all arches will downgrade
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def ensure_tempdir():
|
||
|
|
global tempdir
|
||
|
|
if not tempdir:
|
||
|
|
tempdir = tempfile.mkdtemp(prefix='priority-mismatches')
|
||
|
|
atexit.register(shutil.rmtree, tempdir)
|
||
|
|
|
||
|
|
|
||
|
|
def decompress_open(tagfile):
|
||
|
|
ensure_tempdir()
|
||
|
|
decompressed = tempfile.mktemp(dir=tempdir)
|
||
|
|
fin = gzip.GzipFile(filename=tagfile)
|
||
|
|
with open(decompressed, 'wb') as fout:
|
||
|
|
fout.write(fin.read())
|
||
|
|
return open(decompressed, 'r')
|
||
|
|
|
||
|
|
|
||
|
|
# XXX partial code duplication from component-mismatches
|
||
|
|
def read_germinate(suite, arch, seed):
|
||
|
|
local_germinate = os.path.expanduser('~/mirror/ubuntu-germinate')
|
||
|
|
# XXX hardcoding
|
||
|
|
filename = "%s_ubuntu_%s_%s" % (seed, suite, arch)
|
||
|
|
pkgs = {}
|
||
|
|
|
||
|
|
f = open(local_germinate + '/' + filename)
|
||
|
|
for line in f:
|
||
|
|
# Skip header and footer
|
||
|
|
if line[0] == "-" or line.startswith("Package") or line[0] == " ":
|
||
|
|
continue
|
||
|
|
# Skip empty lines
|
||
|
|
line = line.strip()
|
||
|
|
if not line:
|
||
|
|
continue
|
||
|
|
pkgs[line.split('|', 1)[0].strip()] = None
|
||
|
|
f.close()
|
||
|
|
|
||
|
|
return pkgs
|
||
|
|
|
||
|
|
|
||
|
|
def process(options, arch):
|
||
|
|
suite = options.suite
|
||
|
|
components = options.component.split(',')
|
||
|
|
|
||
|
|
archive = os.path.expanduser('~/mirror/ubuntu/')
|
||
|
|
|
||
|
|
if suite in ("warty", "hoary"):
|
||
|
|
required_seed = None
|
||
|
|
important_seed = "base"
|
||
|
|
standard_seed = None
|
||
|
|
elif suite in ("breezy", "dapper", "edgy", "feisty"):
|
||
|
|
required_seed = None
|
||
|
|
important_seed = "minimal"
|
||
|
|
standard_seed = "standard"
|
||
|
|
else:
|
||
|
|
required_seed = "required"
|
||
|
|
important_seed = "minimal"
|
||
|
|
standard_seed = "standard"
|
||
|
|
|
||
|
|
if required_seed is not None:
|
||
|
|
required_pkgs = read_germinate(suite, arch, required_seed)
|
||
|
|
required_pkgs = [
|
||
|
|
pkg for pkg in required_pkgs if not re_not_base.match(pkg)]
|
||
|
|
important_pkgs = read_germinate(suite, arch, important_seed)
|
||
|
|
important_pkgs = [
|
||
|
|
pkg for pkg in important_pkgs if not re_not_base.match(pkg)]
|
||
|
|
if standard_seed is not None:
|
||
|
|
standard_pkgs = read_germinate(suite, arch, standard_seed).keys()
|
||
|
|
required_pkgs.sort()
|
||
|
|
important_pkgs.sort()
|
||
|
|
standard_pkgs.sort()
|
||
|
|
|
||
|
|
original = {}
|
||
|
|
for component in components:
|
||
|
|
binaries_path = "%s/dists/%s/%s/binary-%s/Packages.gz" % (
|
||
|
|
archive, suite, component, arch)
|
||
|
|
for section in apt_pkg.TagFile(decompress_open(binaries_path)):
|
||
|
|
if 'Package' in section and 'Priority' in section:
|
||
|
|
(pkg, priority) = (section['Package'], section['Priority'])
|
||
|
|
original[pkg] = priority
|
||
|
|
|
||
|
|
packages = sorted(original)
|
||
|
|
|
||
|
|
# XXX hardcoding, but who cares really
|
||
|
|
priorities = {'required': 1, 'important': 2, 'standard': 3,
|
||
|
|
'optional': 4, 'extra': 5, 'source': 99}
|
||
|
|
|
||
|
|
# If there is a required seed:
|
||
|
|
# Force everything in the required seed to >= required.
|
||
|
|
# Force everything not in the required seed to < required.
|
||
|
|
# Force everything in the important seed to >= important.
|
||
|
|
# Force everything not in the important seed to < important.
|
||
|
|
# (This allows debootstrap to determine the base system automatically.)
|
||
|
|
# If there is a standard seed:
|
||
|
|
# Force everything in the standard seed to >= standard.
|
||
|
|
# Force everything not in the standard seed to < standard.
|
||
|
|
|
||
|
|
changed = defaultdict(lambda: defaultdict(list))
|
||
|
|
|
||
|
|
for pkg in packages:
|
||
|
|
priority = original[pkg]
|
||
|
|
|
||
|
|
if required_seed is not None and pkg in required_pkgs:
|
||
|
|
if priorities[priority] > priorities["required"]:
|
||
|
|
priority = "required"
|
||
|
|
elif pkg in important_pkgs:
|
||
|
|
if (required_seed is not None and
|
||
|
|
priorities[priority] < priorities["important"]):
|
||
|
|
priority = "important"
|
||
|
|
elif priorities[priority] > priorities["important"]:
|
||
|
|
priority = "important"
|
||
|
|
else:
|
||
|
|
# XXX assumes important and standard are adjacent
|
||
|
|
if priorities[priority] < priorities["standard"]:
|
||
|
|
priority = "standard"
|
||
|
|
|
||
|
|
if standard_seed is not None:
|
||
|
|
if pkg in standard_pkgs:
|
||
|
|
if priorities[priority] > priorities["standard"]:
|
||
|
|
priority = "standard"
|
||
|
|
else:
|
||
|
|
# XXX assumes standard and optional are adjacent
|
||
|
|
if priorities[priority] < priorities["optional"]:
|
||
|
|
priority = "optional"
|
||
|
|
|
||
|
|
if priority != original[pkg] and (pkg, priority, arch) not in ignore:
|
||
|
|
changed[original[pkg]][priority].append(pkg)
|
||
|
|
|
||
|
|
changes =0
|
||
|
|
oldprios = sorted(changed, key=lambda x: priorities[x])
|
||
|
|
for oldprio in oldprios:
|
||
|
|
newprios = sorted(changed[oldprio], key=lambda x: priorities[x])
|
||
|
|
for newprio in newprios:
|
||
|
|
changes += len(changed[oldprio][newprio])
|
||
|
|
header = ("Packages to change from priority %s to %s" %
|
||
|
|
(oldprio, newprio))
|
||
|
|
print(header)
|
||
|
|
print("-" * len(header))
|
||
|
|
for pkg in changed[oldprio][newprio]:
|
||
|
|
print("%s" % pkg)
|
||
|
|
print()
|
||
|
|
if options.html_output is not None:
|
||
|
|
print("<h3>%s</h3>" % escape(header), file=options.html_output)
|
||
|
|
print("<ul>", file=options.html_output)
|
||
|
|
for pkg in changed[oldprio][newprio]:
|
||
|
|
print(
|
||
|
|
"<li>%s</li>" % escape(pkg), file=options.html_output)
|
||
|
|
print("</ul>", file=options.html_output)
|
||
|
|
|
||
|
|
return changes
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = OptionParser(
|
||
|
|
description='Synchronise package priorities with germinate output.')
|
||
|
|
parser.add_option(
|
||
|
|
"-l", "--launchpad", dest="launchpad_instance", default="production")
|
||
|
|
parser.add_option('-o', '--output-file', help='output to this file')
|
||
|
|
parser.add_option('--html-output-file', help='output HTML to this file')
|
||
|
|
parser.add_option(
|
||
|
|
'--csv-file', help='record CSV time series data in this file')
|
||
|
|
parser.add_option('-a', '--architecture',
|
||
|
|
help='look at germinate output for this architecture')
|
||
|
|
parser.add_option('-c', '--component',
|
||
|
|
default='main,restricted,universe,multiverse',
|
||
|
|
help='set overrides by component')
|
||
|
|
parser.add_option('-s', '--suite', help='set overrides by suite')
|
||
|
|
options, args = parser.parse_args()
|
||
|
|
|
||
|
|
if options.suite is None:
|
||
|
|
launchpad = Launchpad.login_anonymously('priority-mismatches',
|
||
|
|
options.launchpad_instance)
|
||
|
|
options.suite = launchpad.distributions['ubuntu'].current_series.name
|
||
|
|
|
||
|
|
if options.output_file is not None:
|
||
|
|
sys.stdout = open('%s.new' % options.output_file, 'w')
|
||
|
|
if options.html_output_file is not None:
|
||
|
|
options.html_output = open('%s.new' % options.html_output_file, 'w')
|
||
|
|
else:
|
||
|
|
options.html_output = None
|
||
|
|
|
||
|
|
options.time = time.time()
|
||
|
|
options.timestamp = time.strftime(
|
||
|
|
'%a %b %e %H:%M:%S %Z %Y', time.gmtime(options.time))
|
||
|
|
print('Generated: %s' % options.timestamp)
|
||
|
|
print()
|
||
|
|
|
||
|
|
if options.html_output is not None:
|
||
|
|
print(dedent("""\
|
||
|
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
|
||
|
|
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
||
|
|
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
|
||
|
|
<head>
|
||
|
|
<meta http-equiv="Content-Type"
|
||
|
|
content="text/html; charset=utf-8" />
|
||
|
|
<title>Priority mismatches for %s</title>
|
||
|
|
<style type="text/css">
|
||
|
|
body { background: #CCCCB0; color: black; }
|
||
|
|
</style>
|
||
|
|
%s
|
||
|
|
</head>
|
||
|
|
<body>
|
||
|
|
<h1>Priority mismatches for %s</h1>
|
||
|
|
""") % (
|
||
|
|
escape(options.suite), make_chart_header(),
|
||
|
|
escape(options.suite)),
|
||
|
|
file=options.html_output)
|
||
|
|
|
||
|
|
changes = 0
|
||
|
|
if options.architecture is None:
|
||
|
|
for arch in ('amd64', 'arm64', 'armhf', 'i386', 'ppc64el', 's390x'):
|
||
|
|
print(arch)
|
||
|
|
print('=' * len(arch))
|
||
|
|
print()
|
||
|
|
if options.html_output is not None:
|
||
|
|
print("<h2>%s</h2>" % escape(arch), file=options.html_output)
|
||
|
|
changes += process(options, arch)
|
||
|
|
else:
|
||
|
|
changes += process(options, options.architecture)
|
||
|
|
|
||
|
|
if options.html_output_file is not None:
|
||
|
|
print("<h2>Over time</h2>", file=options.html_output)
|
||
|
|
print(
|
||
|
|
make_chart("priority-mismatches.csv", ["changes"]),
|
||
|
|
file=options.html_output)
|
||
|
|
print(
|
||
|
|
"<p><small>Generated: %s</small></p>" % escape(options.timestamp),
|
||
|
|
file=options.html_output)
|
||
|
|
print("</body></html>", file=options.html_output)
|
||
|
|
options.html_output.close()
|
||
|
|
os.rename(
|
||
|
|
'%s.new' % options.html_output_file, options.html_output_file)
|
||
|
|
if options.output_file is not None:
|
||
|
|
sys.stdout.close()
|
||
|
|
os.rename('%s.new' % options.output_file, options.output_file)
|
||
|
|
if options.csv_file is not None:
|
||
|
|
if sys.version < "3":
|
||
|
|
open_mode = "ab"
|
||
|
|
open_kwargs = {}
|
||
|
|
else:
|
||
|
|
open_mode = "a"
|
||
|
|
open_kwargs = {"newline": ""}
|
||
|
|
csv_is_new = not os.path.exists(options.csv_file)
|
||
|
|
with open(options.csv_file, open_mode, **open_kwargs) as csv_file:
|
||
|
|
# Field names deliberately hardcoded; any changes require
|
||
|
|
# manually rewriting the output file.
|
||
|
|
fieldnames = [
|
||
|
|
"time",
|
||
|
|
"changes",
|
||
|
|
]
|
||
|
|
csv_writer = csv.DictWriter(csv_file, fieldnames)
|
||
|
|
if csv_is_new:
|
||
|
|
csv_writer.writeheader()
|
||
|
|
csv_writer.writerow(
|
||
|
|
{"time": int(options.time * 1000), "changes": changes})
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|