Lubuntu
/
britney2-ubuntu
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into autopkgtest

Browse Source
ubuntu/rebased
Niels Thykier 7 years ago
parent 502f4c14a2 e441902d92
commit f752ea4ac7
7 changed files with 149 additions and 147 deletions
Show Stats Download Patch File Download Diff File

199
britney.py
Unescape View File

@ -179,6 +179,7 @@ does for the generation of the update excuses.
* The excuses are written in an HTML file. * The excuses are written in an HTML file.
""" """
import logging
import optparse import optparse
import os import os
import sys import sys
@ -254,9 +255,35 @@ class Britney(object):
the information needed by the other methods of the class. the information needed by the other methods of the class.
""" """
# setup logging - provide the "short level name" (i.e. INFO -> I) that
# we used to use prior to using the logging module.
old_factory = logging.getLogRecordFactory()
short_level_mapping = {
'CRITIAL': 'F',
'INFO': 'I',
'WARNING': 'W',
'ERROR': 'E',
'DEBUG': 'N',
}
def record_factory(*args, **kwargs):
record = old_factory(*args, **kwargs)
try:
record.shortlevelname = short_level_mapping[record.levelname]
except KeyError:
record.shortlevelname = record.levelname
return record
logging.setLogRecordFactory(record_factory)
logging.basicConfig(format='{shortlevelname}: [{asctime}] - {message}', style='{',
datefmt="%Y-%m-%dT%H:%M:%S%z")
self.logger = logging.getLogger()
# parse the command line arguments # parse the command line arguments
self.policies = [] self.policies = []
self._hint_parser = HintParser(self) self._hint_parser = HintParser()
self.suite_info = {} self.suite_info = {}
self.__parse_arguments() self.__parse_arguments()
MigrationItem.set_architectures(self.options.architectures) MigrationItem.set_architectures(self.options.architectures)
@ -274,7 +301,7 @@ class Britney(object):
self.read_hints(os.path.join(self.suite_info['unstable'].path, 'Hints')) self.read_hints(os.path.join(self.suite_info['unstable'].path, 'Hints'))
if self.options.nuninst_cache: if self.options.nuninst_cache:
self.log("Not building the list of non-installable packages, as requested", type="I") self.logger.info("Not building the list of non-installable packages, as requested")
if self.options.print_uninst: if self.options.print_uninst:
nuninst = self.get_nuninst(build=False) nuninst = self.get_nuninst(build=False)
print('* summary') print('* summary')
@ -322,34 +349,34 @@ class Britney(object):
constraints_file = os.path.join(self.options.static_input_dir, 'constraints') constraints_file = os.path.join(self.options.static_input_dir, 'constraints')
faux_packages = os.path.join(self.options.static_input_dir, 'faux-packages') faux_packages = os.path.join(self.options.static_input_dir, 'faux-packages')
except AttributeError: except AttributeError:
self.log("The static_input_dir option is not set", type='I') self.logger.info("The static_input_dir option is not set")
constraints_file = None constraints_file = None
faux_packages = None faux_packages = None
if faux_packages is not None and os.path.exists(faux_packages): if faux_packages is not None and os.path.exists(faux_packages):
self.log("Loading faux packages from %s" % faux_packages, type='I') self.logger.info("Loading faux packages from %s", faux_packages)
self._load_faux_packages(faux_packages) self._load_faux_packages(faux_packages)
elif faux_packages is not None: elif faux_packages is not None:
self.log("No Faux packages as %s does not exist" % faux_packages, type='I') self.logger.info("No Faux packages as %s does not exist", faux_packages)
if constraints_file is not None and os.path.exists(constraints_file): if constraints_file is not None and os.path.exists(constraints_file):
self.log("Loading constraints from %s" % constraints_file, type='I') self.logger.info("Loading constraints from %s", constraints_file)
self.constraints = self._load_constraints(constraints_file) self.constraints = self._load_constraints(constraints_file)
else: else:
if constraints_file is not None: if constraints_file is not None:
self.log("No constraints as %s does not exist" % constraints_file, type='I') self.logger.info("No constraints as %s does not exist", constraints_file)
self.constraints = { self.constraints = {
'keep-installable': [], 'keep-installable': [],
} }
self.log("Compiling Installability tester", type="I") self.logger.info("Compiling Installability tester")
self._inst_tester = build_installability_tester(self.binaries, self.options.architectures) self._inst_tester = build_installability_tester(self.binaries, self.options.architectures)
if not self.options.nuninst_cache: if not self.options.nuninst_cache:
self.log("Building the list of non-installable packages for the full archive", type="I") self.logger.info("Building the list of non-installable packages for the full archive")
self._inst_tester.compute_testing_installability() self._inst_tester.compute_testing_installability()
nuninst = self.get_nuninst(build=True) nuninst = self.get_nuninst(build=True)
for arch in self.options.architectures: for arch in self.options.architectures:
self.log("> Found %d non-installable packages" % len(nuninst[arch]), type="I") self.logger.info("> Found %d non-installable packages", len(nuninst[arch]))
if self.options.print_uninst: if self.options.print_uninst:
self.nuninst_arch_report(nuninst, arch) self.nuninst_arch_report(nuninst, arch)
@ -361,12 +388,12 @@ class Britney(object):
write_nuninst(self.options.noninst_status, nuninst) write_nuninst(self.options.noninst_status, nuninst)
stats = self._inst_tester.compute_stats() stats = self._inst_tester.compute_stats()
self.log("> Installability tester statistics (per architecture)", type="I") self.logger.info("> Installability tester statistics (per architecture)")
for arch in self.options.architectures: for arch in self.options.architectures:
arch_stat = stats[arch] arch_stat = stats[arch]
self.log("> %s" % arch, type="I") self.logger.info("> %s", arch)
for stat in arch_stat.stat_summary(): for stat in arch_stat.stat_summary():
self.log("> - %s" % stat, type="I") self.logger.info("> - %s", stat)
for policy in self.policies: for policy in self.policies:
policy.hints = self.hints policy.hints = self.hints
@ -380,10 +407,9 @@ class Britney(object):
bad.append((f, pkg_entry1[f], pkg_entry2[f])) bad.append((f, pkg_entry1[f], pkg_entry2[f]))
if bad: # pragma: no cover if bad: # pragma: no cover
self.log("Mismatch found %s %s %s differs" % ( self.logger.error("Mismatch found %s %s %s differs", package, pkg_entry1.version, parch)
package, pkg_entry1.version, parch), type="E")
for f, v1, v2 in bad: for f, v1, v2 in bad:
self.log(" ... %s %s != %s" % (check_field_name[f], v1, v2)) self.logger.info(" ... %s %s != %s", check_field_name[f], v1, v2)
raise ValueError("Invalid data set") raise ValueError("Invalid data set")
# Merge ESSENTIAL if necessary # Merge ESSENTIAL if necessary
@ -427,13 +453,18 @@ class Britney(object):
help="set distribution series name") help="set distribution series name")
(self.options, self.args) = parser.parse_args() (self.options, self.args) = parser.parse_args()
if self.options.verbose:
self.logger.setLevel(logging.INFO)
else:
self.logger.setLevel(logging.WARNING)
# integrity checks # integrity checks
if self.options.nuninst_cache and self.options.print_uninst: # pragma: no cover if self.options.nuninst_cache and self.options.print_uninst: # pragma: no cover
self.log("nuninst_cache and print_uninst are mutually exclusive!", type="E") self.logger.error("nuninst_cache and print_uninst are mutually exclusive!")
sys.exit(1) sys.exit(1)
# if the configuration file exists, then read it and set the additional options # if the configuration file exists, then read it and set the additional options
elif not os.path.isfile(self.options.config): # pragma: no cover elif not os.path.isfile(self.options.config): # pragma: no cover
self.log("Unable to read the configuration file (%s), exiting!" % self.options.config, type="E") self.logger.error("Unable to read the configuration file (%s), exiting!", self.options.config)
sys.exit(1) sys.exit(1)
# minimum days for unstable-testing transition and the list of hints # minimum days for unstable-testing transition and the list of hints
@ -463,29 +494,29 @@ class Britney(object):
self.suite_info[suite] = SuiteInfo(name=suite, path=suite_path, excuses_suffix=suffix) self.suite_info[suite] = SuiteInfo(name=suite, path=suite_path, excuses_suffix=suffix)
else: else:
if suite in {'testing', 'unstable'}: # pragma: no cover if suite in {'testing', 'unstable'}: # pragma: no cover
self.log("Mandatory configuration %s is not set in the config" % suite.upper(), type='E') self.logger.error("Mandatory configuration %s is not set in the config", suite.upper())
sys.exit(1) sys.exit(1)
self.log("Optional suite %s is not defined (config option: %s) " % (suite, suite.upper())) self.logger.info("Optional suite %s is not defined (config option: %s) ", suite, suite.upper())
try: try:
release_file = read_release_file(self.suite_info['testing'].path) release_file = read_release_file(self.suite_info['testing'].path)
self.log("Found a Release file in testing - using that for defaults") self.logger.info("Found a Release file in testing - using that for defaults")
except FileNotFoundError: except FileNotFoundError:
self.log("Testing does not have a Release file.") self.logger.info("Testing does not have a Release file.")
release_file = None release_file = None
if getattr(self.options, "components", None): if getattr(self.options, "components", None):
self.options.components = [s.strip() for s in self.options.components.split(",")] self.options.components = [s.strip() for s in self.options.components.split(",")]
elif release_file and not self.options.control_files: elif release_file and not self.options.control_files:
self.options.components = release_file['Components'].split() self.options.components = release_file['Components'].split()
self.log("Using components listed in Release file: %s" % ' '.join(self.options.components)) self.logger.info("Using components listed in Release file: %s", ' '.join(self.options.components))
else: else:
self.options.components = None self.options.components = None
if self.options.control_files and self.options.components: # pragma: no cover if self.options.control_files and self.options.components: # pragma: no cover
# We cannot regenerate the control files correctly when reading from an # We cannot regenerate the control files correctly when reading from an
# actual mirror (we don't which package goes in what component etc.). # actual mirror (we don't which package goes in what component etc.).
self.log("Cannot use --control-files with mirror-layout (components)!", type="E") self.logger.error("Cannot use --control-files with mirror-layout (components)!")
sys.exit(1) sys.exit(1)
if not hasattr(self.options, "heidi_delta_output"): if not hasattr(self.options, "heidi_delta_output"):
@ -501,12 +532,12 @@ class Britney(object):
allarches = sorted(self.options.architectures.split()) allarches = sorted(self.options.architectures.split())
else: else:
if not release_file: # pragma: no cover if not release_file: # pragma: no cover
self.log("No configured architectures and there is no release file for testing", type="E") self.logger.error("No configured architectures and there is no release file for testing")
self.log("Please check if there is a \"Release\" file in %s" % self.suite_info['testing'].path, type="E") self.logger.error("Please check if there is a \"Release\" file in %s", self.suite_info['testing'].path)
self.log("or if the config file contains a non-empty \"ARCHITECTURES\" field", type="E") self.logger.error("or if the config file contains a non-empty \"ARCHITECTURES\" field")
sys.exit(1) sys.exit(1)
allarches = sorted(release_file['Architectures'].split()) allarches = sorted(release_file['Architectures'].split())
self.log("Using architectures listed in Release file: %s" % ' '.join(allarches)) self.logger.info("Using architectures listed in Release file: %s", ' '.join(allarches))
arches = [x for x in allarches if x in self.options.nobreakall_arches] arches = [x for x in allarches if x in self.options.nobreakall_arches]
arches += [x for x in allarches if x not in arches and x not in self.options.outofsync_arches] arches += [x for x in allarches if x not in arches and x not in self.options.outofsync_arches]
arches += [x for x in allarches if x not in arches and x not in self.options.break_arches] arches += [x for x in allarches if x not in arches and x not in self.options.break_arches]
@ -533,18 +564,6 @@ class Britney(object):
def hints(self): def hints(self):
return self._hint_parser.hints return self._hint_parser.hints
def log(self, msg, type="I"):
"""Print info messages according to verbosity level
An easy-and-simple log method which prints messages to the standard
output. The type parameter controls the urgency of the message, and
can be equal to `I' for `Information', `W' for `Warning' and `E' for
`Error'. Warnings and errors are always printed, and information is
printed only if verbose logging is enabled.
"""
if self.options.verbose or type in ("E", "W"):
print("%s: [%s] - %s" % (type, time.asctime(), msg))
def _load_faux_packages(self, faux_packages_file): def _load_faux_packages(self, faux_packages_file):
"""Loads fake packages """Loads fake packages
@ -651,7 +670,7 @@ class Britney(object):
if constraint not in {'present-and-installable'}: # pragma: no cover if constraint not in {'present-and-installable'}: # pragma: no cover
raise ValueError("Unsupported constraint %s for %s (file %s)" % (constraint, pkg_name, constraints_file)) raise ValueError("Unsupported constraint %s for %s (file %s)" % (constraint, pkg_name, constraints_file))
self.log(" - constraint %s" % pkg_name, type='I') self.logger.info(" - constraint %s", pkg_name)
pkg_list = [x.strip() for x in mandatory_field('Package-List').split("\n") if x.strip() != '' and not x.strip().startswith("#")] pkg_list = [x.strip() for x in mandatory_field('Package-List').split("\n") if x.strip() != '' and not x.strip().startswith("#")]
src_data = SourcePackage(faux_version, src_data = SourcePackage(faux_version,
@ -727,11 +746,11 @@ class Britney(object):
for component in self.options.components: for component in self.options.components:
filename = os.path.join(basedir, component, "source", "Sources") filename = os.path.join(basedir, component, "source", "Sources")
filename = possibly_compressed(filename) filename = possibly_compressed(filename)
self.log("Loading source packages from %s" % filename) self.logger.info("Loading source packages from %s", filename)
read_sources_file(filename, sources) read_sources_file(filename, sources)
else: else:
filename = os.path.join(basedir, "Sources") filename = os.path.join(basedir, "Sources")
self.log("Loading source packages from %s" % filename) self.logger.info("Loading source packages from %s", filename)
sources = read_sources_file(filename) sources = read_sources_file(filename)
return sources return sources
@ -741,14 +760,14 @@ class Britney(object):
nprov = [] nprov = []
for or_clause in parts: for or_clause in parts:
if len(or_clause) != 1: # pragma: no cover if len(or_clause) != 1: # pragma: no cover
msg = "Ignoring invalid provides in %s: Alternatives [%s]" % (str(pkg_id), str(or_clause)) msg = "Ignoring invalid provides in %s: Alternatives [%s]"
self.log(msg, type='W') self.logger.warning(msg, str(pkg_id), str(or_clause))
continue continue
for part in or_clause: for part in or_clause:
provided, provided_version, op = part provided, provided_version, op = part
if op != '' and op != '=': # pragma: no cover if op != '' and op != '=': # pragma: no cover
msg = "Ignoring invalid provides in %s: %s (%s %s)" % (str(pkg_id), provided, op, provided_version) msg = "Ignoring invalid provides in %s: %s (%s %s)"
self.log(msg, type='W') self.logger.warning(msg, str(pkg_id), provided, op, provided_version)
continue continue
provided = sys.intern(provided) provided = sys.intern(provided)
provided_version = sys.intern(provided_version) provided_version = sys.intern(provided_version)
@ -757,7 +776,7 @@ class Britney(object):
return nprov return nprov
def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern): def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern):
self.log("Loading binary packages from %s" % filename) self.logger.info("Loading binary packages from %s", filename)
if packages is None: if packages is None:
packages = {} packages = {}
@ -910,7 +929,8 @@ class Britney(object):
for arch in architectures: for arch in architectures:
packages = {} packages = {}
if arch not in listed_archs: if arch not in listed_archs:
self.log("Skipping arch %s for %s: It is not listed in the Release file" % (arch, distribution)) self.logger.info("Skipping arch %s for %s: It is not listed in the Release file",
arch, distribution)
arch2packages[arch] = ({}, {}) arch2packages[arch] = ({}, {})
continue continue
for component in self.options.components: for component in self.options.components:
@ -974,9 +994,9 @@ class Britney(object):
else: else:
filename = os.path.join(hintsdir, who) filename = os.path.join(hintsdir, who)
if not os.path.isfile(filename): if not os.path.isfile(filename):
self.log("Cannot read hints list from %s, no such file!" % filename, type="E") self.logger.error("Cannot read hints list from %s, no such file!", filename)
continue continue
self.log("Loading hints list from %s" % filename) self.logger.info("Loading hints list from %s", filename)
with open(filename, encoding='utf-8') as f: with open(filename, encoding='utf-8') as f:
self._hint_parser.parse_hints(who, self.HINTS[who], filename, f) self._hint_parser.parse_hints(who, self.HINTS[who], filename, f)
@ -992,25 +1012,24 @@ class Britney(object):
if x in ['unblock', 'unblock-udeb']: if x in ['unblock', 'unblock-udeb']:
if apt_pkg.version_compare(hint2.version, hint.version) < 0: if apt_pkg.version_compare(hint2.version, hint.version) < 0:
# This hint is for a newer version, so discard the old one # This hint is for a newer version, so discard the old one
self.log("Overriding %s[%s] = ('%s', '%s') with ('%s', '%s')" % self.logger.warning("Overriding %s[%s] = ('%s', '%s') with ('%s', '%s')",
(x, package, hint2.version, hint2.user, hint.version, hint.user), type="W") x, package, hint2.version, hint2.user, hint.version, hint.user)
hint2.set_active(False) hint2.set_active(False)
else: else:
# This hint is for an older version, so ignore it in favour of the new one # This hint is for an older version, so ignore it in favour of the new one
self.log("Ignoring %s[%s] = ('%s', '%s'), ('%s', '%s') is higher or equal" % self.logger.warning("Ignoring %s[%s] = ('%s', '%s'), ('%s', '%s') is higher or equal",
(x, package, hint.version, hint.user, hint2.version, hint2.user), type="W") x, package, hint.version, hint.user, hint2.version, hint2.user)
hint.set_active(False) hint.set_active(False)
else: else:
self.log("Overriding %s[%s] = ('%s', '%s') with ('%s', '%s')" % self.logger.warning("Overriding %s[%s] = ('%s', '%s') with ('%s', '%s')",
(x, package, hint2.user, hint2, hint.user, hint), x, package, hint2.user, hint2, hint.user, hint)
type="W")
hint2.set_active(False) hint2.set_active(False)
z[package] = key z[package] = key
# Sanity check the hints hash # Sanity check the hints hash
if len(hints["block"]) == 0 and len(hints["block-udeb"]) == 0: if len(hints["block"]) == 0 and len(hints["block-udeb"]) == 0:
self.log("WARNING: No block hints at all, not even udeb ones!", type="W") self.logger.warning("WARNING: No block hints at all, not even udeb ones!")
# Utility methods for package analysis # Utility methods for package analysis
# ------------------------------------ # ------------------------------------
@ -1549,7 +1568,7 @@ class Britney(object):
of this procedure, please refer to the module docstring. of this procedure, please refer to the module docstring.
""" """
self.log("Update Excuses generation started", type="I") self.logger.info("Update Excuses generation started")
# list of local methods and variables (for better performance) # list of local methods and variables (for better performance)
sources = self.sources sources = self.sources
@ -1669,16 +1688,16 @@ class Britney(object):
# write excuses to the output file # write excuses to the output file
if not self.options.dry_run: if not self.options.dry_run:
self.log("> Writing Excuses to %s" % self.options.excuses_output, type="I") self.logger.info("> Writing Excuses to %s", self.options.excuses_output)
sorted_excuses = sorted(excuses.values(), key=lambda x: x.sortkey()) sorted_excuses = sorted(excuses.values(), key=lambda x: x.sortkey())
write_excuses(sorted_excuses, self.options.excuses_output, write_excuses(sorted_excuses, self.options.excuses_output,
output_format="legacy-html") output_format="legacy-html")
if hasattr(self.options, 'excuses_yaml_output'): if hasattr(self.options, 'excuses_yaml_output'):
self.log("> Writing YAML Excuses to %s" % self.options.excuses_yaml_output, type="I") self.logger.info("> Writing YAML Excuses to %s", self.options.excuses_yaml_output)
write_excuses(sorted_excuses, self.options.excuses_yaml_output, write_excuses(sorted_excuses, self.options.excuses_yaml_output,
output_format="yaml") output_format="yaml")
self.log("Update Excuses generation completed", type="I") self.logger.info("Update Excuses generation completed")
# Upgrade run # Upgrade run
# ----------- # -----------
@ -2374,14 +2393,14 @@ class Britney(object):
self.output_write("\n") self.output_write("\n")
def assert_nuninst_is_correct(self): def assert_nuninst_is_correct(self):
self.log("> Update complete - Verifying non-installability counters", type="I") self.logger.info("> Update complete - Verifying non-installability counters")
cached_nuninst = self.nuninst_orig cached_nuninst = self.nuninst_orig
self._inst_tester.compute_testing_installability() self._inst_tester.compute_testing_installability()
computed_nuninst = self.get_nuninst(build=True) computed_nuninst = self.get_nuninst(build=True)
if cached_nuninst != computed_nuninst: # pragma: no cover if cached_nuninst != computed_nuninst: # pragma: no cover
only_on_break_archs = True only_on_break_archs = True
self.log("==================== NUNINST OUT OF SYNC =========================", type="E") self.logger.error("==================== NUNINST OUT OF SYNC =========================")
for arch in self.options.architectures: for arch in self.options.architectures:
expected_nuninst = set(cached_nuninst[arch]) expected_nuninst = set(cached_nuninst[arch])
actual_nuninst = set(computed_nuninst[arch]) actual_nuninst = set(computed_nuninst[arch])
@ -2392,18 +2411,17 @@ class Britney(object):
if (false_negatives or false_positives) and arch not in self.options.break_arches: if (false_negatives or false_positives) and arch not in self.options.break_arches:
only_on_break_archs = False only_on_break_archs = False
if false_negatives: if false_negatives:
self.log(" %s - unnoticed nuninst: %s" % (arch, str(false_negatives)), type="E") self.logger.error(" %s - unnoticed nuninst: %s", arch, str(false_negatives))
if false_positives: if false_positives:
self.log(" %s - invalid nuninst: %s" % (arch, str(false_positives)), type="E") self.logger.error(" %s - invalid nuninst: %s", arch, str(false_positives))
self.log(" %s - actual nuninst: %s" % (arch, str(actual_nuninst)), type="I") self.logger.info(" %s - actual nuninst: %s", arch, str(actual_nuninst))
self.log("==================== NUNINST OUT OF SYNC =========================", type="E") self.logger.error("==================== NUNINST OUT OF SYNC =========================")
if not only_on_break_archs: if not only_on_break_archs:
raise AssertionError("NUNINST OUT OF SYNC") raise AssertionError("NUNINST OUT OF SYNC")
else: else:
self.log("Nuninst is out of sync on some break arches", self.logger.warning("Nuninst is out of sync on some break arches")
type="W")
self.log("> All non-installability counters are ok", type="I") self.logger.info("> All non-installability counters are ok")
def upgrade_testing(self): def upgrade_testing(self):
@ -2414,11 +2432,11 @@ class Britney(object):
commands. commands.
""" """
self.log("Starting the upgrade test", type="I") self.logger.info("Starting the upgrade test")
self.output_write("Generated on: %s\n" % (time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())))) self.output_write("Generated on: %s\n" % (time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time()))))
self.output_write("Arch order is: %s\n" % ", ".join(self.options.architectures)) self.output_write("Arch order is: %s\n" % ", ".join(self.options.architectures))
self.log("> Calculating current uninstallability counters", type="I") self.logger.info("> Calculating current uninstallability counters")
self.nuninst_orig = self.get_nuninst() self.nuninst_orig = self.get_nuninst()
# nuninst_orig may get updated during the upgrade process # nuninst_orig may get updated during the upgrade process
self.nuninst_orig_save = self.get_nuninst() self.nuninst_orig_save = self.get_nuninst()
@ -2474,7 +2492,7 @@ class Britney(object):
# obsolete source packages # obsolete source packages
# a package is obsolete if none of the binary packages in testing # a package is obsolete if none of the binary packages in testing
# are built by it # are built by it
self.log("> Removing obsolete source packages from testing", type="I") self.logger.info("> Removing obsolete source packages from testing")
# local copies for performance # local copies for performance
sources = self.sources['testing'] sources = self.sources['testing']
binaries = self.binaries['testing'] binaries = self.binaries['testing']
@ -2492,15 +2510,15 @@ class Britney(object):
# smooth updates # smooth updates
removals = old_libraries(self.sources, self.binaries, self.options.outofsync_arches) removals = old_libraries(self.sources, self.binaries, self.options.outofsync_arches)
if self.options.smooth_updates: if self.options.smooth_updates:
self.log("> Removing old packages left in testing from smooth updates", type="I") self.logger.info("> Removing old packages left in testing from smooth updates")
if removals: if removals:
self.output_write("Removing packages left in testing for smooth updates (%d):\n%s" % \ self.output_write("Removing packages left in testing for smooth updates (%d):\n%s" % \
(len(removals), old_libraries_format(removals))) (len(removals), old_libraries_format(removals)))
self.do_all(actions=removals) self.do_all(actions=removals)
removals = old_libraries(self.sources, self.binaries, self.options.outofsync_arches) removals = old_libraries(self.sources, self.binaries, self.options.outofsync_arches)
else: else:
self.log("> Not removing old packages left in testing from smooth updates (smooth-updates disabled)", self.logger.info("> Not removing old packages left in testing from smooth updates"
type="I") " (smooth-updates disabled)")
self.output_write("List of old libraries in testing (%d):\n%s" % \ self.output_write("List of old libraries in testing (%d):\n%s" % \
(len(removals), old_libraries_format(removals))) (len(removals), old_libraries_format(removals)))
@ -2511,7 +2529,7 @@ class Britney(object):
if not self.options.dry_run: if not self.options.dry_run:
# re-write control files # re-write control files
if self.options.control_files: if self.options.control_files:
self.log("Writing new testing control files to %s" % self.logger.info("Writing new testing control files to %s",
self.suite_info['testing'].path) self.suite_info['testing'].path)
write_controlfiles(self.sources, self.binaries, write_controlfiles(self.sources, self.binaries,
'testing', self.suite_info['testing'].path) 'testing', self.suite_info['testing'].path)
@ -2520,21 +2538,21 @@ class Britney(object):
policy.save_state(self) policy.save_state(self)
# write HeidiResult # write HeidiResult
self.log("Writing Heidi results to %s" % self.options.heidi_output) self.logger.info("Writing Heidi results to %s", self.options.heidi_output)
write_heidi(self.options.heidi_output, self.sources["testing"], write_heidi(self.options.heidi_output, self.sources["testing"],
self.binaries["testing"], self.binaries["testing"],
outofsync_arches=self.options.outofsync_arches) outofsync_arches=self.options.outofsync_arches)
self.log("Writing delta to %s" % self.options.heidi_delta_output) self.logger.info("Writing delta to %s", self.options.heidi_delta_output)
write_heidi_delta(self.options.heidi_delta_output, write_heidi_delta(self.options.heidi_delta_output,
self.all_selected) self.all_selected)
self.printuninstchange() self.printuninstchange()
self.log("Test completed!", type="I") self.logger.info("Test completed!")
def printuninstchange(self): def printuninstchange(self):
self.log("Checking for newly uninstallable packages", type="I") self.logger.info("Checking for newly uninstallable packages")
text = eval_uninst(self.options.architectures, newly_uninst( text = eval_uninst(self.options.architectures, newly_uninst(
self.nuninst_orig_save, self.nuninst_orig)) self.nuninst_orig_save, self.nuninst_orig))
@ -2548,7 +2566,7 @@ class Britney(object):
This method provides a command line interface for the release team to This method provides a command line interface for the release team to
try hints and evaluate the results. try hints and evaluate the results.
""" """
self.log("> Calculating current uninstallability counters", type="I") self.logger.info("> Calculating current uninstallability counters")
self.nuninst_orig = self.get_nuninst() self.nuninst_orig = self.get_nuninst()
self.nuninst_orig_save = self.get_nuninst() self.nuninst_orig_save = self.get_nuninst()
@ -2595,7 +2613,7 @@ class Britney(object):
try: try:
readline.write_history_file(histfile) readline.write_history_file(histfile)
except IOError as e: except IOError as e:
self.log("Could not write %s: %s" % (histfile, e), type="W") self.logger.warning("Could not write %s: %s", histfile, e)
def do_hint(self, hinttype, who, pkgvers): def do_hint(self, hinttype, who, pkgvers):
"""Process hints """Process hints
@ -2609,7 +2627,7 @@ class Britney(object):
else: else:
_pkgvers = pkgvers _pkgvers = pkgvers
self.log("> Processing '%s' hint from %s" % (hinttype, who), type="I") self.logger.info("> Processing '%s' hint from %s", hinttype, who)
self.output_write("Trying %s from %s: %s\n" % (hinttype, who, " ".join("%s/%s" % (x.uvname, x.version) for x in _pkgvers))) self.output_write("Trying %s from %s: %s\n" % (hinttype, who, " ".join("%s/%s" % (x.uvname, x.version) for x in _pkgvers)))
ok = True ok = True
@ -2665,7 +2683,7 @@ class Britney(object):
excuses relationships. If they build a circular dependency, which we already excuses relationships. If they build a circular dependency, which we already
know as not-working with the standard do_all algorithm, try to `easy` them. know as not-working with the standard do_all algorithm, try to `easy` them.
""" """
self.log("> Processing hints from the auto hinter", type="I") self.logger.info("> Processing hints from the auto hinter")
sources_t = self.sources['testing'] sources_t = self.sources['testing']
excuses = self.excuses excuses = self.excuses
@ -2784,11 +2802,12 @@ class Britney(object):
else: else:
self.upgrade_testing() self.upgrade_testing()
self.log('> Stats from the installability tester', type="I") self.logger.info('> Stats from the installability tester')
for stat in self._inst_tester.stats.stats(): for stat in self._inst_tester.stats.stats():
self.log('> %s' % stat, type="I") self.logger.info('> %s', stat)
else: else:
self.log('Migration computation skipped as requested.', type='I') self.logger.info('Migration computation skipped as requested.')
logging.shutdown()
if __name__ == '__main__': if __name__ == '__main__':

23
britney2/hints.py
Unescape View File

@ -12,6 +12,8 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
import logging
from itertools import chain from itertools import chain
from britney2.migrationitem import MigrationItem from britney2.migrationitem import MigrationItem
@ -131,8 +133,9 @@ def single_hint_taking_list_of_packages(hints, who, hint_type, *args):
class HintParser(object): class HintParser(object):
def __init__(self, britney): def __init__(self):
self._britney = britney logger_name = ".".join((self.__class__.__module__, self.__class__.__name__))
self.logger = logging.getLogger(logger_name)
self.hints = HintCollection() self.hints = HintCollection()
self._hint_table = { self._hint_table = {
'remark': (0, lambda *x: None), 'remark': (0, lambda *x: None),
@ -215,24 +218,20 @@ class HintParser(object):
if hint_name == 'finished': if hint_name == 'finished':
break break
if hint_name not in hint_table: if hint_name not in hint_table:
self.log("Unknown hint found in %s (line %d): '%s'" % (filename, line_no, line), type="W") self.logger.warning("Unknown hint found in %s (line %d): '%s'", filename, line_no, line)
continue continue
if hint_name not in permitted_hints and 'ALL' not in permitted_hints: if hint_name not in permitted_hints and 'ALL' not in permitted_hints:
reason = 'The hint is not a part of the permitted hints for ' + who reason = 'The hint is not a part of the permitted hints for ' + who
self.log("Ignoring \"%s\" hint from %s found in %s (line %d): %s" % ( self.logger.info("Ignoring \"%s\" hint from %s found in %s (line %d): %s",
hint_name, who, filename, line_no, reason), type="I") hint_name, who, filename, line_no, reason)
continue continue
min_args, hint_parser_impl = hint_table[hint_name] min_args, hint_parser_impl = hint_table[hint_name]
if len(l) - 1 < min_args: if len(l) - 1 < min_args:
self.log("Malformed hint found in %s (line %d): Needs at least %d argument(s), got %d" % ( self.logger.warning("Malformed hint found in %s (line %d): Needs at least %d argument(s), got %d",
filename, line_no, min_args, len(l) - 1), type="W") filename, line_no, min_args, len(l) - 1)
continue continue
try: try:
hint_parser_impl(hints, who, *l) hint_parser_impl(hints, who, *l)
except MalformedHintException as e: except MalformedHintException as e:
self.log("Malformed hint found in %s (line %d): \"%s\"" % ( self.logger.warning("Malformed hint found in %s (line %d): \"%s\"", filename, line_no, e.args[0])
filename, line_no, e.args[0]), type="W")
continue continue
def log(self, msg, type="I"):
self._britney.log(msg, type=type)

36
britney2/policies/policy.py
Unescape View File

@ -1,4 +1,5 @@
import json import json
import logging
import os import os
import time import time
from abc import abstractmethod from abc import abstractmethod
@ -30,19 +31,8 @@ class BasePolicy(object):
self.suite_info = suite_info self.suite_info = suite_info
self.applicable_suites = applicable_suites self.applicable_suites = applicable_suites
self.hints = None self.hints = None
logger_name = ".".join((self.__class__.__module__, self.__class__.__name__))
# FIXME: use a proper logging framework self.logger = logging.getLogger(logger_name)
def log(self, msg, type="I"):
"""Print info messages according to verbosity level
An easy-and-simple log method which prints messages to the standard
output. The type parameter controls the urgency of the message, and
can be equal to `I' for `Information', `W' for `Warning' and `E' for
`Error'. Warnings and errors are always printed, and information is
printed only if verbose logging is enabled.
"""
if self.options.verbose or type in ("E", "W"):
print("%s: [%s] - %s" % (type, time.asctime(), msg))
def register_hints(self, hint_parser): # pragma: no cover def register_hints(self, hint_parser): # pragma: no cover
"""Register new hints that this policy accepts """Register new hints that this policy accepts
@ -333,7 +323,7 @@ class AgePolicy(BasePolicy):
if not using_new_name: if not using_new_name:
# If we using the legacy name, then just give up # If we using the legacy name, then just give up
raise raise
self.log("%s does not appear to exist. Creating it" % filename) self.logger.info("%s does not appear to exist. Creating it", filename)
with open(filename, mode='x', encoding='utf-8'): with open(filename, mode='x', encoding='utf-8'):
pass pass
@ -395,7 +385,7 @@ class AgePolicy(BasePolicy):
fd.write("%s %s %d\n" % (pkg, version, date)) fd.write("%s %s %d\n" % (pkg, version, date))
os.rename(filename_tmp, filename) os.rename(filename_tmp, filename)
if old_file is not None and os.path.exists(old_file): if old_file is not None and os.path.exists(old_file):
self.log("Removing old age-policy-dates file %s" % old_file) self.logger.info("Removing old age-policy-dates file %s", old_file)
os.unlink(old_file) os.unlink(old_file)
@ -480,9 +470,8 @@ class RCBugPolicy(BasePolicy):
# Only handle one hint for now # Only handle one hint for now
if 'ignored-bugs' in rcbugs_info: if 'ignored-bugs' in rcbugs_info:
self.log("Ignoring ignore-rc-bugs hint from %s on %s due to another hint from %s" % ( self.logger.info("Ignoring ignore-rc-bugs hint from %s on %s due to another hint from %s",
ignore_hint.user, source_name, rcbugs_info['ignored-bugs']['issued-by'] ignore_hint.user, source_name, rcbugs_info['ignored-bugs']['issued-by'])
))
continue continue
if not ignored_bugs.isdisjoint(bugs_u): if not ignored_bugs.isdisjoint(bugs_u):
bugs_u -= ignored_bugs bugs_u -= ignored_bugs
@ -493,9 +482,8 @@ class RCBugPolicy(BasePolicy):
} }
success_verdict = PolicyVerdict.PASS_HINTED success_verdict = PolicyVerdict.PASS_HINTED
else: else:
self.log("Ignoring ignore-rc-bugs hint from %s on %s as none of %s affect the package" % ( self.logger.info("Ignoring ignore-rc-bugs hint from %s on %s as none of %s affect the package",
ignore_hint.user, source_name, str(ignored_bugs) ignore_hint.user, source_name, str(ignored_bugs))
))
rcbugs_info['shared-bugs'] = sorted(bugs_u & bugs_t) rcbugs_info['shared-bugs'] = sorted(bugs_u & bugs_t)
rcbugs_info['unique-source-bugs'] = sorted(bugs_u - bugs_t) rcbugs_info['unique-source-bugs'] = sorted(bugs_u - bugs_t)
@ -531,11 +519,11 @@ class RCBugPolicy(BasePolicy):
name and the value is the list of open RC bugs for it. name and the value is the list of open RC bugs for it.
""" """
bugs = {} bugs = {}
self.log("Loading RC bugs data from %s" % filename) self.logger.info("Loading RC bugs data from %s", filename)
for line in open(filename, encoding='ascii'): for line in open(filename, encoding='ascii'):
l = line.split() l = line.split()
if len(l) != 2: if len(l) != 2:
self.log("Malformed line found in line %s" % (line), type='W') self.logger.warning("Malformed line found in line %s", line)
continue continue
pkg = l[0] pkg = l[0]
if pkg not in bugs: if pkg not in bugs:
@ -622,7 +610,7 @@ class PiupartsPolicy(BasePolicy):
def _read_piuparts_summary(self, filename, keep_url=True): def _read_piuparts_summary(self, filename, keep_url=True):
summary = {} summary = {}
self.log("Loading piuparts report from {0}".format(filename)) self.logger.info("Loading piuparts report from %s", filename)
with open(filename) as fd: with open(filename) as fd:
if os.fstat(fd.fileno()).st_size < 1: if os.fstat(fd.fileno()).st_size < 1:
return summary return summary

4
doc/hints.rst
Unescape View File

@ -237,7 +237,7 @@ solution for a concrete problem with a valid solution. Although, in
many cases, britney will generally figure out the solution on its own. many cases, britney will generally figure out the solution on its own.
*Caveat*: Due to "uninstallability trading", this hint may cause *Caveat*: Due to "uninstallability trading", this hint may cause
undesireable changes to the target suite. In practise, this is rather undesirable changes to the target suite. In practise, this is rather
rare but the hinter is letting britney decide what "repairs" the rare but the hinter is letting britney decide what "repairs" the
situation. situation.
@ -254,7 +254,7 @@ This hint is generally useful when the provided `<action list>` is more
desirable than the resulting breakage. desirable than the resulting breakage.
*Caveat*: Be sure to test the outcome of these hints. A last minute *Caveat*: Be sure to test the outcome of these hints. A last minute
change can have long lasting undesireable consequences on the end change can have long lasting undesirable consequences on the end
result. result.
Other hints Other hints

2
doc/solutions-to-common-policy-issues.rst
Unescape View File

@ -103,7 +103,7 @@ non-obvious issues:
then libfoo2 depends on libfoo-data v2, then libfoo1 will become then libfoo2 depends on libfoo-data v2, then libfoo1 will become
uninstallable as libfoo-data v2 will "shadow" libfoo-data v1. uninstallable as libfoo-data v2 will "shadow" libfoo-data v1.
Britney complains about "Piupart" Britney complains about "Piuparts"
--------------------------------- ---------------------------------
Britney can be configured to take the results of piuparts (package Britney can be configured to take the results of piuparts (package

26
tests/test_hint_parser.py
Unescape View File

@ -1,18 +1,13 @@
import logging
import unittest import unittest
from britney2.hints import HintParser, single_hint_taking_list_of_packages from britney2.hints import HintParser, single_hint_taking_list_of_packages
from . import MockObject, HINTS_ALL, TEST_HINTER from . import HINTS_ALL, TEST_HINTER
def new_hint_paser(logger=None): def new_hint_parser():
if logger is None: return HintParser()
def empty_logger(x, type='I'):
pass
logger = empty_logger
fake_britney = MockObject(log=logger)
hint_parser = HintParser(fake_britney)
return hint_parser
def parse_should_not_call_this_function(*args, **kwargs): def parse_should_not_call_this_function(*args, **kwargs):
@ -22,8 +17,7 @@ def parse_should_not_call_this_function(*args, **kwargs):
class HintParsing(unittest.TestCase): class HintParsing(unittest.TestCase):
def test_parse_invalid_hints(self): def test_parse_invalid_hints(self):
hint_log = [] hint_parser = new_hint_parser()
hint_parser = new_hint_paser(lambda x, type='I': hint_log.append(x))
hint_parser.register_hint_type('min-10-arg', parse_should_not_call_this_function, min_args=10) hint_parser.register_hint_type('min-10-arg', parse_should_not_call_this_function, min_args=10)
hint_parser.register_hint_type('simple-hint', parse_should_not_call_this_function) hint_parser.register_hint_type('simple-hint', parse_should_not_call_this_function)
@ -47,14 +41,15 @@ class HintParsing(unittest.TestCase):
] ]
for test in tests: for test in tests:
with self.assertLogs() as cm:
hint_parser.parse_hints(TEST_HINTER, test['permissions'], 'test-parse-hint', [test['hint_text']]) hint_parser.parse_hints(TEST_HINTER, test['permissions'], 'test-parse-hint', [test['hint_text']])
assert len(hint_log) == 1
assert test['error_message_contains'] in hint_log[0] assert len(cm.output) == 1
assert test['error_message_contains'] in cm.output[0]
assert hint_parser.hints.is_empty assert hint_parser.hints.is_empty
hint_log.clear()
def test_alias(self): def test_alias(self):
hint_parser = new_hint_paser() hint_parser = new_hint_parser()
hint_parser.register_hint_type('real-name', hint_parser.register_hint_type('real-name',
single_hint_taking_list_of_packages, single_hint_taking_list_of_packages,
aliases=['alias1', 'alias2'] aliases=['alias1', 'alias2']
@ -74,5 +69,6 @@ class HintParsing(unittest.TestCase):
assert not hints.search(type='alias1', package='foo', version='1.0') assert not hints.search(type='alias1', package='foo', version='1.0')
assert not hints.search(type='alias2', package='bar', version='2.0') assert not hints.search(type='alias2', package='bar', version='2.0')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

2
tests/test_policy.py
Unescape View File

@ -24,7 +24,7 @@ def initialize_policy(test_name, policy_class, *args, **kwargs):
} }
policy = policy_class(options, suite_info, *args) policy = policy_class(options, suite_info, *args)
fake_britney = MockObject(log=lambda x, y='I': None) fake_britney = MockObject(log=lambda x, y='I': None)
hint_parser = HintParser(fake_britney) hint_parser = HintParser()
policy.initialise(fake_britney) policy.initialise(fake_britney)
policy.register_hints(hint_parser) policy.register_hints(hint_parser)
hint_parser.parse_hints(TEST_HINTER, HINTS_ALL, 'test-%s' % test_name, hints) hint_parser.parse_hints(TEST_HINTER, HINTS_ALL, 'test-%s' % test_name, hints)

Write Preview
Loading…
Cancel
Save