diff --git a/britney.py b/britney.py index 8e525bc..4c5e380 100755 --- a/britney.py +++ b/britney.py @@ -51,7 +51,7 @@ and Britney.read_binaries). Other than source and binary packages, Britney loads the following data: * BugsV, which contains the list of release-critical bugs for a given - version of a source or binary package (see Britney.read_bugs). + version of a source or binary package (see RCBugPolicy.read_bugs). * Dates, which contains the date of the upload of a given version of a source package (see Britney.read_dates). @@ -206,7 +206,7 @@ from britney_util import (old_libraries_format, undo_changes, write_excuses, write_heidi_delta, write_controlfiles, old_libraries, is_nuninst_asgood_generous, clone_nuninst, check_installability) -from policies.policy import AgePolicy, PolicyVerdict +from policies.policy import AgePolicy, RCBugPolicy, PolicyVerdict from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, MULTIARCH, ESSENTIAL) @@ -332,10 +332,6 @@ class Britney(object): for stat in arch_stat.stat_summary(): self.log("> - %s" % stat, type="I") - # read the release-critical bug summaries for testing and unstable - self.bugs = {'unstable': self.read_bugs(self.options.unstable), - 'testing': self.read_bugs(self.options.testing),} - self.normalize_bugs() for policy in self.policies: policy.hints = self.hints policy.initialise(self) @@ -452,6 +448,7 @@ class Britney(object): self.options.ignore_cruft = False self.policies.append(AgePolicy(self.options, MINDAYS)) + self.policies.append(RCBugPolicy(self.options)) def log(self, msg, type="I"): """Print info messages according to verbosity level @@ -792,78 +789,9 @@ class Britney(object): for provided_pkg, provided_version, _ in dpkg[PROVIDES]: provides[provided_pkg].add((pkg, provided_version)) - # return a tuple with the list of real and virtual packages return (packages, provides) - def read_bugs(self, basedir): - """Read the release critical bug summary from the specified directory - - The RC bug summaries are read from the `BugsV' file within the - directory specified in the `basedir' parameter. The file contains - rows with the format: - - [,...] - - The method returns a dictionary where the key is the binary package - name and the value is the list of open RC bugs for it. - """ - bugs = defaultdict(list) - filename = os.path.join(basedir, "BugsV") - self.log("Loading RC bugs data from %s" % filename) - for line in open(filename, encoding='ascii'): - l = line.split() - if len(l) != 2: - self.log("Malformed line found in line %s" % (line), type='W') - continue - pkg = l[0] - bugs[pkg] += l[1].split(",") - return bugs - - def __maxver(self, pkg, dist): - """Return the maximum version for a given package name - - This method returns None if the specified source package - is not available in the `dist' distribution. If the package - exists, then it returns the maximum version between the - source package and its binary packages. - """ - maxver = None - if pkg in self.sources[dist]: - maxver = self.sources[dist][pkg][VERSION] - for arch in self.options.architectures: - if pkg not in self.binaries[dist][arch][0]: continue - pkgv = self.binaries[dist][arch][0][pkg][VERSION] - if maxver is None or apt_pkg.version_compare(pkgv, maxver) > 0: - maxver = pkgv - return maxver - - def normalize_bugs(self): - """Normalize the release critical bug summaries for testing and unstable - - The method doesn't return any value: it directly modifies the - object attribute `bugs'. - """ - # loop on all the package names from testing and unstable bug summaries - for pkg in set(chain(self.bugs['testing'], self.bugs['unstable'])): - - # make sure that the key is present in both dictionaries - if pkg not in self.bugs['testing']: - self.bugs['testing'][pkg] = [] - elif pkg not in self.bugs['unstable']: - self.bugs['unstable'][pkg] = [] - - if pkg.startswith("src:"): - pkg = pkg[4:] - - # retrieve the maximum version of the package in testing: - maxvert = self.__maxver(pkg, 'testing') - - # if the package is not available in testing, then reset - # the list of RC bugs - if maxvert is None: - self.bugs['testing'][pkg] = [] - def read_hints(self, basedir): """Read the hint commands from the specified directory @@ -1361,6 +1289,31 @@ class Britney(object): excuse.addhtml("Too young, but urgency pushed by %s" % who) excuse.setdaysold(age_info['current-age'], age_min_req) + # if the suite is unstable, then we have to check the release-critical bug lists before + # updating testing; if the unstable package has RC bugs that do not apply to the testing + # one, the check fails and we set update_candidate to False to block the update + if 'rc-bugs' in policy_info: + rcbugs_info = policy_info['rc-bugs'] + new_bugs = rcbugs_info['unique-source-bugs'] + old_bugs = rcbugs_info['unique-target-bugs'] + + excuse.setbugs(old_bugs, new_bugs) + + if new_bugs: + excuse.addhtml("%s has new bugs!" % (src, quote(src))) + excuse.addhtml("Updating %s introduces new bugs: %s" % (src, ", ".join( + ["#%s" % (quote(a), a) for a in new_bugs]))) + update_candidate = False + + if old_bugs: + excuse.addhtml("Updating %s fixes old bugs: %s" % (src, ", ".join( + ["#%s" % (quote(a), a) for a in old_bugs]))) + if new_bugs and len(old_bugs) > len(new_bugs): + excuse.addhtml("%s introduces new bugs, so still ignored (even " + "though it fixes more than it introduces, whine at debian-release)" % src) + all_binaries = self.all_binaries if suite in ('pu', 'tpu') and source_t: @@ -1372,7 +1325,7 @@ class Britney(object): if not any(x for x in source_t[BINARIES] if x[2] == arch and all_binaries[x][ARCHITECTURE] != 'all'): continue - + # if the (t-)p-u package has produced any binaries on # this architecture then we assume it's ok. this allows for # uploads to (t-)p-u which intentionally drop binary @@ -1481,45 +1434,6 @@ class Britney(object): excuse.addreason("no-binaries") update_candidate = False - # if the suite is unstable, then we have to check the release-critical bug lists before - # updating testing; if the unstable package has RC bugs that do not apply to the testing - # one, the check fails and we set update_candidate to False to block the update - if suite == 'unstable': - for pkg in pkgs: - bugs_t = [] - bugs_u = [] - if pkg in self.bugs['testing']: - bugs_t.extend(self.bugs['testing'][pkg]) - if pkg in self.bugs['unstable']: - bugs_u.extend(self.bugs['unstable'][pkg]) - if 'source' in pkgs[pkg]: - spkg = "src:%s" % (pkg) - if spkg in self.bugs['testing']: - bugs_t.extend(self.bugs['testing'][spkg]) - if spkg in self.bugs['unstable']: - bugs_u.extend(self.bugs['unstable'][spkg]) - - new_bugs = sorted(set(bugs_u).difference(bugs_t)) - old_bugs = sorted(set(bugs_t).difference(bugs_u)) - - excuse.setbugs(old_bugs,new_bugs) - - if len(new_bugs) > 0: - excuse.addhtml("%s (%s) has new bugs!" % (pkg, ", ".join(pkgs[pkg]), quote(pkg))) - excuse.addhtml("Updating %s introduces new bugs: %s" % (pkg, ", ".join( - ["#%s" % (quote(a), a) for a in new_bugs]))) - update_candidate = False - excuse.addreason("buggy") - - if len(old_bugs) > 0: - excuse.addhtml("Updating %s fixes old bugs: %s" % (pkg, ", ".join( - ["#%s" % (quote(a), a) for a in old_bugs]))) - if len(old_bugs) > len(new_bugs) and len(new_bugs) > 0: - excuse.addhtml("%s introduces new bugs, so still ignored (even " - "though it fixes more than it introduces, whine at debian-release)" % pkg) - # check if there is a `force' hint for this package, which allows it to go in even if it is not updateable forces = [x for x in self.hints.search('force', package=src) if source_u[VERSION] == x.version] if forces: diff --git a/policies/policy.py b/policies/policy.py index ed6d6a2..12665fd 100644 --- a/policies/policy.py +++ b/policies/policy.py @@ -4,7 +4,7 @@ import apt_pkg import os import time -from consts import VERSION +from consts import VERSION, BINARIES @unique @@ -240,3 +240,99 @@ class AgePolicy(BasePolicy): version, date = dates[pkg] fd.write("%s %s %d\n" % (pkg, version, date)) os.rename(filename_tmp, filename) + + +class RCBugPolicy(BasePolicy): + """RC bug regression policy for source migrations + + The RCBugPolicy will read provided list of RC bugs and block any + source upload that would introduce a *new* RC bug in the target + suite. + + The RCBugPolicy's decision is influenced by the following: + + State files: + * ${UNSTABLE}/BugsV: File containing RC bugs for packages in the + source suite. + - This file needs to be updated externally. + * ${TESTING}/BugsV: File containing RC bugs for packages in the + target suite. + - This file needs to be updated externally. + """ + + def __init__(self, options): + super().__init__(options, {'unstable'}) + self._bugs = {} + + def initialise(self, britney): + super().initialise(britney) + self._bugs['unstable'] = self._read_bugs(self.options.unstable) + self._bugs['testing'] = self._read_bugs(self.options.testing) + + def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist): + # retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing) + if 'rc-bugs' not in policy_info: + policy_info['rc-bugs'] = rcbugs_info = {} + else: + rcbugs_info = policy_info['rc-bugs'] + + bugs_t = set() + bugs_u = set() + + for src_key in (source_name, 'src:%s' % source_name): + if source_data_tdist and src_key in self._bugs['testing']: + bugs_t.update(self._bugs['testing'][src_key]) + if src_key in self._bugs['unstable']: + bugs_u.update(self._bugs['unstable'][src_key]) + + for pkg, _, _ in source_data_srcdist[BINARIES]: + if pkg in self._bugs['unstable']: + bugs_u |= self._bugs['unstable'][pkg] + if source_data_tdist: + for pkg, _, _ in source_data_tdist[BINARIES]: + if pkg in self._bugs['testing']: + bugs_t |= self._bugs['testing'][pkg] + + # If a package is not in testing, it has no RC bugs per + # definition. Unfortunately, it seems that the live-data is + # not always accurate (e.g. live-2011-12-13 suggests that + # obdgpslogger had the same bug in testing and unstable, + # but obdgpslogger was not in testing at that time). + # - For the curious, obdgpslogger was removed on that day + # and the BTS probably had not caught up with that fact. + # (https://tracker.debian.org/news/415935) + assert not bugs_t or source_data_tdist, "%s had bugs in testing but is not in testing" % source_name + + rcbugs_info['shared-bugs'] = sorted(bugs_u & bugs_t) + rcbugs_info['unique-source-bugs'] = sorted(bugs_u - bugs_t) + rcbugs_info['unique-target-bugs'] = sorted(bugs_t - bugs_u) + + if not bugs_u or bugs_u <= bugs_t: + return PolicyVerdict.PASS + return PolicyVerdict.REJECTED_PERMANENTLY + + def _read_bugs(self, basedir): + """Read the release critical bug summary from the specified directory + + The RC bug summaries are read from the `BugsV' file within the + directory specified in the `basedir' parameter. The file contains + rows with the format: + + [,...] + + The method returns a dictionary where the key is the binary package + name and the value is the list of open RC bugs for it. + """ + bugs = {} + filename = os.path.join(basedir, "BugsV") + self.log("Loading RC bugs data from %s" % filename) + for line in open(filename, encoding='ascii'): + l = line.split() + if len(l) != 2: + self.log("Malformed line found in line %s" % (line), type='W') + continue + pkg = l[0] + if pkg not in bugs: + bugs[pkg] = set() + bugs[pkg].update(l[1].split(",")) + return bugs