From 5ec3aea43ac8416b4808d862dddf2602d9a7d52e Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Fri, 25 Mar 2016 15:23:34 +0000 Subject: [PATCH] Move age-handling into a separate file Signed-off-by: Niels Thykier --- britney.py | 167 +++++++---------------------- excuse.py | 1 + policies/__init__.py | 0 policies/policy.py | 242 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 129 deletions(-) create mode 100644 policies/__init__.py create mode 100644 policies/policy.py diff --git a/britney.py b/britney.py index 7518b60..8e525bc 100755 --- a/britney.py +++ b/britney.py @@ -57,7 +57,7 @@ Other than source and binary packages, Britney loads the following data: of a source package (see Britney.read_dates). * Urgencies, which contains the urgency of the upload of a given - version of a source package (see Britney.read_urgencies). + version of a source package (see AgePolicy._read_urgencies). * Hints, which contains lists of commands which modify the standard behaviour of Britney (see Britney.read_hints). @@ -206,6 +206,7 @@ from britney_util import (old_libraries_format, undo_changes, write_excuses, write_heidi_delta, write_controlfiles, old_libraries, is_nuninst_asgood_generous, clone_nuninst, check_installability) +from policies.policy import AgePolicy, PolicyVerdict from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, MULTIARCH, ESSENTIAL) @@ -248,10 +249,9 @@ class Britney(object): This method initializes and populates the data lists, which contain all the information needed by the other methods of the class. """ - # britney's "day" begins at 3pm - self.date_now = int(((time.time() / (60*60)) - 15) / 24) # parse the command line arguments + self.policies = [] self.__parse_arguments() MigrationItem.set_architectures(self.options.architectures) @@ -336,10 +336,9 @@ class Britney(object): self.bugs = {'unstable': self.read_bugs(self.options.unstable), 'testing': self.read_bugs(self.options.testing),} self.normalize_bugs() - - # read additional data - self.dates = self.read_dates(self.options.testing) - self.urgencies = self.read_urgencies(self.options.testing) + for policy in self.policies: + policy.hints = self.hints + policy.initialise(self) def merge_pkg_entries(self, package, parch, pkg_entry1, pkg_entry2, check_fields=check_fields, check_field_name=check_field_name): @@ -402,7 +401,7 @@ class Britney(object): # minimum days for unstable-testing transition and the list of hints # are handled as an ad-hoc case - self.MINDAYS = {} + MINDAYS = {} self.HINTS = {'command-line': self.HINTS_ALL} with open(self.options.config, encoding='utf-8') as config: for line in config: @@ -411,7 +410,7 @@ class Britney(object): k = k.strip() v = v.strip() if k.startswith("MINDAYS_"): - self.MINDAYS[k.split("_")[1].lower()] = int(v) + MINDAYS[k.split("_")[1].lower()] = int(v) elif k.startswith("HINTS_"): self.HINTS[k.split("_")[1].lower()] = \ reduce(lambda x,y: x+y, [hasattr(self, "HINTS_" + i) and getattr(self, "HINTS_" + i) or (i,) for i in v.split()]) @@ -452,6 +451,8 @@ class Britney(object): self.options.ignore_cruft == "0": self.options.ignore_cruft = False + self.policies.append(AgePolicy(self.options, MINDAYS)) + def log(self, msg, type="I"): """Print info messages according to verbosity level @@ -863,90 +864,6 @@ class Britney(object): if maxvert is None: self.bugs['testing'][pkg] = [] - def read_dates(self, basedir): - """Read the upload date for the packages from the specified directory - - The upload dates are read from the `Dates' file within the directory - specified as `basedir' parameter. The file contains rows with the - format: - - - - The dates are expressed as the number of days from 1970-01-01. - - The method returns a dictionary where the key is the binary package - name and the value is a tuple with two items, the version and the date. - """ - dates = {} - filename = os.path.join(basedir, "Dates") - self.log("Loading upload data from %s" % filename) - for line in open(filename, encoding='ascii'): - l = line.split() - if len(l) != 3: continue - try: - dates[l[0]] = (l[1], int(l[2])) - except ValueError: - self.log("Dates, unable to parse \"%s\"" % line, type="E") - return dates - - def write_dates(self, basedir, dates): - """Write the upload date for the packages to the specified directory - - For a more detailed explanation of the format, please check the method - read_dates. - """ - filename = os.path.join(basedir, "Dates") - self.log("Writing upload data to %s" % filename) - with open(filename, 'w', encoding='utf-8') as f: - for pkg in sorted(dates): - f.write("%s %s %d\n" % ((pkg,) + dates[pkg])) - - - def read_urgencies(self, basedir): - """Read the upload urgency of the packages from the specified directory - - The upload urgencies are read from the `Urgency' file within the - directory specified as `basedir' parameter. The file contains rows - with the format: - - - - The method returns a dictionary where the key is the binary package - name and the value is the greatest urgency from the versions of the - package that are higher then the testing one. - """ - - urgencies = {} - filename = os.path.join(basedir, "Urgency") - self.log("Loading upload urgencies from %s" % filename) - for line in open(filename, errors='surrogateescape', encoding='ascii'): - l = line.split() - if len(l) != 3: continue - - # read the minimum days associated with the urgencies - urgency_old = urgencies.get(l[0], None) - mindays_old = self.MINDAYS.get(urgency_old, 1000) - mindays_new = self.MINDAYS.get(l[2], self.MINDAYS[self.options.default_urgency]) - - # if the new urgency is lower (so the min days are higher), do nothing - if mindays_old <= mindays_new: - continue - - # if the package exists in testing and it is more recent, do nothing - tsrcv = self.sources['testing'].get(l[0], None) - if tsrcv and apt_pkg.version_compare(tsrcv[VERSION], l[1]) >= 0: - continue - - # if the package doesn't exist in unstable or it is older, do nothing - usrcv = self.sources['unstable'].get(l[0], None) - if not usrcv or apt_pkg.version_compare(usrcv[VERSION], l[1]) < 0: - continue - - # update the urgency for the package - urgencies[l[0]] = l[2] - - return urgencies - def read_hints(self, basedir): """Read the hint commands from the specified directory @@ -1357,13 +1274,6 @@ class Britney(object): excuse.addhtml("%s source package doesn't exist" % (src)) update_candidate = False - # retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing) - urgency = self.urgencies.get(src, self.options.default_urgency) - if not source_t: - if self.MINDAYS[urgency] < self.MINDAYS[self.options.default_urgency]: - excuse.addhtml("Ignoring %s urgency setting for NEW package" % (urgency)) - urgency = self.options.default_urgency - # if there is a `remove' hint and the requested version is the same as the # version in testing, then stop here and return False for item in self.hints.search('remove', package=src): @@ -1424,30 +1334,32 @@ class Britney(object): # permanence in unstable before updating testing; if the source package is too young, # the check fails and we set update_candidate to False to block the update; consider # the age-days hint, if specified for the package - if suite == 'unstable': - if src not in self.dates: - self.dates[src] = (source_u[VERSION], self.date_now) - elif self.dates[src][0] != source_u[VERSION]: - self.dates[src] = (source_u[VERSION], self.date_now) - - days_old = self.date_now - self.dates[src][1] - min_days = self.MINDAYS[urgency] - - for age_days_hint in [x for x in self.hints.search('age-days', package=src) - if source_u[VERSION] == x.version]: - excuse.addhtml("Overriding age needed from %d days to %d by %s" % (min_days, - int(age_days_hint.days), age_days_hint.user)) - min_days = int(age_days_hint.days) - - excuse.setdaysold(days_old, min_days) - if days_old < min_days: - urgent_hints = [x for x in self.hints.search('urgent', package=src) - if source_u[VERSION] == x.version] - if urgent_hints: - excuse.addhtml("Too young, but urgency pushed by %s" % (urgent_hints[0].user)) + policy_info = excuse.policy_info + policy_verdict = PolicyVerdict.PASS + for policy in self.policies: + if suite in policy.applicable_suites: + v = policy.apply_policy(policy_info, suite, src, source_t, source_u) + if v.value > policy_verdict.value: + policy_verdict = v + + if policy_verdict.is_rejected: + update_candidate = False + + # Joggle some things into excuses + # - remove once the YAML is the canonical source for this information + if 'age' in policy_info: + age_info = policy_info['age'] + age_hint = age_info.get('age-requirement-reduced', None) + age_min_req = age_info['age-requirement'] + if age_hint: + new_req = age_hint['new-requirement'] + who = age_hint['changed-by'] + if new_req: + excuse.addhtml("Overriding age needed from %d days to %d by %s" % ( + age_min_req, new_req, who)) else: - update_candidate = False - excuse.addreason("age") + excuse.addhtml("Too young, but urgency pushed by %s" % who) + excuse.setdaysold(age_info['current-age'], age_min_req) all_binaries = self.all_binaries @@ -1560,7 +1472,7 @@ class Britney(object): excuse.addreason("build-arch") excuse.addreason("build-arch-%s" % arch) - if self.date_now != self.dates[src][1]: + if 'age' in policy_info and policy_info['age']['current-age']: excuse.addhtml(text) # if the source package has no binaries, set update_candidate to False to block the update @@ -2656,11 +2568,8 @@ class Britney(object): write_controlfiles(self.sources, self.binaries, 'testing', self.options.testing) - # write dates - try: - self.write_dates(self.options.outputdir, self.dates) - except AttributeError: - self.write_dates(self.options.testing, self.dates) + for policy in self.policies: + policy.save_state(self) # write HeidiResult self.log("Writing Heidi results to %s" % self.options.heidi_output) diff --git a/excuse.py b/excuse.py index 8942bef..e12318e 100644 --- a/excuse.py +++ b/excuse.py @@ -59,6 +59,7 @@ class Excuse(object): self.oldbugs = set() self.reason = {} self.htmlline = [] + self.policy_info = {} def sortkey(self): if self.daysold == None: diff --git a/policies/__init__.py b/policies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policies/policy.py b/policies/policy.py new file mode 100644 index 0000000..ed6d6a2 --- /dev/null +++ b/policies/policy.py @@ -0,0 +1,242 @@ +from abc import abstractmethod +from enum import Enum, unique +import apt_pkg +import os +import time + +from consts import VERSION + + +@unique +class PolicyVerdict(Enum): + """""" + """ + The migration item passed the policy. + """ + PASS = 1 + """ + The policy was completely overruled by a hint. + """ + PASS_HINTED = 2 + """ + The migration item did not pass the policy, but the failure is believed + to be temporary + """ + REJECTED_TEMPORARILY = 3 + """ + The migration item did not pass the policy and the failure is believed + to be uncorrectable (i.e. a hint or a new version is needed) + """ + REJECTED_PERMANENTLY = 4 + + @property + def is_rejected(self): + return True if self.name.startswith('REJECTED') else False + + +class BasePolicy(object): + + def __init__(self, options, applicable_suites): + self.options = options + self.applicable_suites = applicable_suites + self.hints = None + + # FIXME: use a proper logging framework + def log(self, msg, type="I"): + """Print info messages according to verbosity level + + An easy-and-simple log method which prints messages to the standard + output. The type parameter controls the urgency of the message, and + can be equal to `I' for `Information', `W' for `Warning' and `E' for + `Error'. Warnings and errors are always printed, and information is + printed only if verbose logging is enabled. + """ + if self.options.verbose or type in ("E", "W"): + print("%s: [%s] - %s" % (type, time.asctime(), msg)) + + def initialise(self, britney): + """Called once to make the policy initialise any data structures + + This is useful for e.g. parsing files or other "heavy do-once" work. + """ + pass + + def save_state(self, britney): + """Called once at the end of the run to make the policy save any persistent data + + Note this will *not* be called for "dry-runs" as such runs should not change + the state. + """ + pass + + @abstractmethod + def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist): + pass + + +class AgePolicy(BasePolicy): + """Configurable Aging policy for source migrations + + The AgePolicy will let packages stay in the source suite for a pre-defined + amount of days before letting migrate (based on their urgency, if any). + + The AgePolicy's decision is influenced by the following: + + State files: + * ${TESTING}/Urgency: File containing urgencies for source packages. + Note that urgencies are "sticky" and the most "urgent" urgency will be + used (i.e. the one with lowest age-requirements). + - This file needs to be updated externally, if the policy should take + urgencies into consideration. If empty (or not updated), the policy + will simply use the default urgency (see the "Config" section below) + - In Debian, these values are taken from the .changes file, but that is + not a requirement for Britney. + * ${TESTING}/Dates: File containing the age of all source packages. + - The policy will automatically update this file. + Config: + * DEFAULT_URGENCY: Name of the urgency used for packages without an urgency + (or for unknown urgencies). Will also be used to set the "minimum" + aging requirements for packages not in the target suite. + * MINDAYS_: The age-requirements in days for packages with the + given urgency. + - Commonly used urgencies are: low, medium, high, emergency, critical + Hints: + * urgent /: Disregard the age requirements for a given + source/version. + * age-days X /: Set the age requirements for a given + source/version to X days. Note that X can exceed the highest + age-requirement normally given. + + """ + + def __init__(self, options, mindays): + super().__init__(options, {'unstable'}) + self._min_days = mindays + if options.default_urgency not in mindays: + raise ValueError("Missing age-requirement for default urgency (MINDAYS_%s)" % options.default_urgency) + self._min_days_default = mindays[options.default_urgency] + # britney's "day" begins at 3pm + self._date_now = int(((time.time() / (60*60)) - 15) / 24) + self._dates = {} + self._urgencies = {} + + def initialise(self, britney): + super().initialise(britney) + self._read_dates_file() + self._read_urgencies_file(britney) + + def save_state(self, britney): + super().save_state(britney) + self._write_dates_file() + + def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist): + # retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing) + urgency = self._urgencies.get(source_name, self.options.default_urgency) + if 'age' not in policy_info: + policy_info['age'] = age_info = {} + else: + age_info = policy_info['age'] + + if urgency not in self._min_days: + age_info['unknown-urgency'] = urgency + urgency = self.options.default_urgency + + if not source_data_tdist: + if self._min_days[urgency] < self._min_days_default: + age_info['urgency-reduced'] = { + 'from': urgency, + 'to': self.options.default_urgency, + } + urgency = self.options.default_urgency + + if source_name not in self._dates: + self._dates[source_name] = (source_data_srcdist[VERSION], self._date_now) + elif self._dates[source_name][0] != source_data_srcdist[VERSION]: + self._dates[source_name] = (source_data_srcdist[VERSION], self._date_now) + + days_old = self._date_now - self._dates[source_name][1] + min_days = self._min_days[urgency] + age_info['age-requirement'] = min_days + age_info['current-age'] = days_old + + for age_days_hint in [x for x in self.hints.search('age-days', package=source_name) + if source_data_srcdist[VERSION] == x.version]: + new_req = int(age_days_hint.days) + age_info['age-requirement-reduced'] = { + 'new-requirement': new_req, + 'changed-by': age_days_hint.user + } + min_days = new_req + + if days_old < min_days: + urgent_hints = [x for x in self.hints.search('urgent', package=source_name) + if source_data_srcdist[VERSION] == x.version] + if urgent_hints: + age_info['age-requirement-reduced'] = { + 'new-requirement': 0, + 'changed-by': urgent_hints[0].user + } + return PolicyVerdict.PASS_HINTED + else: + return PolicyVerdict.REJECTED_TEMPORARILY + + return PolicyVerdict.PASS + + def _read_dates_file(self): + """Parse the dates file""" + dates = self._dates + filename = os.path.join(self.options.testing, 'Dates') + with open(filename, encoding='utf-8') as fd: + for line in fd: + # + l = line.split() + if len(l) != 3: + continue + try: + dates[l[0]] = (l[1], int(l[2])) + except ValueError: + pass + + def _read_urgencies_file(self, britney): + urgencies = self._urgencies + filename = os.path.join(self.options.testing, 'Urgency') + min_days_default = self._min_days_default + with open(filename, errors='surrogateescape', encoding='ascii') as fd: + for line in fd: + # + l = line.split() + if len(l) != 3: + continue + + # read the minimum days associated with the urgencies + urgency_old = urgencies.get(l[0], None) + mindays_old = self._min_days.get(urgency_old, 1000) + mindays_new = self._min_days.get(l[2], min_days_default) + + # if the new urgency is lower (so the min days are higher), do nothing + if mindays_old <= mindays_new: + continue + + # if the package exists in testing and it is more recent, do nothing + tsrcv = britney.sources['testing'].get(l[0], None) + if tsrcv and apt_pkg.version_compare(tsrcv[VERSION], l[1]) >= 0: + continue + + # if the package doesn't exist in unstable or it is older, do nothing + usrcv = britney.sources['unstable'].get(l[0], None) + if not usrcv or apt_pkg.version_compare(usrcv[VERSION], l[1]) < 0: + continue + + # update the urgency for the package + urgencies[l[0]] = l[2] + + def _write_dates_file(self): + dates = self._dates + directory = self.options.testing + filename = os.path.join(directory, 'Dates') + filename_tmp = os.path.join(directory, 'Dates_new') + with open(filename_tmp, 'w', encoding='utf-8') as fd: + for pkg in sorted(dates): + version, date = dates[pkg] + fd.write("%s %s %d\n" % (pkg, version, date)) + os.rename(filename_tmp, filename)