Move age-handling into a separate file

Signed-off-by: Niels Thykier <niels@thykier.net>

britney.py

@@ -57,7 +57,7 @@ Other than source and binary packages, Britney loads the following data:
     of a source package (see Britney.read_dates).
 
   * Urgencies, which contains the urgency of the upload of a given
-    version of a source package (see Britney.read_urgencies).
+    version of a source package (see AgePolicy._read_urgencies).
 
   * Hints, which contains lists of commands which modify the standard behaviour
     of Britney (see Britney.read_hints).
@@ -206,6 +206,7 @@ from britney_util import (old_libraries_format, undo_changes,
                           write_excuses, write_heidi_delta, write_controlfiles,
                           old_libraries, is_nuninst_asgood_generous,
                           clone_nuninst, check_installability)
+from policies.policy import AgePolicy, PolicyVerdict
 from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
                    SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
                    PROVIDES, MULTIARCH, ESSENTIAL)
@@ -248,10 +249,9 @@ class Britney(object):
         This method initializes and populates the data lists, which contain all
         the information needed by the other methods of the class.
         """
-        # britney's "day" begins at 3pm
-        self.date_now = int(((time.time() / (60*60)) - 15) / 24)
 
         # parse the command line arguments
+        self.policies = []
         self.__parse_arguments()
         MigrationItem.set_architectures(self.options.architectures)
 
@@ -336,10 +336,9 @@ class Britney(object):
         self.bugs = {'unstable': self.read_bugs(self.options.unstable),
                      'testing': self.read_bugs(self.options.testing),}
         self.normalize_bugs()
-
+        for policy in self.policies:
-        # read additional data
+            policy.hints = self.hints
-        self.dates = self.read_dates(self.options.testing)
+            policy.initialise(self)
-        self.urgencies = self.read_urgencies(self.options.testing)
 
     def merge_pkg_entries(self, package, parch, pkg_entry1, pkg_entry2,
                           check_fields=check_fields, check_field_name=check_field_name):
@@ -402,7 +401,7 @@ class Britney(object):
 
         # minimum days for unstable-testing transition and the list of hints
         # are handled as an ad-hoc case
-        self.MINDAYS = {}
+        MINDAYS = {}
         self.HINTS = {'command-line': self.HINTS_ALL}
         with open(self.options.config, encoding='utf-8') as config:
             for line in config:
@@ -411,7 +410,7 @@ class Britney(object):
                     k = k.strip()
                     v = v.strip()
                     if k.startswith("MINDAYS_"):
-                        self.MINDAYS[k.split("_")[1].lower()] = int(v)
+                        MINDAYS[k.split("_")[1].lower()] = int(v)
                     elif k.startswith("HINTS_"):
                         self.HINTS[k.split("_")[1].lower()] = \
                             reduce(lambda x,y: x+y, [hasattr(self, "HINTS_" + i) and getattr(self, "HINTS_" + i) or (i,) for i in v.split()])
@@ -452,6 +451,8 @@ class Britney(object):
             self.options.ignore_cruft == "0":
             self.options.ignore_cruft = False
 
+        self.policies.append(AgePolicy(self.options, MINDAYS))
+
     def log(self, msg, type="I"):
         """Print info messages according to verbosity level
         
@@ -863,90 +864,6 @@ class Britney(object):
             if maxvert is None:
                 self.bugs['testing'][pkg] = []
 
-    def read_dates(self, basedir):
-        """Read the upload date for the packages from the specified directory
-        
-        The upload dates are read from the `Dates' file within the directory
-        specified as `basedir' parameter. The file contains rows with the
-        format:
-
-        <package-name> <version> <date-of-upload>
-
-        The dates are expressed as the number of days from 1970-01-01.
-
-        The method returns a dictionary where the key is the binary package
-        name and the value is a tuple with two items, the version and the date.
-        """
-        dates = {}
-        filename = os.path.join(basedir, "Dates")
-        self.log("Loading upload data from %s" % filename)
-        for line in open(filename, encoding='ascii'):
-            l = line.split()
-            if len(l) != 3: continue
-            try:
-                dates[l[0]] = (l[1], int(l[2]))
-            except ValueError:
-                self.log("Dates, unable to parse \"%s\"" % line, type="E")
-        return dates
-
-    def write_dates(self, basedir, dates):
-        """Write the upload date for the packages to the specified directory
-
-        For a more detailed explanation of the format, please check the method
-        read_dates.
-        """
-        filename = os.path.join(basedir, "Dates")
-        self.log("Writing upload data to %s" % filename)
-        with open(filename, 'w', encoding='utf-8') as f:
-            for pkg in sorted(dates):
-                f.write("%s %s %d\n" % ((pkg,) + dates[pkg]))
-
-
-    def read_urgencies(self, basedir):
-        """Read the upload urgency of the packages from the specified directory
-        
-        The upload urgencies are read from the `Urgency' file within the
-        directory specified as `basedir' parameter. The file contains rows
-        with the format:
-
-        <package-name> <version> <urgency>
-
-        The method returns a dictionary where the key is the binary package
-        name and the value is the greatest urgency from the versions of the
-        package that are higher then the testing one.
-        """
-
-        urgencies = {}
-        filename = os.path.join(basedir, "Urgency")
-        self.log("Loading upload urgencies from %s" % filename)
-        for line in open(filename, errors='surrogateescape', encoding='ascii'):
-            l = line.split()
-            if len(l) != 3: continue
-
-            # read the minimum days associated with the urgencies
-            urgency_old = urgencies.get(l[0], None)
-            mindays_old = self.MINDAYS.get(urgency_old, 1000)
-            mindays_new = self.MINDAYS.get(l[2], self.MINDAYS[self.options.default_urgency])
-
-            # if the new urgency is lower (so the min days are higher), do nothing
-            if mindays_old <= mindays_new:
-                continue
-
-            # if the package exists in testing and it is more recent, do nothing
-            tsrcv = self.sources['testing'].get(l[0], None)
-            if tsrcv and apt_pkg.version_compare(tsrcv[VERSION], l[1]) >= 0:
-                continue
-
-            # if the package doesn't exist in unstable or it is older, do nothing
-            usrcv = self.sources['unstable'].get(l[0], None)
-            if not usrcv or apt_pkg.version_compare(usrcv[VERSION], l[1]) < 0:
-                continue
-
-            # update the urgency for the package
-            urgencies[l[0]] = l[2]
-
-        return urgencies
-
     def read_hints(self, basedir):
         """Read the hint commands from the specified directory
         
@@ -1357,13 +1274,6 @@ class Britney(object):
             excuse.addhtml("%s source package doesn't exist" % (src))
             update_candidate = False
 
-        # retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing)
-        urgency = self.urgencies.get(src, self.options.default_urgency)
-        if not source_t:
-            if self.MINDAYS[urgency] < self.MINDAYS[self.options.default_urgency]:
-                excuse.addhtml("Ignoring %s urgency setting for NEW package" % (urgency))
-                urgency = self.options.default_urgency
-
         # if there is a `remove' hint and the requested version is the same as the
         # version in testing, then stop here and return False
         for item in self.hints.search('remove', package=src):
@@ -1424,30 +1334,32 @@ class Britney(object):
         # permanence in unstable before updating testing; if the source package is too young,
         # the check fails and we set update_candidate to False to block the update; consider
         # the age-days hint, if specified for the package
-        if suite == 'unstable':
+        policy_info = excuse.policy_info
-            if src not in self.dates:
+        policy_verdict = PolicyVerdict.PASS
-                self.dates[src] = (source_u[VERSION], self.date_now)
+        for policy in self.policies:
-            elif self.dates[src][0] != source_u[VERSION]:
+            if suite in policy.applicable_suites:
-                self.dates[src] = (source_u[VERSION], self.date_now)
+                v = policy.apply_policy(policy_info, suite, src, source_t, source_u)
+                if v.value > policy_verdict.value:
+                    policy_verdict = v
 
-            days_old = self.date_now - self.dates[src][1]
+        if policy_verdict.is_rejected:
-            min_days = self.MINDAYS[urgency]
-
-            for age_days_hint in [x for x in self.hints.search('age-days', package=src)
-                                  if source_u[VERSION] == x.version]:
-                excuse.addhtml("Overriding age needed from %d days to %d by %s" % (min_days,
-                    int(age_days_hint.days), age_days_hint.user))
-                min_days = int(age_days_hint.days)
-
-            excuse.setdaysold(days_old, min_days)
-            if days_old < min_days:
-                urgent_hints = [x for x in self.hints.search('urgent', package=src)
-                                if source_u[VERSION] == x.version]
-                if urgent_hints:
-                    excuse.addhtml("Too young, but urgency pushed by %s" % (urgent_hints[0].user))
-                else:
             update_candidate = False
-                    excuse.addreason("age")
+
+        # Joggle some things into excuses
+        # - remove once the YAML is the canonical source for this information
+        if 'age' in policy_info:
+            age_info = policy_info['age']
+            age_hint = age_info.get('age-requirement-reduced', None)
+            age_min_req = age_info['age-requirement']
+            if age_hint:
+                new_req = age_hint['new-requirement']
+                who = age_hint['changed-by']
+                if new_req:
+                    excuse.addhtml("Overriding age needed from %d days to %d by %s" % (
+                        age_min_req, new_req, who))
+                else:
+                    excuse.addhtml("Too young, but urgency pushed by %s" % who)
+            excuse.setdaysold(age_info['current-age'], age_min_req)
 
         all_binaries = self.all_binaries
 
@@ -1560,7 +1472,7 @@ class Britney(object):
                         excuse.addreason("build-arch")
                         excuse.addreason("build-arch-%s" % arch)
 
-                if self.date_now != self.dates[src][1]:
+                if 'age' in policy_info and policy_info['age']['current-age']:
                     excuse.addhtml(text)
 
         # if the source package has no binaries, set update_candidate to False to block the update
@@ -2656,11 +2568,8 @@ class Britney(object):
                 write_controlfiles(self.sources, self.binaries,
                                    'testing', self.options.testing)
 
-            # write dates
+            for policy in self.policies:
-            try:
+                policy.save_state(self)
-                self.write_dates(self.options.outputdir, self.dates)
-            except AttributeError:
-                self.write_dates(self.options.testing, self.dates)
 
             # write HeidiResult
             self.log("Writing Heidi results to %s" % self.options.heidi_output)

excuse.py

@@ -59,6 +59,7 @@ class Excuse(object):
         self.oldbugs = set()
         self.reason = {}
         self.htmlline = []
+        self.policy_info = {}
 
     def sortkey(self):
         if self.daysold == None:

policies/policy.py

@@ -0,0 +1,242 @@
+from abc import abstractmethod
+from enum import Enum, unique
+import apt_pkg
+import os
+import time
+
+from consts import VERSION
+
+
+@unique
+class PolicyVerdict(Enum):
+    """"""
+    """
+    The migration item passed the policy.
+    """
+    PASS = 1
+    """
+    The policy was completely overruled by a hint.
+    """
+    PASS_HINTED = 2
+    """
+    The migration item did not pass the policy, but the failure is believed
+    to be temporary
+    """
+    REJECTED_TEMPORARILY = 3
+    """
+    The migration item did not pass the policy and the failure is believed
+    to be uncorrectable (i.e. a hint or a new version is needed)
+    """
+    REJECTED_PERMANENTLY = 4
+
+    @property
+    def is_rejected(self):
+        return True if self.name.startswith('REJECTED') else False
+
+
+class BasePolicy(object):
+
+    def __init__(self, options, applicable_suites):
+        self.options = options
+        self.applicable_suites = applicable_suites
+        self.hints = None
+
+    # FIXME: use a proper logging framework
+    def log(self, msg, type="I"):
+        """Print info messages according to verbosity level
+
+        An easy-and-simple log method which prints messages to the standard
+        output. The type parameter controls the urgency of the message, and
+        can be equal to `I' for `Information', `W' for `Warning' and `E' for
+        `Error'. Warnings and errors are always printed, and information is
+        printed only if verbose logging is enabled.
+        """
+        if self.options.verbose or type in ("E", "W"):
+            print("%s: [%s] - %s" % (type, time.asctime(), msg))
+
+    def initialise(self, britney):
+        """Called once to make the policy initialise any data structures
+
+        This is useful for e.g. parsing files or other "heavy do-once" work.
+        """
+        pass
+
+    def save_state(self, britney):
+        """Called once at the end of the run to make the policy save any persistent data
+
+        Note this will *not* be called for "dry-runs" as such runs should not change
+        the state.
+        """
+        pass
+
+    @abstractmethod
+    def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist):
+        pass
+
+
+class AgePolicy(BasePolicy):
+    """Configurable Aging policy for source migrations
+
+    The AgePolicy will let packages stay in the source suite for a pre-defined
+    amount of days before letting migrate (based on their urgency, if any).
+
+    The AgePolicy's decision is influenced by the following:
+
+    State files:
+     * ${TESTING}/Urgency: File containing urgencies for source packages.
+       Note that urgencies are "sticky" and the most "urgent" urgency will be
+       used (i.e. the one with lowest age-requirements).
+       - This file needs to be updated externally, if the policy should take
+         urgencies into consideration.  If empty (or not updated), the policy
+         will simply use the default urgency (see the "Config" section below)
+       - In Debian, these values are taken from the .changes file, but that is
+         not a requirement for Britney.
+     * ${TESTING}/Dates: File containing the age of all source packages.
+       - The policy will automatically update this file.
+    Config:
+     * DEFAULT_URGENCY: Name of the urgency used for packages without an urgency
+       (or for unknown urgencies).  Will also  be used to set the "minimum"
+       aging requirements for packages not in the target suite.
+     * MINDAYS_<URGENCY>: The age-requirements in days for packages with the
+       given urgency.
+       - Commonly used urgencies are: low, medium, high, emergency, critical
+    Hints:
+     * urgent <source>/<version>: Disregard the age requirements for a given
+       source/version.
+     * age-days X <source>/<version>: Set the age requirements for a given
+       source/version to X days.  Note that X can exceed the highest
+       age-requirement normally given.
+
+    """
+
+    def __init__(self, options, mindays):
+        super().__init__(options, {'unstable'})
+        self._min_days = mindays
+        if options.default_urgency not in mindays:
+            raise ValueError("Missing age-requirement for default urgency (MINDAYS_%s)" % options.default_urgency)
+        self._min_days_default = mindays[options.default_urgency]
+        # britney's "day" begins at 3pm
+        self._date_now = int(((time.time() / (60*60)) - 15) / 24)
+        self._dates = {}
+        self._urgencies = {}
+
+    def initialise(self, britney):
+        super().initialise(britney)
+        self._read_dates_file()
+        self._read_urgencies_file(britney)
+
+    def save_state(self, britney):
+        super().save_state(britney)
+        self._write_dates_file()
+
+    def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist):
+        # retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing)
+        urgency = self._urgencies.get(source_name, self.options.default_urgency)
+        if 'age' not in policy_info:
+            policy_info['age'] = age_info = {}
+        else:
+            age_info = policy_info['age']
+
+        if urgency not in self._min_days:
+            age_info['unknown-urgency'] = urgency
+            urgency = self.options.default_urgency
+
+        if not source_data_tdist:
+            if self._min_days[urgency] < self._min_days_default:
+                age_info['urgency-reduced'] = {
+                    'from': urgency,
+                    'to': self.options.default_urgency,
+                }
+                urgency = self.options.default_urgency
+
+        if source_name not in self._dates:
+            self._dates[source_name] = (source_data_srcdist[VERSION], self._date_now)
+        elif self._dates[source_name][0] != source_data_srcdist[VERSION]:
+            self._dates[source_name] = (source_data_srcdist[VERSION], self._date_now)
+
+        days_old = self._date_now - self._dates[source_name][1]
+        min_days = self._min_days[urgency]
+        age_info['age-requirement'] = min_days
+        age_info['current-age'] = days_old
+
+        for age_days_hint in [x for x in self.hints.search('age-days', package=source_name)
+                              if source_data_srcdist[VERSION] == x.version]:
+            new_req = int(age_days_hint.days)
+            age_info['age-requirement-reduced'] = {
+                'new-requirement': new_req,
+                'changed-by': age_days_hint.user
+            }
+            min_days = new_req
+
+        if days_old < min_days:
+            urgent_hints = [x for x in self.hints.search('urgent', package=source_name)
+                            if source_data_srcdist[VERSION] == x.version]
+            if urgent_hints:
+                age_info['age-requirement-reduced'] = {
+                    'new-requirement': 0,
+                    'changed-by': urgent_hints[0].user
+                }
+                return PolicyVerdict.PASS_HINTED
+            else:
+                return PolicyVerdict.REJECTED_TEMPORARILY
+
+        return PolicyVerdict.PASS
+
+    def _read_dates_file(self):
+        """Parse the dates file"""
+        dates = self._dates
+        filename = os.path.join(self.options.testing, 'Dates')
+        with open(filename, encoding='utf-8') as fd:
+            for line in fd:
+                # <source> <version> <date>
+                l = line.split()
+                if len(l) != 3:
+                    continue
+                try:
+                    dates[l[0]] = (l[1], int(l[2]))
+                except ValueError:
+                    pass
+
+    def _read_urgencies_file(self, britney):
+        urgencies = self._urgencies
+        filename = os.path.join(self.options.testing, 'Urgency')
+        min_days_default = self._min_days_default
+        with open(filename, errors='surrogateescape', encoding='ascii') as fd:
+            for line in fd:
+                # <source> <version> <urgency>
+                l = line.split()
+                if len(l) != 3:
+                    continue
+
+                # read the minimum days associated with the urgencies
+                urgency_old = urgencies.get(l[0], None)
+                mindays_old = self._min_days.get(urgency_old, 1000)
+                mindays_new = self._min_days.get(l[2], min_days_default)
+
+                # if the new urgency is lower (so the min days are higher), do nothing
+                if mindays_old <= mindays_new:
+                    continue
+
+                # if the package exists in testing and it is more recent, do nothing
+                tsrcv = britney.sources['testing'].get(l[0], None)
+                if tsrcv and apt_pkg.version_compare(tsrcv[VERSION], l[1]) >= 0:
+                    continue
+
+                # if the package doesn't exist in unstable or it is older, do nothing
+                usrcv = britney.sources['unstable'].get(l[0], None)
+                if not usrcv or apt_pkg.version_compare(usrcv[VERSION], l[1]) < 0:
+                    continue
+
+                # update the urgency for the package
+                urgencies[l[0]] = l[2]
+
+    def _write_dates_file(self):
+        dates = self._dates
+        directory = self.options.testing
+        filename = os.path.join(directory, 'Dates')
+        filename_tmp = os.path.join(directory, 'Dates_new')
+        with open(filename_tmp, 'w', encoding='utf-8') as fd:
+            for pkg in sorted(dates):
+                version, date = dates[pkg]
+                fd.write("%s %s %d\n" % (pkg, version, date))
+        os.rename(filename_tmp, filename)