Let britney reschedule migration-reference/0 runs when they are too old

Currently, britney only schedules reference runs when they don't exist. It does
strip out runs against older versions of the autopkgtest, but the current version
may exist for a while and the reference run can be old. So, add an option to
ignore old results.

britney.conf

@@ -105,3 +105,4 @@ ADT_SUCCESS_BOUNTY     = 3
 ADT_BASELINE           = reference
 ADT_RETRY_URL_MECH     = run_id
 ADT_RETRY_OLDER_THAN   = 1
+ADT_REFERENCE_MAX_AGE  = 7

britney2/policies/autopkgtest.py

@@ -159,6 +159,12 @@ class AutopkgtestPolicy(BasePolicy):
             # Make adt_baseline optional
             setattr(self.options, 'adt_baseline', None)
 
+        if not hasattr(self.options, 'adt_reference_max_age'):
+            self.options.adt_reference_max_age = float('inf')
+        else:
+            self.options.adt_reference_max_age = \
+              int(self.options.adt_reference_max_age) * SECPERDAY
+
         # read the cached results that we collected so far
         if os.path.exists(self.results_cache_file):
             with open(self.results_cache_file) as f:
@@ -168,9 +174,10 @@ class AutopkgtestPolicy(BasePolicy):
 
             # The cache can contain results against versions of packages that
             # are not in any suite anymore. Strip those out, as we don't want
-            # to use those results.
+            # to use those results. Additionally, old references may be
+            # filtered out.
             if self.options.adt_baseline == 'reference':
-                self.filter_results_for_old_versions()
+                self.filter_old_results()
 
         else:
             self.logger.info('%s does not exist, re-downloading all results from swift', self.results_cache_file)
@@ -266,8 +273,13 @@ class AutopkgtestPolicy(BasePolicy):
                 result.append(self._now)
         return test_results
 
-    def filter_results_for_old_versions(self):
+    def filter_old_results(self):
-        '''Remove results for old versions from the cache'''
+        '''Remove results for old versions and reference runs from the cache.
+
+        For now, only delete reference runs. If we delete regular
+        results after a while, packages with lots of triggered tests may
+        never have all the results at the same time.
+'''
 
         test_results = self.test_results
         test_results_new = deepcopy(test_results)
@@ -275,6 +287,9 @@ class AutopkgtestPolicy(BasePolicy):
         for (trigger, trigger_data) in test_results.items():
             for (src, results) in trigger_data.items():
                 for (arch, result) in results.items():
+                    if trigger == REF_TRIG and \
+                      result[3] < self._now - self.options.adt_reference_max_age:
+                        del test_results_new[trigger][src][arch]
                     if not self.test_version_in_any_suite(src, result[1]):
                         del test_results_new[trigger][src][arch]
                 if len(test_results_new[trigger][src]) == 0:
@@ -874,6 +889,11 @@ class AutopkgtestPolicy(BasePolicy):
                 "Ignoring result for source %s and trigger %s as the tested version %s isn't found in any suite",
                 src, trigger, ver)
             return
+        if trigger == REF_TRIG and \
+           seen < self._now - self.options.adt_reference_max_age:
+            self.logger.debug(
+                "Ignoring reference result for source %s as it's too old", src)
+            return
 
         result = self.test_results.setdefault(trigger, {}).setdefault(
             src, {}).setdefault(arch, [Result.FAIL, None, '', 0])

tests/test_policy.py

@@ -583,6 +583,7 @@ class TestAutopkgtestPolicy(unittest.TestCase):
             AutopkgtestPolicy,
             adt_amqp=self.amqp,
             adt_retry_older_than=1,
+            adt_baseline='reference',
             pkg_universe=simple_universe,
             inst_tester=simple_inst_tester)
         autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name)
@@ -594,6 +595,26 @@ class TestAutopkgtestPolicy(unittest.TestCase):
         amqp = self.read_amqp()
         assert len(amqp) == 0
 
+    def test_fail_reference_too_old(self):
+        src_name = 'pkg'
+        policy = initialize_policy(
+            'autopkgtest/fail-to-fail',
+            AutopkgtestPolicy,
+            adt_amqp=self.amqp,
+            adt_retry_older_than=1,
+            adt_baseline='reference',
+            adt_reference_max_age=1,
+            pkg_universe=simple_universe,
+            inst_tester=simple_inst_tester)
+        autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name)
+        assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'RUNNING-REFERENCE'
+        assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][1] == \
+            'data/autopkgtest/testing/amd64/' + src_name[0] + '/' + src_name + '/2/log.gz'
+        assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][2] == \
+            'packages/' + src_name[0] + '/' + src_name + '/testing/amd64'
+        amqp = self.read_amqp()
+        assert 'migration-reference/0' in amqp
+
 
 if __name__ == '__main__':
     unittest.main()