autopkgtest: remember old results instead of discarting them

For packages with lots of reverse dependencies, new versions of those reverse
dependencies may keep on showing up in testing. If migration is blocked until
the results for these new version, migration may take extremely long. If there
are results for the current trigger but for the previous version of the reverse
dependency, use those until the fresh resuts are available.

Similar for the reference runs.

britney2/policies/autopkgtest.py

@@ -45,12 +45,18 @@ class Result(Enum):
     PASS = 2
     NEUTRAL = 3
     NONE = 4
+    OLD_FAIL = 5
+    OLD_PASS = 6
+    OLD_NEUTRAL = 7
 
 
 EXCUSES_LABELS = {
     "PASS": '<span style="background:#87d96c">Pass</span>',
+    "OLD_PASS": '<span style="background:#87d96c">Pass</span>',
     "NEUTRAL": '<span style="background:#e5c545">No test results</span>',
+    "OLD_NEUTRAL": '<span style="background:#e5c545">No test results</span>',
     "FAIL": '<span style="background:#ff6666">Failed</span>',
+    "OLD_FAIL": '<span style="background:#ff6666">Failed</span>',
     "ALWAYSFAIL": '<span style="background:#e5c545">Not a regression</span>',
     "REGRESSION": '<span style="background:#ff6666">Regression</span>',
     "IGNORE-FAIL": '<span style="background:#e5c545">Ignored failure</span>',
@@ -88,6 +94,18 @@ def all_leaf_results(test_results):
             yield from arch.values()
 
 
+def mark_result_as_old(result):
+    '''Convert current result into corresponding old result'''
+
+    if result == Result.FAIL:
+        result = Result.OLD_FAIL
+    elif result == Result.PASS:
+        result = Result.OLD_PASS
+    elif result == Result.NEUTRAL:
+        result = Result.OLD_NEUTRAL
+    return result
+
+
 class AutopkgtestPolicy(BasePolicy):
     """autopkgtest regression policy for source migrations
 
@@ -281,22 +299,15 @@ class AutopkgtestPolicy(BasePolicy):
 '''
 
         test_results = self.test_results
-        test_results_new = deepcopy(test_results)
 
         for (trigger, trigger_data) in test_results.items():
             for (src, results) in trigger_data.items():
                 for (arch, result) in results.items():
                     if trigger == REF_TRIG and \
                       result[3] < self._now - self.options.adt_reference_max_age:
-                        del test_results_new[trigger][src][arch]
+                        result[0] = mark_result_as_old(result[0])
                     elif not self.test_version_in_any_suite(src, result[1]):
-                        del test_results_new[trigger][src][arch]
+                        result[0] = mark_result_as_old(result[0])
-                if len(test_results_new[trigger][src]) == 0:
-                    del test_results_new[trigger][src]
-            if len(test_results_new[trigger]) == 0:
-                del test_results_new[trigger]
-
-        self.test_results = test_results_new
 
     def test_version_in_any_suite(self, src, version):
         '''Check if the mentioned version of src is found in a suite
@@ -946,10 +957,12 @@ class AutopkgtestPolicy(BasePolicy):
             result_state = result[0]
             version = result[1]
             baseline = self.result_in_baseline(src, arch)
-            if result_state == Result.FAIL and \
+            if result_state in {Result.OLD_PASS, Result.OLD_FAIL, Result.OLD_NEUTRAL}:
-               baseline[0] in {Result.PASS, Result.NEUTRAL} and \
+                pass
-               self.options.adt_retry_older_than and \
+            elif result_state == Result.FAIL and \
-               result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now:
+                    baseline[0] in {Result.PASS, Result.NEUTRAL, Result.OLD_PASS, Result.OLD_NEUTRAL} and \
+                    self.options.adt_retry_older_than and \
+                    result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now:
                 # We might want to retry this failure, so continue
                 pass
             elif not uses_swift:
@@ -1045,7 +1058,7 @@ class AutopkgtestPolicy(BasePolicy):
             ver = r[1]
             run_id = r[2]
 
-            if r[0] == Result.FAIL:
+            if r[0] in {Result.FAIL, Result.OLD_FAIL}:
                 # Special-case triggers from linux-meta*: we cannot compare
                 # results against different kernels, as e. g. a DKMS module
                 # might work against the default kernel but fail against a
@@ -1057,9 +1070,7 @@ class AutopkgtestPolicy(BasePolicy):
 
                 if baseline_result == Result.FAIL:
                     result = 'ALWAYSFAIL'
-                elif self.has_force_badtest(src, ver, arch):
+                elif baseline_result in {Result.NONE, Result.OLD_FAIL}:
-                    result = 'IGNORE-FAIL'
-                elif baseline_result == Result.NONE:
                     # Check if the autopkgtest exists in the target suite and request it
                     test_in_target = False
                     try:
@@ -1070,11 +1081,17 @@ class AutopkgtestPolicy(BasePolicy):
                         pass
                     if test_in_target:
                         self.request_test_if_not_queued(src, arch, REF_TRIG)
-                        result = 'RUNNING-REFERENCE'
+                        if baseline_result == Result.NONE:
+                            result = 'RUNNING-REFERENCE'
+                        else:
+                            result = 'ALWAYSFAIL'
                     else:
                         result = 'REGRESSION'
                 else:
                     result = 'REGRESSION'
+
+                if self.has_force_badtest(src, ver, arch):
+                    result = 'IGNORE-FAIL'
             else:
                 result = r[0].name
 

tests/test_policy.py

@@ -560,19 +560,17 @@ class TestAutopkgtestPolicy(unittest.TestCase):
         amqp = self.read_amqp()
         assert amqp[0:-1] == 'debci-testing-amd64:' + src_name + ' {"triggers": ["' + src_name + '/2.0 broken/2.0"]}'
 
-    def test_fail_old_test_result(self):
+    def test_remember_old_test_result(self):
         src_name = 'broken'
         policy = initialize_policy(
-            'autopkgtest/fail-old-test-result',
+            'autopkgtest/remember-old-test-result',
             AutopkgtestPolicy,
             adt_amqp=self.amqp,
             pkg_universe=breaks_universe,
             inst_tester=breaks_inst_tester,
             adt_baseline='reference')
-        autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name)
+        autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name)
         assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'PASS'
-        assert autopkgtest_policy_info['inter'][ARCH][0] == 'RUNNING'
-        assert autopkgtest_policy_info['inter'][ARCH][1] == 'status/pending'
         amqp = self.read_amqp()
         assert amqp[0:-1] == 'debci-testing-amd64:inter {"triggers": ["' + src_name + '/2.0"]}'
 
@@ -595,7 +593,7 @@ class TestAutopkgtestPolicy(unittest.TestCase):
         amqp = self.read_amqp()
         assert len(amqp) == 0
 
-    def test_fail_reference_too_old(self):
+    def test_reference_too_old(self):
         src_name = 'pkg'
         policy = initialize_policy(
             'autopkgtest/fail-to-fail',
@@ -606,8 +604,8 @@ class TestAutopkgtestPolicy(unittest.TestCase):
             adt_reference_max_age=1,
             pkg_universe=simple_universe,
             inst_tester=simple_inst_tester)
-        autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name)
+        autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name)
-        assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'RUNNING-REFERENCE'
+        assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'ALWAYSFAIL'
         assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][1] == \
             'data/autopkgtest/testing/amd64/' + src_name[0] + '/' + src_name + '/2/log.gz'
         assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][2] == \