From dc4ea6ca2c1a6250d41f8cfd717e14f95fcebfb2 Mon Sep 17 00:00:00 2001 From: Paul Gevers Date: Fri, 1 May 2020 21:34:04 +0200 Subject: [PATCH] autopkgtest: remember old results instead of discarting them For packages with lots of reverse dependencies, new versions of those reverse dependencies may keep on showing up in testing. If migration is blocked until the results for these new version, migration may take extremely long. If there are results for the current trigger but for the previous version of the reverse dependency, use those until the fresh resuts are available. Similar for the reference runs. --- britney2/policies/autopkgtest.py | 53 ++++++++++++------- .../debci.json | 0 tests/test_policy.py | 14 +++-- 3 files changed, 41 insertions(+), 26 deletions(-) rename tests/policy-test-data/autopkgtest/{fail-old-test-result => remember-old-test-result}/debci.json (100%) diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py index 5414dfa..a68b000 100644 --- a/britney2/policies/autopkgtest.py +++ b/britney2/policies/autopkgtest.py @@ -45,12 +45,18 @@ class Result(Enum): PASS = 2 NEUTRAL = 3 NONE = 4 + OLD_FAIL = 5 + OLD_PASS = 6 + OLD_NEUTRAL = 7 EXCUSES_LABELS = { "PASS": 'Pass', + "OLD_PASS": 'Pass', "NEUTRAL": 'No test results', + "OLD_NEUTRAL": 'No test results', "FAIL": 'Failed', + "OLD_FAIL": 'Failed', "ALWAYSFAIL": 'Not a regression', "REGRESSION": 'Regression', "IGNORE-FAIL": 'Ignored failure', @@ -88,6 +94,18 @@ def all_leaf_results(test_results): yield from arch.values() +def mark_result_as_old(result): + '''Convert current result into corresponding old result''' + + if result == Result.FAIL: + result = Result.OLD_FAIL + elif result == Result.PASS: + result = Result.OLD_PASS + elif result == Result.NEUTRAL: + result = Result.OLD_NEUTRAL + return result + + class AutopkgtestPolicy(BasePolicy): """autopkgtest regression policy for source migrations @@ -281,22 +299,15 @@ class AutopkgtestPolicy(BasePolicy): ''' test_results = self.test_results - test_results_new = deepcopy(test_results) for (trigger, trigger_data) in test_results.items(): for (src, results) in trigger_data.items(): for (arch, result) in results.items(): if trigger == REF_TRIG and \ result[3] < self._now - self.options.adt_reference_max_age: - del test_results_new[trigger][src][arch] + result[0] = mark_result_as_old(result[0]) elif not self.test_version_in_any_suite(src, result[1]): - del test_results_new[trigger][src][arch] - if len(test_results_new[trigger][src]) == 0: - del test_results_new[trigger][src] - if len(test_results_new[trigger]) == 0: - del test_results_new[trigger] - - self.test_results = test_results_new + result[0] = mark_result_as_old(result[0]) def test_version_in_any_suite(self, src, version): '''Check if the mentioned version of src is found in a suite @@ -946,10 +957,12 @@ class AutopkgtestPolicy(BasePolicy): result_state = result[0] version = result[1] baseline = self.result_in_baseline(src, arch) - if result_state == Result.FAIL and \ - baseline[0] in {Result.PASS, Result.NEUTRAL} and \ - self.options.adt_retry_older_than and \ - result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now: + if result_state in {Result.OLD_PASS, Result.OLD_FAIL, Result.OLD_NEUTRAL}: + pass + elif result_state == Result.FAIL and \ + baseline[0] in {Result.PASS, Result.NEUTRAL, Result.OLD_PASS, Result.OLD_NEUTRAL} and \ + self.options.adt_retry_older_than and \ + result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now: # We might want to retry this failure, so continue pass elif not uses_swift: @@ -1045,7 +1058,7 @@ class AutopkgtestPolicy(BasePolicy): ver = r[1] run_id = r[2] - if r[0] == Result.FAIL: + if r[0] in {Result.FAIL, Result.OLD_FAIL}: # Special-case triggers from linux-meta*: we cannot compare # results against different kernels, as e. g. a DKMS module # might work against the default kernel but fail against a @@ -1057,9 +1070,7 @@ class AutopkgtestPolicy(BasePolicy): if baseline_result == Result.FAIL: result = 'ALWAYSFAIL' - elif self.has_force_badtest(src, ver, arch): - result = 'IGNORE-FAIL' - elif baseline_result == Result.NONE: + elif baseline_result in {Result.NONE, Result.OLD_FAIL}: # Check if the autopkgtest exists in the target suite and request it test_in_target = False try: @@ -1070,11 +1081,17 @@ class AutopkgtestPolicy(BasePolicy): pass if test_in_target: self.request_test_if_not_queued(src, arch, REF_TRIG) - result = 'RUNNING-REFERENCE' + if baseline_result == Result.NONE: + result = 'RUNNING-REFERENCE' + else: + result = 'ALWAYSFAIL' else: result = 'REGRESSION' else: result = 'REGRESSION' + + if self.has_force_badtest(src, ver, arch): + result = 'IGNORE-FAIL' else: result = r[0].name diff --git a/tests/policy-test-data/autopkgtest/fail-old-test-result/debci.json b/tests/policy-test-data/autopkgtest/remember-old-test-result/debci.json similarity index 100% rename from tests/policy-test-data/autopkgtest/fail-old-test-result/debci.json rename to tests/policy-test-data/autopkgtest/remember-old-test-result/debci.json diff --git a/tests/test_policy.py b/tests/test_policy.py index 1b6286a..9d1ae3b 100644 --- a/tests/test_policy.py +++ b/tests/test_policy.py @@ -560,19 +560,17 @@ class TestAutopkgtestPolicy(unittest.TestCase): amqp = self.read_amqp() assert amqp[0:-1] == 'debci-testing-amd64:' + src_name + ' {"triggers": ["' + src_name + '/2.0 broken/2.0"]}' - def test_fail_old_test_result(self): + def test_remember_old_test_result(self): src_name = 'broken' policy = initialize_policy( - 'autopkgtest/fail-old-test-result', + 'autopkgtest/remember-old-test-result', AutopkgtestPolicy, adt_amqp=self.amqp, pkg_universe=breaks_universe, inst_tester=breaks_inst_tester, adt_baseline='reference') - autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name) + autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name) assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'PASS' - assert autopkgtest_policy_info['inter'][ARCH][0] == 'RUNNING' - assert autopkgtest_policy_info['inter'][ARCH][1] == 'status/pending' amqp = self.read_amqp() assert amqp[0:-1] == 'debci-testing-amd64:inter {"triggers": ["' + src_name + '/2.0"]}' @@ -595,7 +593,7 @@ class TestAutopkgtestPolicy(unittest.TestCase): amqp = self.read_amqp() assert len(amqp) == 0 - def test_fail_reference_too_old(self): + def test_reference_too_old(self): src_name = 'pkg' policy = initialize_policy( 'autopkgtest/fail-to-fail', @@ -606,8 +604,8 @@ class TestAutopkgtestPolicy(unittest.TestCase): adt_reference_max_age=1, pkg_universe=simple_universe, inst_tester=simple_inst_tester) - autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name) - assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'RUNNING-REFERENCE' + autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name) + assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'ALWAYSFAIL' assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][1] == \ 'data/autopkgtest/testing/amd64/' + src_name[0] + '/' + src_name + '/2/log.gz' assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][2] == \