mirror of
https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu
synced 2025-02-23 11:21:13 +00:00
autopkgtest: remember old results instead of discarting them
For packages with lots of reverse dependencies, new versions of those reverse dependencies may keep on showing up in testing. If migration is blocked until the results for these new version, migration may take extremely long. If there are results for the current trigger but for the previous version of the reverse dependency, use those until the fresh resuts are available. Similar for the reference runs.
This commit is contained in:
Paul Gevers
parent
6e6214efe8
commit
dc4ea6ca2c
@ -45,12 +45,18 @@ class Result(Enum):
|
||||
PASS = 2
|
||||
NEUTRAL = 3
|
||||
NONE = 4
|
||||
OLD_FAIL = 5
|
||||
OLD_PASS = 6
|
||||
OLD_NEUTRAL = 7
|
||||
|
||||
|
||||
EXCUSES_LABELS = {
|
||||
"PASS": '<span style="background:#87d96c">Pass</span>',
|
||||
"OLD_PASS": '<span style="background:#87d96c">Pass</span>',
|
||||
"NEUTRAL": '<span style="background:#e5c545">No test results</span>',
|
||||
"OLD_NEUTRAL": '<span style="background:#e5c545">No test results</span>',
|
||||
"FAIL": '<span style="background:#ff6666">Failed</span>',
|
||||
"OLD_FAIL": '<span style="background:#ff6666">Failed</span>',
|
||||
"ALWAYSFAIL": '<span style="background:#e5c545">Not a regression</span>',
|
||||
"REGRESSION": '<span style="background:#ff6666">Regression</span>',
|
||||
"IGNORE-FAIL": '<span style="background:#e5c545">Ignored failure</span>',
|
||||
@ -88,6 +94,18 @@ def all_leaf_results(test_results):
|
||||
yield from arch.values()
|
||||
|
||||
|
||||
def mark_result_as_old(result):
|
||||
'''Convert current result into corresponding old result'''
|
||||
|
||||
if result == Result.FAIL:
|
||||
result = Result.OLD_FAIL
|
||||
elif result == Result.PASS:
|
||||
result = Result.OLD_PASS
|
||||
elif result == Result.NEUTRAL:
|
||||
result = Result.OLD_NEUTRAL
|
||||
return result
|
||||
|
||||
|
||||
class AutopkgtestPolicy(BasePolicy):
|
||||
"""autopkgtest regression policy for source migrations
|
||||
|
||||
@ -281,22 +299,15 @@ class AutopkgtestPolicy(BasePolicy):
|
||||
'''
|
||||
|
||||
test_results = self.test_results
|
||||
test_results_new = deepcopy(test_results)
|
||||
|
||||
for (trigger, trigger_data) in test_results.items():
|
||||
for (src, results) in trigger_data.items():
|
||||
for (arch, result) in results.items():
|
||||
if trigger == REF_TRIG and \
|
||||
result[3] < self._now - self.options.adt_reference_max_age:
|
||||
del test_results_new[trigger][src][arch]
|
||||
result[0] = mark_result_as_old(result[0])
|
||||
elif not self.test_version_in_any_suite(src, result[1]):
|
||||
del test_results_new[trigger][src][arch]
|
||||
if len(test_results_new[trigger][src]) == 0:
|
||||
del test_results_new[trigger][src]
|
||||
if len(test_results_new[trigger]) == 0:
|
||||
del test_results_new[trigger]
|
||||
|
||||
self.test_results = test_results_new
|
||||
result[0] = mark_result_as_old(result[0])
|
||||
|
||||
def test_version_in_any_suite(self, src, version):
|
||||
'''Check if the mentioned version of src is found in a suite
|
||||
@ -946,10 +957,12 @@ class AutopkgtestPolicy(BasePolicy):
|
||||
result_state = result[0]
|
||||
version = result[1]
|
||||
baseline = self.result_in_baseline(src, arch)
|
||||
if result_state == Result.FAIL and \
|
||||
baseline[0] in {Result.PASS, Result.NEUTRAL} and \
|
||||
self.options.adt_retry_older_than and \
|
||||
result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now:
|
||||
if result_state in {Result.OLD_PASS, Result.OLD_FAIL, Result.OLD_NEUTRAL}:
|
||||
pass
|
||||
elif result_state == Result.FAIL and \
|
||||
baseline[0] in {Result.PASS, Result.NEUTRAL, Result.OLD_PASS, Result.OLD_NEUTRAL} and \
|
||||
self.options.adt_retry_older_than and \
|
||||
result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now:
|
||||
# We might want to retry this failure, so continue
|
||||
pass
|
||||
elif not uses_swift:
|
||||
@ -1045,7 +1058,7 @@ class AutopkgtestPolicy(BasePolicy):
|
||||
ver = r[1]
|
||||
run_id = r[2]
|
||||
|
||||
if r[0] == Result.FAIL:
|
||||
if r[0] in {Result.FAIL, Result.OLD_FAIL}:
|
||||
# Special-case triggers from linux-meta*: we cannot compare
|
||||
# results against different kernels, as e. g. a DKMS module
|
||||
# might work against the default kernel but fail against a
|
||||
@ -1057,9 +1070,7 @@ class AutopkgtestPolicy(BasePolicy):
|
||||
|
||||
if baseline_result == Result.FAIL:
|
||||
result = 'ALWAYSFAIL'
|
||||
elif self.has_force_badtest(src, ver, arch):
|
||||
result = 'IGNORE-FAIL'
|
||||
elif baseline_result == Result.NONE:
|
||||
elif baseline_result in {Result.NONE, Result.OLD_FAIL}:
|
||||
# Check if the autopkgtest exists in the target suite and request it
|
||||
test_in_target = False
|
||||
try:
|
||||
@ -1070,11 +1081,17 @@ class AutopkgtestPolicy(BasePolicy):
|
||||
pass
|
||||
if test_in_target:
|
||||
self.request_test_if_not_queued(src, arch, REF_TRIG)
|
||||
result = 'RUNNING-REFERENCE'
|
||||
if baseline_result == Result.NONE:
|
||||
result = 'RUNNING-REFERENCE'
|
||||
else:
|
||||
result = 'ALWAYSFAIL'
|
||||
else:
|
||||
result = 'REGRESSION'
|
||||
else:
|
||||
result = 'REGRESSION'
|
||||
|
||||
if self.has_force_badtest(src, ver, arch):
|
||||
result = 'IGNORE-FAIL'
|
||||
else:
|
||||
result = r[0].name
|
||||
|
||||
|
||||
@ -560,19 +560,17 @@ class TestAutopkgtestPolicy(unittest.TestCase):
|
||||
amqp = self.read_amqp()
|
||||
assert amqp[0:-1] == 'debci-testing-amd64:' + src_name + ' {"triggers": ["' + src_name + '/2.0 broken/2.0"]}'
|
||||
|
||||
def test_fail_old_test_result(self):
|
||||
def test_remember_old_test_result(self):
|
||||
src_name = 'broken'
|
||||
policy = initialize_policy(
|
||||
'autopkgtest/fail-old-test-result',
|
||||
'autopkgtest/remember-old-test-result',
|
||||
AutopkgtestPolicy,
|
||||
adt_amqp=self.amqp,
|
||||
pkg_universe=breaks_universe,
|
||||
inst_tester=breaks_inst_tester,
|
||||
adt_baseline='reference')
|
||||
autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name)
|
||||
autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name)
|
||||
assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'PASS'
|
||||
assert autopkgtest_policy_info['inter'][ARCH][0] == 'RUNNING'
|
||||
assert autopkgtest_policy_info['inter'][ARCH][1] == 'status/pending'
|
||||
amqp = self.read_amqp()
|
||||
assert amqp[0:-1] == 'debci-testing-amd64:inter {"triggers": ["' + src_name + '/2.0"]}'
|
||||
|
||||
@ -595,7 +593,7 @@ class TestAutopkgtestPolicy(unittest.TestCase):
|
||||
amqp = self.read_amqp()
|
||||
assert len(amqp) == 0
|
||||
|
||||
def test_fail_reference_too_old(self):
|
||||
def test_reference_too_old(self):
|
||||
src_name = 'pkg'
|
||||
policy = initialize_policy(
|
||||
'autopkgtest/fail-to-fail',
|
||||
@ -606,8 +604,8 @@ class TestAutopkgtestPolicy(unittest.TestCase):
|
||||
adt_reference_max_age=1,
|
||||
pkg_universe=simple_universe,
|
||||
inst_tester=simple_inst_tester)
|
||||
autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.REJECTED_TEMPORARILY, src_name)
|
||||
assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'RUNNING-REFERENCE'
|
||||
autopkgtest_policy_info = apply_src_policy(policy, PolicyVerdict.PASS, src_name)
|
||||
assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][0] == 'ALWAYSFAIL'
|
||||
assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][1] == \
|
||||
'data/autopkgtest/testing/amd64/' + src_name[0] + '/' + src_name + '/2/log.gz'
|
||||
assert autopkgtest_policy_info[src_name + '/2.0'][ARCH][2] == \
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user