From fa1a7b4cdcf8e3bc45960f100a785758ac0db539 Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Wed, 10 Jun 2020 10:30:13 +0100 Subject: [PATCH] autopkgtest: Add new `force-reset-test` hint type This allows hinters to reset the baseline for a package/arch, so that previous failures are ignore and not considered regressions --- britney2/policies/autopkgtest.py | 67 +++++++++- tests/test_autopkgtest.py | 217 +++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 2 deletions(-) diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py index 757f681..b3c6dc5 100644 --- a/britney2/policies/autopkgtest.py +++ b/britney2/policies/autopkgtest.py @@ -158,6 +158,7 @@ class AutopkgtestPolicy(BasePolicy): def register_hints(self, hint_parser): hint_parser.register_hint_type('force-badtest', britney2.hints.split_into_one_hint_per_package) hint_parser.register_hint_type('force-skiptest', britney2.hints.split_into_one_hint_per_package) + hint_parser.register_hint_type('force-reset-test', britney2.hints.split_into_one_hint_per_package) def initialise(self, britney): super().initialise(britney) @@ -1096,6 +1097,11 @@ class AutopkgtestPolicy(BasePolicy): # determine current test result status baseline_result = self.result_in_baseline(src, arch)[0] + # determine current test result status + until = self.find_max_lower_force_reset_test(src, ver, arch) + ever_passed = self.check_ever_passed_before(src, ver, arch, until) + + fail_result = 'REGRESSION' if ever_passed else 'ALWAYSFAIL' url = None run_id = None @@ -1136,9 +1142,13 @@ class AutopkgtestPolicy(BasePolicy): else: result = 'ALWAYSFAIL' else: - result = 'REGRESSION' + result = fail_result else: - result = 'REGRESSION' + result = fail_result + + if result == 'REGRESSION' and \ + self.has_higher_force_reset_test(src, ver, arch): + result = 'ALWAYSFAIL' if self.has_force_badtest(src, ver, arch): result = 'IGNORE-FAIL' @@ -1181,6 +1191,59 @@ class AutopkgtestPolicy(BasePolicy): return (result, ver, run_id, url) + def check_ever_passed_before(self, src, max_ver, arch, min_ver=None): + '''Check if tests for src ever passed on arch for specified range + + If min_ver is specified, it checks that all versions in + [min_ver, max_ver) have passed; otherwise it checks that + [min_ver, inf) have passed.''' + + # FIXME: add caching + for srcmap in self.test_results.values(): + try: + too_high = apt_pkg.version_compare(srcmap[src][arch][1], max_ver) > 0 + too_low = apt_pkg.version_compare(srcmap[src][arch][1], min_ver) <= 0 if min_ver else False + + if too_high or too_low: + continue + + if srcmap[src][arch][0] in (Result.PASS, Result.OLD_PASS): + return True + except KeyError: + pass + return False + + def find_max_lower_force_reset_test(self, src, ver, arch): + '''Find the maximum force-reset-test hint before/including ver''' + hints = self.hints.search('force-reset-test', package=src) + found_ver = None + + if hints: + for hint in hints: + for mi in hint.packages: + if (mi.architecture in ['source', arch] and + mi.version != 'all' and + apt_pkg.version_compare(mi.version, ver) <= 0 and + (found_ver is None or apt_pkg.version_compare(found_ver, mi.version) < 0)): + found_ver = mi.version + + return found_ver + + def has_higher_force_reset_test(self, src, ver, arch): + '''Find if there is a minimum force-reset-test hint after/including ver''' + hints = self.hints.search('force-reset-test', package=src) + + if hints: + self.logger.info('Checking hints for %s/%s/%s: %s' % (src, ver, arch, [str(h) for h in hints])) + for hint in hints: + for mi in hint.packages: + if (mi.architecture in ['source', arch] and + mi.version != 'all' and + apt_pkg.version_compare(mi.version, ver) >= 0): + return True + + return False + def has_force_badtest(self, src, ver, arch): '''Check if src/ver/arch has a force-badtest hint''' diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 67547fc..5067bfe 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -1985,6 +1985,223 @@ class AT(TestAutopkgtestBase): {'lightgreen': [('reason', 'block')]} ) + def test_hint_force_reset_test_goodbad_alwaysfail(self): + '''force-reset-test hint marks as alwaysfail''' + + self.data.add_default_packages(lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/1') + + self.run_it( + [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, { + 'lightgreen/2': {'amd64': 'ALWAYSFAIL'}, + }), + }, + {'lightgreen': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_reset_test_goodbad_alwaysfail_arch(self): + '''force-reset-test hint marks as alwaysfail per arch''' + + self.data.add_default_packages(lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')), + 'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/1/amd64') + + self.run_it( + [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (False, { + 'lightgreen/2': {'amd64': 'ALWAYSFAIL', 'i386': 'REGRESSION'}, + }), + }, + {'lightgreen': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_reset_test_bad_good_pass(self): + '''force-reset-test hint followed by pass is pass''' + + self.data.add_default_packages(lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/1') + + self.run_it( + [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, { + 'lightgreen/2': {'amd64': 'PASS'}, + }), + }, + {'lightgreen': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_reset_test_bad_good_bad_regression(self): + '''force-reset-test hint followed by good, bad is regression''' + + self.data.add_default_packages(lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'testing/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/1') + + self.run_it( + [('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (False, { + 'lightgreen/3': {'amd64': 'REGRESSION'}, + }), + }, + {'lightgreen': [('old-version', '1'), ('new-version', '3')] + }) + + def test_hint_force_reset_test_bad_good_bad_regression_different_trigger(self): + '''force-reset-test hint followed by good, bad is regression (not self-triggered)''' + + self.data.add_default_packages(green=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 0.1', tr('lightgreen/0.1')), + 'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 1', tr('green/2')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/0.1') + + self.run_it( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, { + 'lightgreen/1': {'amd64': 'REGRESSION'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_reset_test_multiple_hints(self): + '''force-reset-test multiple hints check ranges''' + + self.data.add_default_packages(green=False, lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150102_100100@': (4, 'lightgreen 1', tr('green/2')), + 'testing/amd64/l/lightgreen/20150103_100101@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'testing/amd64/l/lightgreen/20150104_100101@': (4, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.run_it( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, { + 'lightgreen/1': {'amd64': 'REGRESSION'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '3')], + }) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/1') + self.run_it( + [], + {'green': (True, { + 'lightgreen/1': {'amd64': 'ALWAYSFAIL'}, + }), + 'lightgreen': (False, { + 'lightgreen/3': {'amd64': 'REGRESSION'}, + }), + + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '3')], + }) + self.create_hint('autopkgtest', 'force-reset-test lightgreen/3') + self.run_it( + [], + {'green': (True, { + 'lightgreen/1': {'amd64': 'ALWAYSFAIL'}, + }), + 'lightgreen': (True, { + 'lightgreen/3': {'amd64': 'ALWAYSFAIL'}, + }), + + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '3')], + }) + + def test_hint_force_reset_test_earlier_hints(self): + '''force-reset-test for a later version applies backwards''' + + self.data.add_default_packages(green=False, lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150102_100101@': (4, 'lightgreen 1', tr('green/2')), + 'testing/amd64/l/lightgreen/20150103_100102@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'testing/amd64/l/lightgreen/20150104_100102@': (4, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/3') + self.run_it( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, { + 'lightgreen/1': {'amd64': 'ALWAYSFAIL'}, + }), + 'lightgreen': (True, { + 'lightgreen/3': {'amd64': 'ALWAYSFAIL'}, + }), + + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '3')], + }) + + def test_hint_force_reset_test_earlier_hints_pass(self): + '''force-reset-test for a later version which is PASS is still PASS''' + + self.data.add_default_packages(green=False, lightgreen=False) + + self.swift.set_results({'autopkgtest-testing': { + 'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 1', tr('green/2')), + 'testing/amd64/l/lightgreen/20150103_100102@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'testing/amd64/l/lightgreen/20150104_100102@': (0, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.create_hint('autopkgtest', 'force-reset-test lightgreen/3') + self.run_it( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, { + 'lightgreen/1': {'amd64': 'PASS'}, + }), + 'lightgreen': (True, { + 'lightgreen/3': {'amd64': 'PASS'}, + }), + + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '3')], + }) + ################################################################ # Kernel related tests ################################################################