autopkgtest: Add new `force-reset-test` hint type

This allows hinters to reset the baseline for a package/arch, so that
previous failures are ignore and not considered regressions

britney2/policies/autopkgtest.py

@@ -158,6 +158,7 @@ class AutopkgtestPolicy(BasePolicy):
     def register_hints(self, hint_parser):
         hint_parser.register_hint_type('force-badtest', britney2.hints.split_into_one_hint_per_package)
         hint_parser.register_hint_type('force-skiptest', britney2.hints.split_into_one_hint_per_package)
+        hint_parser.register_hint_type('force-reset-test', britney2.hints.split_into_one_hint_per_package)
 
     def initialise(self, britney):
         super().initialise(britney)
@@ -1096,6 +1097,11 @@ class AutopkgtestPolicy(BasePolicy):
         # determine current test result status
         baseline_result = self.result_in_baseline(src, arch)[0]
 
+        # determine current test result status
+        until = self.find_max_lower_force_reset_test(src, ver, arch)
+        ever_passed = self.check_ever_passed_before(src, ver, arch, until)
+
+        fail_result = 'REGRESSION' if ever_passed else 'ALWAYSFAIL'
 
         url = None
         run_id = None
@@ -1136,9 +1142,13 @@ class AutopkgtestPolicy(BasePolicy):
                         else:
                             result = 'ALWAYSFAIL'
                     else:
-                        result = 'REGRESSION'
+                        result = fail_result
                 else:
-                    result = 'REGRESSION'
+                    result = fail_result
+
+                if result == 'REGRESSION' and \
+                   self.has_higher_force_reset_test(src, ver, arch):
+                    result = 'ALWAYSFAIL'
 
                 if self.has_force_badtest(src, ver, arch):
                     result = 'IGNORE-FAIL'
@@ -1181,6 +1191,59 @@ class AutopkgtestPolicy(BasePolicy):
 
         return (result, ver, run_id, url)
 
+    def check_ever_passed_before(self, src, max_ver, arch, min_ver=None):
+        '''Check if tests for src ever passed on arch for specified range
+
+        If min_ver is specified, it checks that all versions in
+        [min_ver, max_ver) have passed; otherwise it checks that
+        [min_ver, inf) have passed.'''
+
+        # FIXME: add caching
+        for srcmap in self.test_results.values():
+            try:
+                too_high = apt_pkg.version_compare(srcmap[src][arch][1], max_ver) > 0
+                too_low = apt_pkg.version_compare(srcmap[src][arch][1], min_ver) <= 0 if min_ver else False
+
+                if too_high or too_low:
+                    continue
+
+                if srcmap[src][arch][0] in (Result.PASS, Result.OLD_PASS):
+                    return True
+            except KeyError:
+                pass
+        return False
+
+    def find_max_lower_force_reset_test(self, src, ver, arch):
+        '''Find the maximum force-reset-test hint before/including ver'''
+        hints = self.hints.search('force-reset-test', package=src)
+        found_ver = None
+
+        if hints:
+            for hint in hints:
+                for mi in hint.packages:
+                    if (mi.architecture in ['source', arch] and
+                            mi.version != 'all' and
+                            apt_pkg.version_compare(mi.version, ver) <= 0 and
+                            (found_ver is None or apt_pkg.version_compare(found_ver, mi.version) < 0)):
+                        found_ver = mi.version
+
+        return found_ver
+
+    def has_higher_force_reset_test(self, src, ver, arch):
+        '''Find if there is a minimum force-reset-test hint after/including ver'''
+        hints = self.hints.search('force-reset-test', package=src)
+
+        if hints:
+            self.logger.info('Checking hints for %s/%s/%s: %s' % (src, ver, arch, [str(h) for h in hints]))
+            for hint in hints:
+                for mi in hint.packages:
+                    if (mi.architecture in ['source', arch] and
+                            mi.version != 'all' and
+                            apt_pkg.version_compare(mi.version, ver) >= 0):
+                        return True
+
+        return False
+
     def has_force_badtest(self, src, ver, arch):
         '''Check if src/ver/arch has a force-badtest hint'''
 

tests/test_autopkgtest.py

@@ -1985,6 +1985,223 @@ class AT(TestAutopkgtestBase):
             {'lightgreen': [('reason', 'block')]}
         )
 
+    def test_hint_force_reset_test_goodbad_alwaysfail(self):
+        '''force-reset-test hint marks as alwaysfail'''
+
+        self.data.add_default_packages(lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
+
+        self.run_it(
+            [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (True, {
+                              'lightgreen/2': {'amd64': 'ALWAYSFAIL'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_reset_test_goodbad_alwaysfail_arch(self):
+        '''force-reset-test hint marks as alwaysfail per arch'''
+
+        self.data.add_default_packages(lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
+            'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/1/amd64')
+
+        self.run_it(
+            [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (False, {
+                             'lightgreen/2': {'amd64': 'ALWAYSFAIL', 'i386': 'REGRESSION'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_reset_test_bad_good_pass(self):
+        '''force-reset-test hint followed by pass is pass'''
+
+        self.data.add_default_packages(lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
+
+        self.run_it(
+            [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (True, {
+                              'lightgreen/2': {'amd64': 'PASS'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_reset_test_bad_good_bad_regression(self):
+        '''force-reset-test hint followed by good, bad is regression'''
+
+        self.data.add_default_packages(lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+            'testing/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 3', tr('lightgreen/3')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
+
+        self.run_it(
+            [('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (False, {
+                              'lightgreen/3': {'amd64': 'REGRESSION'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '3')]
+            })
+
+    def test_hint_force_reset_test_bad_good_bad_regression_different_trigger(self):
+        '''force-reset-test hint followed by good, bad is regression (not self-triggered)'''
+
+        self.data.add_default_packages(green=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 0.1', tr('lightgreen/0.1')),
+            'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 1', tr('green/2')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/0.1')
+
+        self.run_it(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (False, {
+                              'lightgreen/1': {'amd64': 'REGRESSION'},
+                             }),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_reset_test_multiple_hints(self):
+        '''force-reset-test multiple hints check ranges'''
+
+        self.data.add_default_packages(green=False, lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150102_100100@': (4, 'lightgreen 1', tr('green/2')),
+            'testing/amd64/l/lightgreen/20150103_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+            'testing/amd64/l/lightgreen/20150104_100101@': (4, 'lightgreen 3', tr('lightgreen/3')),
+        }})
+
+        self.run_it(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
+             ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (False, {
+                              'lightgreen/1': {'amd64': 'REGRESSION'},
+                             }),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
+        self.run_it(
+            [],
+            {'green': (True, {
+                              'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
+                             }),
+             'lightgreen': (False, {
+                              'lightgreen/3': {'amd64': 'REGRESSION'},
+                             }),
+
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/3')
+        self.run_it(
+            [],
+            {'green': (True, {
+                              'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
+                             }),
+             'lightgreen': (True, {
+                              'lightgreen/3': {'amd64': 'ALWAYSFAIL'},
+                             }),
+
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+
+    def test_hint_force_reset_test_earlier_hints(self):
+        '''force-reset-test for a later version applies backwards'''
+
+        self.data.add_default_packages(green=False, lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150102_100101@': (4, 'lightgreen 1', tr('green/2')),
+            'testing/amd64/l/lightgreen/20150103_100102@': (0, 'lightgreen 2', tr('lightgreen/2')),
+            'testing/amd64/l/lightgreen/20150104_100102@': (4, 'lightgreen 3', tr('lightgreen/3')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/3')
+        self.run_it(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
+             ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (True, {
+                              'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
+                             }),
+             'lightgreen': (True, {
+                              'lightgreen/3': {'amd64': 'ALWAYSFAIL'},
+                             }),
+
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+
+    def test_hint_force_reset_test_earlier_hints_pass(self):
+        '''force-reset-test for a later version which is PASS is still PASS'''
+
+        self.data.add_default_packages(green=False, lightgreen=False)
+
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 1', tr('green/2')),
+            'testing/amd64/l/lightgreen/20150103_100102@': (0, 'lightgreen 2', tr('lightgreen/2')),
+            'testing/amd64/l/lightgreen/20150104_100102@': (0, 'lightgreen 3', tr('lightgreen/3')),
+        }})
+
+        self.create_hint('autopkgtest', 'force-reset-test lightgreen/3')
+        self.run_it(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
+             ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (True, {
+                              'lightgreen/1': {'amd64': 'PASS'},
+                             }),
+             'lightgreen': (True, {
+                              'lightgreen/3': {'amd64': 'PASS'},
+                             }),
+
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+
     ################################################################
     # Kernel related tests
     ################################################################