Allow marking a version as accepted regression

LP: #1688516

britney2/policies/autopkgtest.py

@@ -103,6 +103,7 @@ class AutopkgtestPolicy(BasePolicy):
 
     def register_hints(self, hint_parser):
         hint_parser.register_hint_type('force-badtest', britney2.hints.split_into_one_hint_per_package)
+        hint_parser.register_hint_type('force-badtest-until', britney2.hints.split_into_one_hint_per_package)
         hint_parser.register_hint_type('force-skiptest', britney2.hints.split_into_one_hint_per_package)
 
     def initialise(self, britney):
@@ -681,12 +682,22 @@ class AutopkgtestPolicy(BasePolicy):
             # save pending.json right away, so that we don't re-request if britney crashes
             self.save_pending_json()
 
-    def check_ever_passed(self, src, arch):
+    def check_ever_passed_before(self, src, max_ver, arch, min_ver=None):
-        '''Check if tests for src ever passed on arch'''
+        '''Check if tests for src ever passed on arch for specified range
+
+        If min_ver is specified, it checks that all versions in
+        [min_ver, max_ver) have passed; otherwise it checks that
+        [min_ver, inf) have passed.'''
 
         # FIXME: add caching
         for srcmap in self.test_results.values():
             try:
+                too_high = apt_pkg.version_compare(srcmap[src][arch][1], max_ver) > 0
+                too_low = apt_pkg.version_compare(srcmap[src][arch][1], min_ver) <= 0 if min_ver else False
+
+                if too_high or too_low:
+                    continue
+
                 if srcmap[src][arch][0]:
                     return True
             except KeyError:
@@ -700,7 +711,8 @@ class AutopkgtestPolicy(BasePolicy):
         EXCUSES_LABELS. log_url is None if the test is still running.
         '''
         # determine current test result status
-        ever_passed = self.check_ever_passed(src, arch)
+        until = self.find_max_force_badtest_until(src, ver, arch)
+        ever_passed = self.check_ever_passed_before(src, ver, arch, until)
         url = None
         try:
             r = self.test_results[trigger][src][arch]
@@ -747,6 +759,22 @@ class AutopkgtestPolicy(BasePolicy):
 
         return (result, ver, url)
 
+    def find_max_force_badtest_until(self, src, ver, arch):
+        '''Find the maximum force-badtest-until hint before/including ver'''
+        hints = self.britney.hints.search('force-badtest-until', package=src)
+        found_ver = None
+
+        if hints:
+            for hint in hints:
+                for mi in hint.packages:
+                    if (mi.architecture in ['source', arch] and
+                            mi.version != 'all' and
+                            apt_pkg.version_compare(mi.version, ver) <= 0 and
+                            (found_ver is None or apt_pkg.version_compare(found_ver, mi.version) < 0)):
+                        found_ver = mi.version
+
+        return found_ver
+
     def has_force_badtest(self, src, ver, arch):
         '''Check if src/ver/arch has a force-badtest hint'''
 

tests/test_autopkgtest.py

@@ -1607,6 +1607,137 @@ class T(TestBase):
             {'green': [('old-version', '1'), ('new-version', '2')]
             })
 
+    def test_hint_force_badtest_until_goodbad_alwaysfail(self):
+        '''force-badtest-until hint marks as alwaysfail'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        self.create_hint('pitti', 'force-badtest-until lightgreen/1')
+
+        self.do_test(
+            [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (True, {
+                              'lightgreen/2': {'amd64': 'ALWAYSFAIL'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_badtest_until_goodbad_alwaysfail_arch(self):
+        '''force-badtest-until hint marks as alwaysfail per arch'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        self.create_hint('pitti', 'force-badtest-until lightgreen/1/amd64')
+
+        self.do_test(
+            [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (False, {
+                             'lightgreen/2': {'amd64': 'ALWAYSFAIL', 'i386': 'REGRESSION'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_badtest_until_bad_good_pass(self):
+        '''force-badtest-until hint followed by pass is pass'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')),
+            'series/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        self.create_hint('pitti', 'force-badtest-until lightgreen/1')
+
+        self.do_test(
+            [('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (True, {
+                              'lightgreen/2': {'amd64': 'PASS'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]
+            })
+
+    def test_hint_force_badtest_until_bad_good_bad_regression(self):
+        '''force-badtest-until hint followed by good, bad is regression'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')),
+            'series/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+            'series/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 3', tr('lightgreen/3')),
+        }})
+
+        self.create_hint('pitti', 'force-badtest-until lightgreen/1')
+
+        self.do_test(
+            [('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'lightgreen': (False, {
+                              'lightgreen/3': {'amd64': 'REGRESSION'},
+                             }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '3')]
+            })
+
+    def test_hint_force_badtest_until_multiple_hints(self):
+        '''force-badtest-until multiple hints check ranges'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/amd64/l/lightgreen/20150100_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/2')),
+            'series/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+            'series/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 3', tr('lightgreen/3')),
+        }})
+
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
+             ('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (False, {
+                              'lightgreen/1': {'amd64': 'REGRESSION'},
+                             }),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+
+        self.create_hint('pitti', 'force-badtest-until lightgreen/1')
+        self.do_test(
+            [],
+            {'green': (True, {
+                              'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
+                             }),
+             'lightgreen': (False, {
+                              'lightgreen/3': {'amd64': 'REGRESSION'},
+                             }),
+
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+        self.create_hint('pitti', 'force-badtest-until lightgreen/3')
+        self.do_test(
+            [],
+            {'green': (True, {
+                              'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
+                             }),
+             'lightgreen': (True, {
+                              'lightgreen/3': {'amd64': 'ALWAYSFAIL'},
+                             }),
+
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '3')],
+            })
+
+
     def test_hint_force_badtest_multi_version(self):
         '''force-badtest hint'''