Enable autopkgtesting on built arches when not all have been built yet

- autopkgtest now honors break_arches option
- incomplete testing is now treated with penalty behavior

britney.py

@@ -1058,6 +1058,7 @@ class Britney(object):
             if not packages:
                 excuse.addhtml("%s/%s unsatisfiable Depends: %s" % (pkg, arch, block_txt.strip()))
                 excuse.addreason("depends")
+                excuse.add_depends_breaks_arch(arch)
                 if arch not in self.options.break_arches:
                     is_all_ok = False
                 continue

britney2/excuse.py

@@ -78,6 +78,7 @@ class Excuse(object):
         self.deps = {}
         self.sane_deps = []
         self.break_deps = []
+        self.break_arch = []
         self.bugs = []
         self.newbugs = set()
         self.oldbugs = set()
@@ -139,6 +140,11 @@ class Excuse(object):
         if (name, arch) not in self.break_deps:
             self.break_deps.append( (name, arch) )
 
+    def add_depends_breaks_arch(self,  arch):
+        """Add an arch that breaks by dependency"""
+        if  arch not in self.break_arch:
+            self.break_arch.append(arch)
+
     def invalidate_dep(self, name):
         """Invalidate dependency"""
         if name not in self.invalid_deps: self.invalid_deps.append(name)

britney2/policies/autopkgtest.py

@@ -174,95 +174,105 @@ class AutopkgtestPolicy(BasePolicy):
         os.rename(self.pending_tests_file + '.new', self.pending_tests_file)
 
     def apply_policy_impl(self, tests_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse):
-        # skip/delay autopkgtests until package is built
+        # initialize
-        binaries_info = self.britney.sources[suite][source_name]
-        if excuse.missing_builds or not binaries_info.binaries or 'depends' in excuse.reason:
-            self.log('%s has missing builds or is uninstallable, skipping autopkgtest policy' % excuse.name)
-            return PolicyVerdict.REJECTED_TEMPORARILY
-
-        self.log('Checking autopkgtests for %s' % source_name)
-        trigger = source_name + '/' + source_data_srcdist.version
-
-        # build a (testsrc, testver) → arch → (status, log_url) map; we trigger/check test
-        # results per archtitecture for technical/efficiency reasons, but we
-        # want to evaluate and present the results by tested source package
-        # first
-        pkg_arch_result = {}
-        for arch in self.adt_arches:
-            # request tests (unless they were already requested earlier or have a result)
-            tests = self.tests_for_source(source_name, source_data_srcdist.version, arch)
-            is_huge = False
-            try:
-                is_huge = len(tests) > int(self.options.adt_huge)
-            except AttributeError:
-                pass
-            for (testsrc, testver) in tests:
-                self.pkg_test_request(testsrc, arch, trigger, huge=is_huge)
-                (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger)
-                pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url)
-
-        # add test result details to Excuse
         verdict = PolicyVerdict.PASS
         src_has_own_test = False
-        cloud_url = self.options.adt_ci_url + "packages/%(h)s/%(s)s/%(r)s/%(a)s"
-        for (testsrc, testver) in sorted(pkg_arch_result):
-            arch_results = pkg_arch_result[(testsrc, testver)]
-            r = set([v[0] for v in arch_results.values()])
-            if 'REGRESSION' in r:
-                verdict = PolicyVerdict.REJECTED_PERMANENTLY
-            elif 'RUNNING' in r and verdict == PolicyVerdict.PASS:
-                verdict = PolicyVerdict.REJECTED_TEMPORARILY
-            # skip version if still running on all arches
-            if not r - {'RUNNING', 'RUNNING-ALWAYSFAIL'}:
-                testver = None
 
-            # Keep track if this source package has tests of its own for the
+        # skip/delay autopkgtests until new package is built somewhere
-            # bounty system
+        binaries_info = self.britney.sources[suite][source_name]
-            if testsrc == source_name:
+        if not binaries_info.binaries:
-                src_has_own_test = True
+            self.log('%s hasn''t been built anywhere, skipping autopkgtest policy' % excuse.name)
+            verdict = PolicyVerdict.REJECTED_TEMPORARILY
 
-            html_archmsg = []
+        if verdict == PolicyVerdict.PASS:
-            for arch in sorted(arch_results):
+            self.log('Checking autopkgtests for %s' % source_name)
-                (status, log_url) = arch_results[arch]
+            trigger = source_name + '/' + source_data_srcdist.version
-                artifact_url = None
+
-                retry_url = None
+            # build a (testsrc, testver) → arch → (status, log_url) map; we trigger/check test
-                history_url = None
+            # results per archtitecture for technical/efficiency reasons, but we
-                if self.options.adt_ppas:
+            # want to evaluate and present the results by tested source package
-                    if log_url.endswith('log.gz'):
+            # first
-                        artifact_url = log_url.replace('log.gz', 'artifacts.tar.gz')
+            pkg_arch_result = {}
+            for arch in self.adt_arches:
+                if arch in excuse.missing_builds:
+                    verdict = PolicyVerdict.REJECTED_TEMPORARILY
+                    self.log('%s hasn''t been built on arch %s, delay autopkgtest there' % (source_name, arch))
+                elif arch in excuse.break_arch:
+                    verdict = PolicyVerdict.REJECTED_TEMPORARILY
+                    self.log('%s is uninstallable on arch %s, delay autopkgtest there' % (source_name, arch))
                 else:
-                    history_url = cloud_url % {
+                    # request tests (unless they were already requested earlier or have a result)
-                        'h': srchash(testsrc), 's': testsrc,
+                    tests = self.tests_for_source(source_name, source_data_srcdist.version, arch)
-                        'r': self.options.series, 'a': arch}
+                    is_huge = False
-                if status == 'REGRESSION':
+                    try:
-                    retry_url = self.options.adt_ci_url + 'request.cgi?' + \
+                        is_huge = len(tests) > int(self.options.adt_huge)
-                            urllib.parse.urlencode([('release', self.options.series),
+                    except AttributeError:
-                                                    ('arch', arch),
+                        pass
-                                                    ('package', testsrc),
+                    for (testsrc, testver) in tests:
-                                                    ('trigger', trigger)] +
+                        self.pkg_test_request(testsrc, arch, trigger, huge=is_huge)
-                                                   [('ppa', p) for p in self.options.adt_ppas])
+                        (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger)
-                if testver:
+                        pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url)
-                    testname = '%s/%s' % (testsrc, testver)
-                else:
-                    testname = testsrc
 
-                tests_info.setdefault(testname, {})[arch] = \
+            # add test result details to Excuse
-                        [status, log_url, history_url, artifact_url, retry_url]
+            cloud_url = self.options.adt_ci_url + "packages/%(h)s/%(s)s/%(r)s/%(a)s"
+            for (testsrc, testver) in sorted(pkg_arch_result):
+                arch_results = pkg_arch_result[(testsrc, testver)]
+                r = set([v[0] for v in arch_results.values()])
+                if 'REGRESSION' in r:
+                    verdict = PolicyVerdict.REJECTED_PERMANENTLY
+                elif 'RUNNING' in r and verdict == PolicyVerdict.PASS:
+                    verdict = PolicyVerdict.REJECTED_TEMPORARILY
+                # skip version if still running on all arches
+                if not r - {'RUNNING', 'RUNNING-ALWAYSFAIL'}:
+                    testver = None
 
-                # render HTML snippet for testsrc entry for current arch
+                # Keep track if this source package has tests of its own for the
-                if history_url:
+                # bounty system
-                    message = '<a href="%s">%s</a>' % (history_url, arch)
+                if testsrc == source_name:
-                else:
+                    src_has_own_test = True
-                    message = arch
-                message += ': <a href="%s">%s</a>' % (log_url, EXCUSES_LABELS[status])
-                if retry_url:
-                    message += ' <a href="%s" style="text-decoration: none;">♻ </a> ' % retry_url
-                if artifact_url:
-                    message += ' <a href="%s">[artifacts]</a>' % artifact_url
-                html_archmsg.append(message)
 
-            # render HTML line for testsrc entry
+                html_archmsg = []
-            excuse.addhtml("autopkgtest for %s: %s" % (testname, ', '.join(html_archmsg)))
+                for arch in sorted(arch_results):
+                    (status, log_url) = arch_results[arch]
+                    artifact_url = None
+                    retry_url = None
+                    history_url = None
+                    if self.options.adt_ppas:
+                        if log_url.endswith('log.gz'):
+                            artifact_url = log_url.replace('log.gz', 'artifacts.tar.gz')
+                    else:
+                        history_url = cloud_url % {
+                            'h': srchash(testsrc), 's': testsrc,
+                            'r': self.options.series, 'a': arch}
+                    if status == 'REGRESSION':
+                        retry_url = self.options.adt_ci_url + 'request.cgi?' + \
+                                urllib.parse.urlencode([('release', self.options.series),
+                                                        ('arch', arch),
+                                                        ('package', testsrc),
+                                                        ('trigger', trigger)] +
+                                                       [('ppa', p) for p in self.options.adt_ppas])
+                    if testver:
+                        testname = '%s/%s' % (testsrc, testver)
+                    else:
+                        testname = testsrc
+
+                    tests_info.setdefault(testname, {})[arch] = \
+                            [status, log_url, history_url, artifact_url, retry_url]
+
+                    # render HTML snippet for testsrc entry for current arch
+                    if history_url:
+                        message = '<a href="%s">%s</a>' % (history_url, arch)
+                    else:
+                        message = arch
+                    message += ': <a href="%s">%s</a>' % (log_url, EXCUSES_LABELS[status])
+                    if retry_url:
+                        message += ' <a href="%s" style="text-decoration: none;">♻ </a> ' % retry_url
+                    if artifact_url:
+                        message += ' <a href="%s">[artifacts]</a>' % artifact_url
+                    html_archmsg.append(message)
+
+                # render HTML line for testsrc entry
+                excuse.addhtml("autopkgtest for %s: %s" % (testname, ', '.join(html_archmsg)))
 
 
         if verdict != PolicyVerdict.PASS:
@@ -279,10 +289,10 @@ class AutopkgtestPolicy(BasePolicy):
 
         if self.options.adt_success_bounty and verdict == PolicyVerdict.PASS and src_has_own_test:
             excuse.add_bounty('autopkgtest', int(self.options.adt_success_bounty))
-        if self.options.adt_regression_penalty and verdict == PolicyVerdict.REJECTED_PERMANENTLY:
+        if self.options.adt_regression_penalty and \
+          verdict in [PolicyVerdict.REJECTED_PERMANENTLY, PolicyVerdict.REJECTED_TEMPORARILY]:
             excuse.add_penalty('autopkgtest', int(self.options.adt_regression_penalty))
-            # In case we give penalties instead of blocking, we must pass in
+            # In case we give penalties instead of blocking, we must always pass
-            # case of regression.
             verdict = PolicyVerdict.PASS
 
         return verdict

tests/test_autopkgtest.py

@@ -705,7 +705,7 @@ class T(TestBase):
         self.assertEqual(self.pending_requests, {})
 
     def test_partial_unbuilt(self):
-        '''Unbuilt package on some arches should not trigger tests'''
+        '''Unbuilt package on some arches should not trigger tests on those arches'''
 
         self.data.add_default_packages(green=False)
 
@@ -715,6 +715,12 @@ class T(TestBase):
                                        'Conflicts': 'blue'},
                       testsuite='autopkgtest', add_src=False)
 
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')),
+            'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')),
+            'testing/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')),
+        }})
+
         exc = self.do_test(
             [],
             {'green': (False, {})},
@@ -723,13 +729,13 @@ class T(TestBase):
                                            'on-unimportant-architectures': []})
                       ]
             })[1]
-        # autopkgtest should not be triggered for unbuilt pkg
+        # autopkgtest should not be triggered on arches with unbuilt pkg
-        self.assertEqual(exc['green']['policy_info']['autopkgtest'], {'verdict': 'REJECTED_TEMPORARILY'})
+        self.assertEqual(exc['green']['policy_info']['autopkgtest']['verdict'], 'REJECTED_TEMPORARILY')
         self.assertEqual(self.amqp_requests, set())
         self.assertEqual(self.pending_requests, {})
 
     def test_partial_unbuilt_block(self):
-        '''Unbuilt blocked package on some arches should not trigger tests'''
+        '''Unbuilt blocked package on some arches should not trigger tests on those arches'''
 
         self.data.add_default_packages(green=False)
 
@@ -741,6 +747,12 @@ class T(TestBase):
                                        'Conflicts': 'blue'},
                       testsuite='autopkgtest', add_src=False)
 
+        self.swift.set_results({'autopkgtest-testing': {
+            'testing/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')),
+            'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')),
+            'testing/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')),
+        }})
+
         exc = self.do_test(
             [],
             {'green': (False, {})},
@@ -749,8 +761,8 @@ class T(TestBase):
                                            'on-unimportant-architectures': []})
                       ]
             })[1]
-        # autopkgtest should not be triggered for unbuilt pkg
+        # autopkgtest should not be triggered on arches with unbuilt pkg
-        self.assertEqual(exc['green']['policy_info']['autopkgtest'], {'verdict': 'REJECTED_TEMPORARILY'})
+        self.assertEqual(exc['green']['policy_info']['autopkgtest']['verdict'], 'REJECTED_TEMPORARILY')
         self.assertEqual(self.amqp_requests, set())
         self.assertEqual(self.pending_requests, {})
 
@@ -2490,8 +2502,8 @@ class T(TestBase):
             },
             {'green': [('old-version', '1'), ('new-version', '2')]})[1]
 
-        # while no autopkgtest results are known, default age applies
+        # while no autopkgtest results are known, penalty applies
-        self.assertEqual(exc['green']['policy_info']['age']['age-requirement'], 13)
+        self.assertEqual(exc['green']['policy_info']['age']['age-requirement'], 40)
 
         # second run collects the results
         self.swift.set_results({'autopkgtest-testing': {