From a16e4e5a55dbcbb4257d2e489097876145573611 Mon Sep 17 00:00:00 2001 From: Paul Gevers Date: Tue, 24 Oct 2017 20:43:13 +0200 Subject: [PATCH] Enable autopkgtesting on built arches when not all have been built yet - autopkgtest now honors break_arches option - incomplete testing is now treated with penalty behavior --- britney.py | 1 + britney2/excuse.py | 6 + britney2/policies/autopkgtest.py | 182 ++++++++++++++++--------------- tests/test_autopkgtest.py | 28 +++-- 4 files changed, 123 insertions(+), 94 deletions(-) diff --git a/britney.py b/britney.py index 0cc9703..b840a1f 100755 --- a/britney.py +++ b/britney.py @@ -1058,6 +1058,7 @@ class Britney(object): if not packages: excuse.addhtml("%s/%s unsatisfiable Depends: %s" % (pkg, arch, block_txt.strip())) excuse.addreason("depends") + excuse.add_depends_breaks_arch(arch) if arch not in self.options.break_arches: is_all_ok = False continue diff --git a/britney2/excuse.py b/britney2/excuse.py index 2bb18df..81d7781 100644 --- a/britney2/excuse.py +++ b/britney2/excuse.py @@ -78,6 +78,7 @@ class Excuse(object): self.deps = {} self.sane_deps = [] self.break_deps = [] + self.break_arch = [] self.bugs = [] self.newbugs = set() self.oldbugs = set() @@ -139,6 +140,11 @@ class Excuse(object): if (name, arch) not in self.break_deps: self.break_deps.append( (name, arch) ) + def add_depends_breaks_arch(self, arch): + """Add an arch that breaks by dependency""" + if arch not in self.break_arch: + self.break_arch.append(arch) + def invalidate_dep(self, name): """Invalidate dependency""" if name not in self.invalid_deps: self.invalid_deps.append(name) diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py index bbb53d2..fc5f82f 100644 --- a/britney2/policies/autopkgtest.py +++ b/britney2/policies/autopkgtest.py @@ -174,95 +174,105 @@ class AutopkgtestPolicy(BasePolicy): os.rename(self.pending_tests_file + '.new', self.pending_tests_file) def apply_policy_impl(self, tests_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse): - # skip/delay autopkgtests until package is built - binaries_info = self.britney.sources[suite][source_name] - if excuse.missing_builds or not binaries_info.binaries or 'depends' in excuse.reason: - self.log('%s has missing builds or is uninstallable, skipping autopkgtest policy' % excuse.name) - return PolicyVerdict.REJECTED_TEMPORARILY - - self.log('Checking autopkgtests for %s' % source_name) - trigger = source_name + '/' + source_data_srcdist.version - - # build a (testsrc, testver) → arch → (status, log_url) map; we trigger/check test - # results per archtitecture for technical/efficiency reasons, but we - # want to evaluate and present the results by tested source package - # first - pkg_arch_result = {} - for arch in self.adt_arches: - # request tests (unless they were already requested earlier or have a result) - tests = self.tests_for_source(source_name, source_data_srcdist.version, arch) - is_huge = False - try: - is_huge = len(tests) > int(self.options.adt_huge) - except AttributeError: - pass - for (testsrc, testver) in tests: - self.pkg_test_request(testsrc, arch, trigger, huge=is_huge) - (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger) - pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url) - - # add test result details to Excuse + # initialize verdict = PolicyVerdict.PASS src_has_own_test = False - cloud_url = self.options.adt_ci_url + "packages/%(h)s/%(s)s/%(r)s/%(a)s" - for (testsrc, testver) in sorted(pkg_arch_result): - arch_results = pkg_arch_result[(testsrc, testver)] - r = set([v[0] for v in arch_results.values()]) - if 'REGRESSION' in r: - verdict = PolicyVerdict.REJECTED_PERMANENTLY - elif 'RUNNING' in r and verdict == PolicyVerdict.PASS: - verdict = PolicyVerdict.REJECTED_TEMPORARILY - # skip version if still running on all arches - if not r - {'RUNNING', 'RUNNING-ALWAYSFAIL'}: - testver = None - - # Keep track if this source package has tests of its own for the - # bounty system - if testsrc == source_name: - src_has_own_test = True - - html_archmsg = [] - for arch in sorted(arch_results): - (status, log_url) = arch_results[arch] - artifact_url = None - retry_url = None - history_url = None - if self.options.adt_ppas: - if log_url.endswith('log.gz'): - artifact_url = log_url.replace('log.gz', 'artifacts.tar.gz') - else: - history_url = cloud_url % { - 'h': srchash(testsrc), 's': testsrc, - 'r': self.options.series, 'a': arch} - if status == 'REGRESSION': - retry_url = self.options.adt_ci_url + 'request.cgi?' + \ - urllib.parse.urlencode([('release', self.options.series), - ('arch', arch), - ('package', testsrc), - ('trigger', trigger)] + - [('ppa', p) for p in self.options.adt_ppas]) - if testver: - testname = '%s/%s' % (testsrc, testver) + + # skip/delay autopkgtests until new package is built somewhere + binaries_info = self.britney.sources[suite][source_name] + if not binaries_info.binaries: + self.log('%s hasn''t been built anywhere, skipping autopkgtest policy' % excuse.name) + verdict = PolicyVerdict.REJECTED_TEMPORARILY + + if verdict == PolicyVerdict.PASS: + self.log('Checking autopkgtests for %s' % source_name) + trigger = source_name + '/' + source_data_srcdist.version + + # build a (testsrc, testver) → arch → (status, log_url) map; we trigger/check test + # results per archtitecture for technical/efficiency reasons, but we + # want to evaluate and present the results by tested source package + # first + pkg_arch_result = {} + for arch in self.adt_arches: + if arch in excuse.missing_builds: + verdict = PolicyVerdict.REJECTED_TEMPORARILY + self.log('%s hasn''t been built on arch %s, delay autopkgtest there' % (source_name, arch)) + elif arch in excuse.break_arch: + verdict = PolicyVerdict.REJECTED_TEMPORARILY + self.log('%s is uninstallable on arch %s, delay autopkgtest there' % (source_name, arch)) else: - testname = testsrc + # request tests (unless they were already requested earlier or have a result) + tests = self.tests_for_source(source_name, source_data_srcdist.version, arch) + is_huge = False + try: + is_huge = len(tests) > int(self.options.adt_huge) + except AttributeError: + pass + for (testsrc, testver) in tests: + self.pkg_test_request(testsrc, arch, trigger, huge=is_huge) + (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger) + pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url) + + # add test result details to Excuse + cloud_url = self.options.adt_ci_url + "packages/%(h)s/%(s)s/%(r)s/%(a)s" + for (testsrc, testver) in sorted(pkg_arch_result): + arch_results = pkg_arch_result[(testsrc, testver)] + r = set([v[0] for v in arch_results.values()]) + if 'REGRESSION' in r: + verdict = PolicyVerdict.REJECTED_PERMANENTLY + elif 'RUNNING' in r and verdict == PolicyVerdict.PASS: + verdict = PolicyVerdict.REJECTED_TEMPORARILY + # skip version if still running on all arches + if not r - {'RUNNING', 'RUNNING-ALWAYSFAIL'}: + testver = None + + # Keep track if this source package has tests of its own for the + # bounty system + if testsrc == source_name: + src_has_own_test = True + + html_archmsg = [] + for arch in sorted(arch_results): + (status, log_url) = arch_results[arch] + artifact_url = None + retry_url = None + history_url = None + if self.options.adt_ppas: + if log_url.endswith('log.gz'): + artifact_url = log_url.replace('log.gz', 'artifacts.tar.gz') + else: + history_url = cloud_url % { + 'h': srchash(testsrc), 's': testsrc, + 'r': self.options.series, 'a': arch} + if status == 'REGRESSION': + retry_url = self.options.adt_ci_url + 'request.cgi?' + \ + urllib.parse.urlencode([('release', self.options.series), + ('arch', arch), + ('package', testsrc), + ('trigger', trigger)] + + [('ppa', p) for p in self.options.adt_ppas]) + if testver: + testname = '%s/%s' % (testsrc, testver) + else: + testname = testsrc - tests_info.setdefault(testname, {})[arch] = \ - [status, log_url, history_url, artifact_url, retry_url] + tests_info.setdefault(testname, {})[arch] = \ + [status, log_url, history_url, artifact_url, retry_url] - # render HTML snippet for testsrc entry for current arch - if history_url: - message = '%s' % (history_url, arch) - else: - message = arch - message += ': %s' % (log_url, EXCUSES_LABELS[status]) - if retry_url: - message += ' ♻ ' % retry_url - if artifact_url: - message += ' [artifacts]' % artifact_url - html_archmsg.append(message) + # render HTML snippet for testsrc entry for current arch + if history_url: + message = '%s' % (history_url, arch) + else: + message = arch + message += ': %s' % (log_url, EXCUSES_LABELS[status]) + if retry_url: + message += ' ♻ ' % retry_url + if artifact_url: + message += ' [artifacts]' % artifact_url + html_archmsg.append(message) - # render HTML line for testsrc entry - excuse.addhtml("autopkgtest for %s: %s" % (testname, ', '.join(html_archmsg))) + # render HTML line for testsrc entry + excuse.addhtml("autopkgtest for %s: %s" % (testname, ', '.join(html_archmsg))) if verdict != PolicyVerdict.PASS: @@ -279,10 +289,10 @@ class AutopkgtestPolicy(BasePolicy): if self.options.adt_success_bounty and verdict == PolicyVerdict.PASS and src_has_own_test: excuse.add_bounty('autopkgtest', int(self.options.adt_success_bounty)) - if self.options.adt_regression_penalty and verdict == PolicyVerdict.REJECTED_PERMANENTLY: + if self.options.adt_regression_penalty and \ + verdict in [PolicyVerdict.REJECTED_PERMANENTLY, PolicyVerdict.REJECTED_TEMPORARILY]: excuse.add_penalty('autopkgtest', int(self.options.adt_regression_penalty)) - # In case we give penalties instead of blocking, we must pass in - # case of regression. + # In case we give penalties instead of blocking, we must always pass verdict = PolicyVerdict.PASS return verdict diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index ab12231..02a9be3 100755 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -705,7 +705,7 @@ class T(TestBase): self.assertEqual(self.pending_requests, {}) def test_partial_unbuilt(self): - '''Unbuilt package on some arches should not trigger tests''' + '''Unbuilt package on some arches should not trigger tests on those arches''' self.data.add_default_packages(green=False) @@ -715,6 +715,12 @@ class T(TestBase): 'Conflicts': 'blue'}, testsuite='autopkgtest', add_src=False) + self.swift.set_results({'autopkgtest-testing': { + 'testing/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'testing/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + exc = self.do_test( [], {'green': (False, {})}, @@ -723,13 +729,13 @@ class T(TestBase): 'on-unimportant-architectures': []}) ] })[1] - # autopkgtest should not be triggered for unbuilt pkg - self.assertEqual(exc['green']['policy_info']['autopkgtest'], {'verdict': 'REJECTED_TEMPORARILY'}) + # autopkgtest should not be triggered on arches with unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest']['verdict'], 'REJECTED_TEMPORARILY') self.assertEqual(self.amqp_requests, set()) self.assertEqual(self.pending_requests, {}) def test_partial_unbuilt_block(self): - '''Unbuilt blocked package on some arches should not trigger tests''' + '''Unbuilt blocked package on some arches should not trigger tests on those arches''' self.data.add_default_packages(green=False) @@ -741,6 +747,12 @@ class T(TestBase): 'Conflicts': 'blue'}, testsuite='autopkgtest', add_src=False) + self.swift.set_results({'autopkgtest-testing': { + 'testing/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'testing/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + exc = self.do_test( [], {'green': (False, {})}, @@ -749,8 +761,8 @@ class T(TestBase): 'on-unimportant-architectures': []}) ] })[1] - # autopkgtest should not be triggered for unbuilt pkg - self.assertEqual(exc['green']['policy_info']['autopkgtest'], {'verdict': 'REJECTED_TEMPORARILY'}) + # autopkgtest should not be triggered on arches with unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest']['verdict'], 'REJECTED_TEMPORARILY') self.assertEqual(self.amqp_requests, set()) self.assertEqual(self.pending_requests, {}) @@ -2490,8 +2502,8 @@ class T(TestBase): }, {'green': [('old-version', '1'), ('new-version', '2')]})[1] - # while no autopkgtest results are known, default age applies - self.assertEqual(exc['green']['policy_info']['age']['age-requirement'], 13) + # while no autopkgtest results are known, penalty applies + self.assertEqual(exc['green']['policy_info']['age']['age-requirement'], 40) # second run collects the results self.swift.set_results({'autopkgtest-testing': {