diff --git a/autopkgtest.py b/autopkgtest.py index 91618e8..d197e00 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -39,7 +39,6 @@ from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION) adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") -ADT_PASS = ["PASS", "ALWAYSFAIL"] ADT_EXCUSES_LABELS = { "PASS": 'Pass', "ALWAYSFAIL": 'Always failed', @@ -647,23 +646,33 @@ class AutoPackageTest(object): def results(self, trigsrc, trigver): '''Return test results for triggering package - Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests - that got triggered by trigsrc/trigver. + Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING) + iterator for all package tests that got triggered by trigsrc/trigver. ''' - # deprecated results for old Jenkins/lp:auto-package-testing, will go - # away - for status, src, ver in self.pkgcauses[trigsrc][trigver]: - # Check for regression - if status == 'FAIL': - passed_once = False - for lver in self.pkglist[src]: - for trigsrc in self.pkglist[src][lver]['causes']: - for trigver, status \ - in self.pkglist[src][lver]['causes'][trigsrc]: - if status == 'PASS': - passed_once = True - if not passed_once: - status = 'ALWAYSFAIL' - else: - status = 'REGRESSION' - yield status, src, ver + for testsrc, testver in self.tests_for_source(trigsrc, trigver): + passed = True + arch_status = {} + for arch in self.britney.options.adt_arches.split(): + try: + if self.test_results[testsrc][arch][1][testver][0]: + arch_status[arch] = 'PASS' + else: + if self.test_results[testsrc][arch][2]: + arch_status[arch] = 'REGRESSION' + passed = False + else: + arch_status[arch] = 'ALWAYSFAIL' + except KeyError: + try: + self.pending_tests[testsrc][testver][arch] + arch_status[arch] = 'RUNNING' + passed = False + except KeyError: + # neither done nor pending -> exclusion, or disabled + continue + + # disabled or ignored? + if not arch_status: + continue + + yield (passed, testsrc, testver, arch_status) diff --git a/britney.py b/britney.py index 23455c5..46531f8 100755 --- a/britney.py +++ b/britney.py @@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes, from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) -from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash +from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash from boottest import BootTest @@ -1849,28 +1849,21 @@ class Britney(object): if not self.options.dry_run: autopkgtest.submit() autopkgtest.collect(autopkgtest_packages) - jenkins_public = "https://jenkins.qa.ubuntu.com/job" - jenkins_private = ( - "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" % - self.options.series.title()) cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" + adtpass = True for e in autopkgtest_excuses: - adtpass = True - for status, adtsrc, adtver in autopkgtest.results( + for adtpass, adtsrc, adtver, arch_status in autopkgtest.results( e.name, e.ver[1]): - public_url = "%s/%s-adt-%s/lastBuild" % ( - jenkins_public, self.options.series, - adtsrc.replace("+", "-")) - private_url = "%s/%s-adt-%s/lastBuild" % ( - jenkins_private, self.options.series, - adtsrc.replace("+", "-")) - adt_label = ADT_EXCUSES_LABELS.get(status, status) - e.addhtml( - "autopkgtest for %s %s: %s (Jenkins: " - "public, " - "private)" % - (adtsrc, adtver, adt_label, public_url, private_url)) - if status not in ADT_PASS: + archmsg = [] + for arch in sorted(arch_status): + url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc, + 'r': self.options.series, 'a': arch} + archmsg.append('%s: %s' % + (url, arch, ADT_EXCUSES_LABELS[arch_status[arch]])) + e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg))) + + # hints can override failures + if not adtpass: hints = self.hints.search( 'force-badtest', package=adtsrc) hints.extend( @@ -1882,34 +1875,7 @@ class Britney(object): e.addhtml( "Should wait for %s %s test, but forced by " "%s" % (adtsrc, adtver, forces[0].user)) - else: - adtpass = False - - # temporary: also show results from cloud based tests, - # until that becomes the primary mechanism - for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]): - msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver) - archmsg = [] - for arch in self.options.adt_arches.split(): - url = cloud_url % {'h': srchash(testsrc), 's': testsrc, - 'r': self.options.series, 'a': arch} - try: - r = autopkgtest.test_results[testsrc][arch][1][testver][0] - status = r and 'PASS' or 'REGRESSION' - except KeyError: - try: - autopkgtest.pending_tests[testsrc][testver][arch] - status = 'RUNNING' - except KeyError: - # neither done nor pending -> exclusion, or disabled - continue - - archmsg.append('%s: %s' % - (url, arch, ADT_EXCUSES_LABELS[status])) - - if archmsg: - e.addhtml(msg + ', '.join(archmsg)) - # end of temporary code + adtpass = True if not adtpass and e.is_valid: hints = self.hints.search('force-skiptest', package=e.name) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 8d04ede..95376ef 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -125,9 +125,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -153,7 +151,7 @@ lightgreen 1 i386 green 2 self.assertEqual(self.pending_requests, expected_pending) # if we run britney again this should *not* trigger any new tests - self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<']) + self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<']) self.assertEqual(self.amqp_requests, set()) # but the set of pending tests doesn't change self.assertEqual(self.pending_requests, expected_pending) @@ -164,9 +162,7 @@ lightgreen 1 i386 green 2 # first run requests tests and marks them as pending self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -233,9 +229,7 @@ lightgreen 1 i386 green 2 # first run requests tests and marks them as pending self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -252,11 +246,9 @@ lightgreen 1 i386 green 2 out = self.do_test( [], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', - r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass', r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress', r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass']) @@ -267,15 +259,67 @@ lightgreen 1 i386 green 2 self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) + def test_multi_rdepends_with_tests_regression(self): + '''Multiple reverse dependencies with tests (regression)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_always_failed(self): + '''Multiple reverse dependencies with tests (always failed)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (4, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + def test_package_pair_running(self): '''Two packages in unstable that need to go in together (running)''' self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'\blightgreen\b.*>1 to .*>2<']) @@ -313,9 +357,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'], ['in progress']) @@ -341,9 +383,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression', r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression']) @@ -392,9 +432,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',