mirror of
https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu
synced 2025-10-29 23:54:08 +00:00
Switch autopkgtest evaluation to cloud results
Change AutoPackageTest.results() to evaluate the Swift results instead of the adt-britney ones. TODO: - Add more tests (like for adt-britney) - Drop triggering of adt-britney tests - Drop adt-britney tests (which fail now) - Adjust TestBoottestEnd2End.test_with_adt
This commit is contained in:
Martin Pitt
commit
31e647f113
@ -39,7 +39,6 @@ from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION)
|
||||
|
||||
adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney")
|
||||
|
||||
ADT_PASS = ["PASS", "ALWAYSFAIL"]
|
||||
ADT_EXCUSES_LABELS = {
|
||||
"PASS": '<span style="background:#87d96c">Pass</span>',
|
||||
"ALWAYSFAIL": '<span style="background:#e5c545">Always failed</span>',
|
||||
@ -652,23 +651,33 @@ class AutoPackageTest(object):
|
||||
def results(self, trigsrc, trigver):
|
||||
'''Return test results for triggering package
|
||||
|
||||
Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests
|
||||
that got triggered by trigsrc/trigver.
|
||||
Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING)
|
||||
iterator for all package tests that got triggered by trigsrc/trigver.
|
||||
'''
|
||||
# deprecated results for old Jenkins/lp:auto-package-testing, will go
|
||||
# away
|
||||
for status, src, ver in self.pkgcauses[trigsrc][trigver]:
|
||||
# Check for regression
|
||||
if status == 'FAIL':
|
||||
passed_once = False
|
||||
for lver in self.pkglist[src]:
|
||||
for trigsrc in self.pkglist[src][lver]['causes']:
|
||||
for trigver, status \
|
||||
in self.pkglist[src][lver]['causes'][trigsrc]:
|
||||
if status == 'PASS':
|
||||
passed_once = True
|
||||
if not passed_once:
|
||||
status = 'ALWAYSFAIL'
|
||||
for testsrc, testver in self.tests_for_source(trigsrc, trigver):
|
||||
passed = True
|
||||
arch_status = {}
|
||||
for arch in self.britney.options.adt_arches.split():
|
||||
try:
|
||||
if self.test_results[testsrc][arch][1][testver][0]:
|
||||
arch_status[arch] = 'PASS'
|
||||
else:
|
||||
status = 'REGRESSION'
|
||||
yield status, src, ver
|
||||
if self.test_results[testsrc][arch][2]:
|
||||
arch_status[arch] = 'REGRESSION'
|
||||
passed = False
|
||||
else:
|
||||
arch_status[arch] = 'ALWAYSFAIL'
|
||||
except KeyError:
|
||||
try:
|
||||
self.pending_tests[testsrc][testver][arch]
|
||||
arch_status[arch] = 'RUNNING'
|
||||
passed = False
|
||||
except KeyError:
|
||||
# neither done nor pending -> exclusion, or disabled
|
||||
continue
|
||||
|
||||
# disabled or ignored?
|
||||
if not arch_status:
|
||||
continue
|
||||
|
||||
yield (passed, testsrc, testver, arch_status)
|
||||
|
||||
66
britney.py
66
britney.py
@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes,
|
||||
from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
|
||||
SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
|
||||
PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL)
|
||||
from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash
|
||||
from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash
|
||||
from boottest import BootTest
|
||||
|
||||
|
||||
@ -1849,28 +1849,21 @@ class Britney(object):
|
||||
if not self.options.dry_run:
|
||||
autopkgtest.submit()
|
||||
autopkgtest.collect(autopkgtest_packages)
|
||||
jenkins_public = "https://jenkins.qa.ubuntu.com/job"
|
||||
jenkins_private = (
|
||||
"http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %
|
||||
self.options.series.title())
|
||||
cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s"
|
||||
for e in autopkgtest_excuses:
|
||||
adtpass = True
|
||||
for status, adtsrc, adtver in autopkgtest.results(
|
||||
for passed, adtsrc, adtver, arch_status in autopkgtest.results(
|
||||
e.name, e.ver[1]):
|
||||
public_url = "%s/%s-adt-%s/lastBuild" % (
|
||||
jenkins_public, self.options.series,
|
||||
adtsrc.replace("+", "-"))
|
||||
private_url = "%s/%s-adt-%s/lastBuild" % (
|
||||
jenkins_private, self.options.series,
|
||||
adtsrc.replace("+", "-"))
|
||||
adt_label = ADT_EXCUSES_LABELS.get(status, status)
|
||||
e.addhtml(
|
||||
"autopkgtest for %s %s: %s (Jenkins: "
|
||||
"<a href=\"%s\">public</a>, "
|
||||
"<a href=\"%s\">private</a>)" %
|
||||
(adtsrc, adtver, adt_label, public_url, private_url))
|
||||
if status not in ADT_PASS:
|
||||
archmsg = []
|
||||
for arch in sorted(arch_status):
|
||||
url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc,
|
||||
'r': self.options.series, 'a': arch}
|
||||
archmsg.append('<a href="%s">%s: %s</a>' %
|
||||
(url, arch, ADT_EXCUSES_LABELS[arch_status[arch]]))
|
||||
e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg)))
|
||||
|
||||
# hints can override failures
|
||||
if not passed:
|
||||
hints = self.hints.search(
|
||||
'force-badtest', package=adtsrc)
|
||||
hints.extend(
|
||||
@ -1882,40 +1875,11 @@ class Britney(object):
|
||||
e.addhtml(
|
||||
"Should wait for %s %s test, but forced by "
|
||||
"%s" % (adtsrc, adtver, forces[0].user))
|
||||
else:
|
||||
passed = True
|
||||
|
||||
if not passed:
|
||||
adtpass = False
|
||||
|
||||
# temporary: also show results from cloud based tests,
|
||||
# until that becomes the primary mechanism
|
||||
for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]):
|
||||
msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver)
|
||||
archmsg = []
|
||||
for arch in self.options.adt_arches.split():
|
||||
url = cloud_url % {'h': srchash(testsrc), 's': testsrc,
|
||||
'r': self.options.series, 'a': arch}
|
||||
try:
|
||||
if autopkgtest.test_results[testsrc][arch][1][testver][0]:
|
||||
status = 'PASS'
|
||||
else:
|
||||
if autopkgtest.test_results[testsrc][arch][2]:
|
||||
status = 'REGRESSION'
|
||||
else:
|
||||
status = 'ALWAYSFAIL'
|
||||
except KeyError:
|
||||
try:
|
||||
autopkgtest.pending_tests[testsrc][testver][arch]
|
||||
status = 'RUNNING'
|
||||
except KeyError:
|
||||
# neither done nor pending -> exclusion, or disabled
|
||||
continue
|
||||
|
||||
archmsg.append('<a href="%s">%s: %s</a>' %
|
||||
(url, arch, ADT_EXCUSES_LABELS[status]))
|
||||
|
||||
if archmsg:
|
||||
e.addhtml(msg + ', '.join(archmsg))
|
||||
# end of temporary code
|
||||
|
||||
if not adtpass and e.is_valid:
|
||||
hints = self.hints.search('force-skiptest', package=e.name)
|
||||
hints.extend(self.hints.search('force', package=e.name))
|
||||
|
||||
@ -89,7 +89,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
|
||||
(excuses, out) = self.run_britney()
|
||||
self.swift.stop()
|
||||
|
||||
#print('-------\nexcuses: %s\n-----' % excuses)
|
||||
if 'SHOW_EXCUSES' in os.environ:
|
||||
print('-------\nexcuses: %s\n-----' % excuses)
|
||||
if 'SHOW_OUTPUT' in os.environ:
|
||||
print('-------\nout: %s\n-----' % out)
|
||||
if considered:
|
||||
@ -125,9 +126,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
|
||||
|
||||
self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
|
||||
@ -153,7 +152,7 @@ lightgreen 1 i386 green 2
|
||||
self.assertEqual(self.pending_requests, expected_pending)
|
||||
|
||||
# if we run britney again this should *not* trigger any new tests
|
||||
self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1</a> to .*>2<'])
|
||||
self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1</a> to .*>2<'])
|
||||
self.assertEqual(self.amqp_requests, set())
|
||||
# but the set of pending tests doesn't change
|
||||
self.assertEqual(self.pending_requests, expected_pending)
|
||||
@ -164,9 +163,7 @@ lightgreen 1 i386 green 2
|
||||
# first run requests tests and marks them as pending
|
||||
self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
|
||||
@ -233,9 +230,7 @@ lightgreen 1 i386 green 2
|
||||
# first run requests tests and marks them as pending
|
||||
self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
|
||||
@ -252,9 +247,7 @@ lightgreen 1 i386 green 2
|
||||
|
||||
out = self.do_test(
|
||||
[],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
|
||||
@ -267,15 +260,96 @@ lightgreen 1 i386 green 2
|
||||
self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
|
||||
self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
|
||||
|
||||
def test_multi_rdepends_with_tests_regression(self):
|
||||
'''Multiple reverse dependencies with tests (regression)'''
|
||||
|
||||
self.swift.set_results({'autopkgtest-series': {
|
||||
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
|
||||
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
|
||||
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
|
||||
'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
|
||||
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
|
||||
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
|
||||
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
|
||||
'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
|
||||
'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
|
||||
}})
|
||||
|
||||
out = self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
|
||||
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
|
||||
|
||||
self.assertEqual(self.pending_requests, '')
|
||||
# not expecting any failures to retrieve from swift
|
||||
self.assertNotIn('Failure', out, out)
|
||||
|
||||
def test_multi_rdepends_with_tests_regression_last_pass(self):
|
||||
'''Multiple reverse dependencies with tests (regression), last one passes
|
||||
|
||||
This ensures that we don't just evaluate the test result of the last
|
||||
test, but all of them.
|
||||
'''
|
||||
|
||||
self.swift.set_results({'autopkgtest-series': {
|
||||
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
|
||||
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
|
||||
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
|
||||
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
|
||||
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
|
||||
'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
|
||||
'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
|
||||
}})
|
||||
|
||||
out = self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
|
||||
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
|
||||
|
||||
self.assertEqual(self.pending_requests, '')
|
||||
# not expecting any failures to retrieve from swift
|
||||
self.assertNotIn('Failure', out, out)
|
||||
|
||||
def test_multi_rdepends_with_tests_always_failed(self):
|
||||
'''Multiple reverse dependencies with tests (always failed)'''
|
||||
|
||||
self.swift.set_results({'autopkgtest-series': {
|
||||
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
|
||||
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
|
||||
'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
|
||||
'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
|
||||
'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
|
||||
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
|
||||
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
|
||||
'series/amd64/g/green/20150101_100200@': (4, 'green 2'),
|
||||
'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
|
||||
}})
|
||||
|
||||
out = self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
VALID_CANDIDATE,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed',
|
||||
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
|
||||
|
||||
self.assertEqual(self.pending_requests, '')
|
||||
# not expecting any failures to retrieve from swift
|
||||
self.assertNotIn('Failure', out, out)
|
||||
|
||||
def test_package_pair_running(self):
|
||||
'''Two packages in unstable that need to go in together (running)'''
|
||||
|
||||
self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
|
||||
('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'\blightgreen\b.*>1</a> to .*>2<'])
|
||||
|
||||
@ -313,9 +387,7 @@ lightgreen 2 i386 lightgreen 2
|
||||
|
||||
self.do_test(
|
||||
[('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\blightgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
|
||||
['in progress'])
|
||||
@ -341,9 +413,7 @@ lightgreen 2 i386 lightgreen 2
|
||||
|
||||
self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
|
||||
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
|
||||
@ -392,9 +462,7 @@ lightgreen 2 i386 lightgreen 2
|
||||
self.do_test(
|
||||
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
|
||||
('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\bgreen\b.*>1</a> to .*>2<',
|
||||
r'\blightgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
|
||||
@ -440,9 +508,7 @@ lightgreen 2 i386 lightgreen 2
|
||||
|
||||
self.do_test(
|
||||
[('lightgreen', {'Version': '2'}, 'autopkgtest')],
|
||||
# FIXME: while we only submit requests through AMQP, but don't consider
|
||||
# their results, we don't expect this to hold back stuff.
|
||||
VALID_CANDIDATE,
|
||||
NOT_CONSIDERED,
|
||||
[r'\blightgreen\b.*>1</a> to .*>2<',
|
||||
r'autopkgtest for lightgreen 2: .*amd64.*in progress.*i386.*in progress',
|
||||
r'autopkgtest for rainbow 1: .*amd64.*in progress.*i386.*in progress'])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user