Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-04-15 13:11:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Autopkgtest: Track/cache results by triggering package

Browse Source 
When downloading results, check the ADT_TEST_TRIGGERS var in testinfo.json to
get back the original trigger that previous britney runs requested the test run
for. This allows us to much more precisely map a test result to an original
test request.

In order to tell apart test results which pass/fail depending on the package
which triggers them (in particular, if that is a kernel), we must keep track of
pass/fail results on a per-trigger granularity. Rearrange the results map
accordingly.

Keep the old "latest test result applies to all triggers of this pkg/version"
logic around as long as we still have existing test results without
testinfo.json.

ATTENTION: This breaks the format of results.cache, so this needs to be removed
when rolling this out.
This commit is contained in:
Martin Pitt 2015-09-18 06:46:34 +02:00
parent 0eddac8476
commit 19cd69cb47
2 changed files with 150 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
autopkgtest.py
View File

@ -41,19 +41,6 @@ def srchash(src):
return src[0]
def merge_triggers(trigs1, trigs2):
'''Merge two (pkg, ver) trigger iterables
Return [(pkg, ver), ...] list with only the highest version for each
package.
'''
pkgvers = {}
for pkg, ver in itertools.chain(trigs1, trigs2):
if apt_pkg.version_compare(ver, pkgvers.setdefault(pkg, '0')) >= 0:
pkgvers[pkg] = ver
return list(pkgvers.items())
def latest_item(ver_map, min_version=None):
'''Return (ver, value) from version -> value map with latest version number
@ -102,16 +89,16 @@ class AutoPackageTest(object):
os.mkdir(self.test_state_dir)
self.read_pending_tests()
# results map: src -> arch -> [latest_stamp, ver -> (passed, triggers), ever_passed]
# - "passed" is a bool
# results map: src -> arch -> [latest_stamp, ver -> trigger -> passed, ever_passed]
# - It's tempting to just use a global "latest" time stamp, but due to
# swift's "eventual consistency" we might miss results with older time
# stamps from other packages that we don't see in the current run, but
# will in the next one. This doesn't hurt for older results of the same
# package.
# - triggers is a list of (source, version) pairs which unstable
# packages triggered this test run. We need to track this to avoid
# unnecessarily re-running tests.
# - trigger is "source/version" of an unstable package that triggered
# this test run. We need to track this to avoid unnecessarily
# re-running tests.
# - "passed" is a bool
# - ever_passed is a bool whether there is any successful test of
# src/arch of any version. This is used for detecting "regression"
# vs. "always failed"
@ -314,12 +301,13 @@ class AutoPackageTest(object):
# check for existing results for both the requested and the current
# unstable version: test runs might see newly built versions which we
# didn't see in britney yet
pkg_arch_results = self.test_results.get(src, {}).get(arch, [None, {}, None])[1]
ver_trig_results = self.test_results.get(src, {}).get(arch, [None, {}, None])[1]
unstable_ver = self.britney.sources['unstable'][src][VERSION]
for result_ver in [ver, unstable_ver]:
if result_ver not in pkg_arch_results or apt_pkg.version_compare(result_ver, ver) < 0:
if result_ver not in ver_trig_results or apt_pkg.version_compare(result_ver, ver) < 0:
continue
for (tsrc, tver) in pkg_arch_results[result_ver][1]:
for trigger in ver_trig_results[result_ver]:
(tsrc, tver) = trigger.split('/', 1)
if tsrc == trigsrc and apt_pkg.version_compare(tver, trigver) >= 0:
self.log_verbose('There already is a result for %s/%s/%s triggered by %s/%s' %
(src, result_ver, arch, tsrc, tver))
@ -395,6 +383,11 @@ class AutoPackageTest(object):
exitcode = int(tar.extractfile('exitcode').read().strip())
srcver = tar.extractfile('testpkg-version').read().decode().strip()
(ressrc, ver) = srcver.split()
try:
testinfo = json.loads(tar.extractfile('testinfo.json').read().decode())
except KeyError:
self.log_error('warning: %s does not have a testinfo.json' % url)
testinfo = {}
except (KeyError, ValueError, tarfile.TarError) as e:
self.log_error('%s is damaged, ignoring: %s' % (url, str(e)))
# ignore this; this will leave an orphaned request in pending.txt
@ -407,37 +400,78 @@ class AutoPackageTest(object):
(url, ressrc, src))
return
# parse recorded triggers in test result
if 'custom_environment' in testinfo:
for e in testinfo['custom_environment']:
if e.startswith('ADT_TEST_TRIGGERS='):
result_triggers = [tuple(i.split('/', 1)) for i in e.split('=', 1)[1].split() if '/' in i]
break
else:
result_triggers = None
stamp = os.path.basename(os.path.dirname(url))
# allow some skipped tests, but nothing else
passed = exitcode in [0, 2]
self.log_verbose('Fetched test result for %s/%s/%s %s: %s' % (
src, ver, arch, stamp, passed and 'pass' or 'fail'))
self.log_verbose('Fetched test result for %s/%s/%s %s (triggers: %s): %s' % (
src, ver, arch, stamp, result_triggers, passed and 'pass' or 'fail'))
# remove matching test requests, remember triggers
satisfied_triggers = set()
for pending_ver, pending_archinfo in self.pending_tests.get(src, {}).copy().items():
# don't consider newer requested versions
if apt_pkg.version_compare(pending_ver, ver) <= 0:
if apt_pkg.version_compare(pending_ver, ver) > 0:
continue
if result_triggers:
# explicitly recording/retrieving test triggers is the
# preferred (and robust) way of matching results to pending
# requests
for result_trigger in result_triggers:
try:
self.pending_tests[src][pending_ver][arch].remove(result_trigger)
self.log_verbose('-> matches pending request %s/%s/%s for trigger %s' %
(src, pending_ver, arch, str(result_trigger)))
satisfied_triggers.add(result_trigger)
except (KeyError, ValueError):
self.log_verbose('-> does not match any pending request for %s/%s/%s' %
(src, pending_ver, arch))
else:
# ... but we still need to support results without
# testinfo.json and recorded triggers until we stop caring about
# existing wily and trusty results; match the latest result to all
# triggers for src that have at least the requested version
try:
t = pending_archinfo[arch]
self.log_verbose('-> matches pending request for triggers %s' % str(t))
self.log_verbose('-> matches pending request %s/%s for triggers %s' %
(src, pending_ver, str(t)))
satisfied_triggers.update(t)
del self.pending_tests[src][pending_ver][arch]
except KeyError:
self.log_error('-> does not match any pending request!')
pass
self.log_verbose('-> does not match any pending request for %s/%s' %
(src, pending_ver))
# FIXME: this is a hack that mostly applies to re-running tests
# manually without giving a trigger. Tests which don't get
# triggered by a particular kernel version are fine with that, so
# add some heuristic once we drop the above code.
if trigger:
satisfied_triggers.add(trigger)
# add this result
src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}, False])
if ver is not None:
if passed:
# update ever_passed field
src_arch_results[2] = True
src_arch_results[1][ver] = (passed, merge_triggers(
src_arch_results[1].get(ver, (None, []))[1], satisfied_triggers))
if passed:
# update ever_passed field
src_arch_results[2] = True
if satisfied_triggers:
for trig in satisfied_triggers:
src_arch_results[1].setdefault(ver, {})[trig[0] + '/' + trig[1]] = passed
else:
# this result did not match any triggers? then we are in backwards
# compat mode for results without recorded triggers; update all
# results
for trig in src_arch_results[1].setdefault(ver, {}):
src_arch_results[1][ver][trig] = passed
# update latest_stamp
if stamp > src_arch_results[0]:
src_arch_results[0] = stamp
@ -446,15 +480,12 @@ class AutoPackageTest(object):
'''Return (src, arch) set for failed tests for given trigger pkg'''
result = set()
trigger = trigsrc + '/' + trigver
for src, srcinfo in self.test_results.items():
for arch, (stamp, vermap, ever_passed) in srcinfo.items():
for ver, (passed, triggers) in vermap.items():
if not passed:
# triggers might contain tuples or lists (after loading
# from json), so iterate/check manually
for s, v in triggers:
if trigsrc == s and trigver == v:
result.add((src, arch))
for ver, trig_results in vermap.items():
if trig_results.get(trigger) is False:
result.add((src, arch))
return result
#
@ -498,7 +529,7 @@ class AutoPackageTest(object):
kernel_triggers = set()
nonkernel_triggers = set()
for archinfo in verinfo.values():
for (t, v) in archinfo[arch]:
for (t, v) in archinfo.get(arch, []):
if t.startswith('linux-meta'):
kernel_triggers.add(t + '/' + v)
else:
@ -584,6 +615,7 @@ class AutoPackageTest(object):
'''
# (src, ver) -> arch -> ALWAYSFAIL|PASS|FAIL|RUNNING
pkg_arch_result = {}
trigger = trigsrc + '/' + trigver
for arch in self.britney.options.adt_arches.split():
for testsrc, testver in self.tests_for_source(trigsrc, trigver, arch):
@ -595,15 +627,13 @@ class AutoPackageTest(object):
# runs might see built versions which we didn't see in
# britney yet
try:
(status, triggers) = ver_map[testver]
if not status:
trigger_results = ver_map[testver]
if not trigger_results[trigger]:
raise KeyError
except KeyError:
(testver, (status, triggers)) = latest_item(ver_map, testver)
(testver, trigger_results) = latest_item(ver_map, testver)
# triggers might contain tuples or lists
if (trigsrc, trigver) not in triggers and [trigsrc, trigver] not in triggers:
raise KeyError('No result for trigger %s/%s yet' % (trigsrc, trigver))
status = trigger_results[trigger]
if status:
result = 'PASS'
else:
@ -623,8 +653,8 @@ class AutoPackageTest(object):
if not hasattr(self.britney.options, 'adt_swift_url'):
continue
# FIXME: Ignore this error for now as it crashes britney, but investigate!
self.log_error('FIXME: Result for %s/%s/%s (triggered by %s/%s) is neither known nor pending!' %
(testsrc, testver, arch, trigsrc, trigver))
self.log_error('FIXME: Result for %s/%s/%s (triggered by %s) is neither known nor pending!' %
(testsrc, testver, arch, trigger))
continue
pkg_arch_result.setdefault((testsrc, testver), {})[arch] = result

74
tests/test_autopkgtest.py
View File

@ -103,7 +103,8 @@ class TestAutoPkgTest(TestBase):
print('------- output -----\n%s\n' % out)
for src, (is_candidate, testmap) in expect_status.items():
self.assertEqual(excuses_dict[src]['is-candidate'], is_candidate)
self.assertEqual(excuses_dict[src]['is-candidate'], is_candidate,
src + ': ' + str(excuses_dict[src]))
for testsrc, archmap in testmap.items():
for arch, status in archmap.items():
self.assertEqual(excuses_dict[src]['tests']['autopkgtest'][testsrc][arch][0],
@ -241,11 +242,11 @@ lightgreen 1 i386 green 2
res = json.load(f)
self.assertEqual(res['green']['i386'],
['20150101_100200@',
{'1': [False, []], '2': [True, [['green', '2']]]},
{'1': {}, '2': {'green/2': True}},
True])
self.assertEqual(res['lightgreen']['amd64'],
['20150101_100101@',
{'1': [True, [['green', '2']]]},
{'1': {'green/2': True}},
True])
# third run should not trigger any new tests, should all be in the
@ -265,6 +266,46 @@ lightgreen 1 i386 green 2
def test_multi_rdepends_with_tests_mixed(self):
'''Multiple reverse dependencies with tests (mixed results)'''
# first run requests tests and marks them as pending
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
})
},
{'green': [('old-version', '1'), ('new-version', '2')]})
# second run collects the results
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', {'custom_environment': ['ADT_TEST_TRIGGERS=green/2']}),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', {'custom_environment': ['ADT_TEST_TRIGGERS=green/2']}),
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', {'custom_environment': ['ADT_TEST_TRIGGERS=green/2']}),
'series/i386/g/green/20150101_100200@': (0, 'green 2', {'custom_environment': ['ADT_TEST_TRIGGERS=green/2']}),
'series/amd64/g/green/20150101_100201@': (4, 'green 2', {'custom_environment': ['ADT_TEST_TRIGGERS=green/2']}),
# unrelated results (wrong trigger), ignore this!
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', {'custom_environment': ['ADT_TEST_TRIGGERS=green/1']}),
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', {'custom_environment': ['ADT_TEST_TRIGGERS=blue/1']}),
}})
out = self.do_test(
[],
{'green': (False, {'green 2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'},
'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'RUNNING'},
'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'PASS'},
})
})
# not expecting any failures to retrieve from swift
self.assertNotIn('Failure', out, out)
# there should be some pending ones
self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
def test_multi_rdepends_with_tests_mixed_no_recorded_triggers(self):
'''Multiple reverse dependencies with tests (mixed results), no recorded triggers'''
# first run requests tests and marks them as pending
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
@ -1194,6 +1235,33 @@ fancy 1 i386 linux-meta-lts-grumpy 1
'''
self.assertEqual(self.pending_requests, expected_pending)
def test_dkms_results_per_kernel(self):
'''DKMS results get mapped to the triggering kernel version'''
self.data.add('dkms', False, {})
self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'})
# works against linux-meta and -64only, fails against grumpy i386, no
# result yet for grumpy amd64
self.swift.set_results({'autopkgtest-series': {
'series/i386/f/fancy/20150101_100101@': (0, 'fancy 1', {'custom_environment': ['ADT_TEST_TRIGGERS=linux-meta/1']}),
'series/amd64/f/fancy/20150101_100101@': (0, 'fancy 1', {'custom_environment': ['ADT_TEST_TRIGGERS=linux-meta/1']}),
'series/amd64/f/fancy/20150101_100201@': (0, 'fancy 1', {'custom_environment': ['ADT_TEST_TRIGGERS=linux-meta-64only/1']}),
'series/i386/f/fancy/20150101_100301@': (4, 'fancy 1', {'custom_environment': ['ADT_TEST_TRIGGERS=linux-meta-lts-grumpy/1']}),
}})
self.do_test(
[('linux-image-generic', {'Source': 'linux-meta'}, None),
('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None),
('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None),
],
{'linux-meta': (True, {'fancy 1': {'amd64': 'PASS', 'i386': 'PASS'}}),
'linux-meta-lts-grumpy': (False, {'fancy 1': {'amd64': 'RUNNING', 'i386': 'REGRESSION'}}),
'linux-meta-64only': (True, {'fancy 1': {'amd64': 'PASS'}}),
})
self.assertEqual(self.pending_requests, 'fancy 1 amd64 linux-meta-lts-grumpy 1\n')
def test_kernel_triggers_lxc(self):
'''LXC test gets triggered by kernel uploads'''