Lubuntu
/
britney2-ubuntu
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Stop accepting results without recorded trigger

Browse Source 
In situations where we don't have an up to date existing results.cache, the
fallback for handling test results would attribute old test results to new
requests, as we don't have a solid way to map them. This is detrimental for ad
hoc britney instances, like for testing PPAs, and also when we need to rebuild
our cache.

Ignore test results without a package trigger, and drop the code for handling
those.

The main risk here is that if we need to rebuild the cache from scratch we
might miss historic "PASS" results which haven't run since we switched to
recording triggers two months ago. But in these two months most of the
interesting packages have run, in particular for the development series and for
stable kernels, and non-kernel SRUs are not auto-promoted anyway.
bzr-import-20160707
Martin Pitt 9 years ago
parent 501aa206d8
commit 2ebfc0bb8f
2 changed files with 60 additions and 100 deletions
Show Stats Download Patch File Download Diff File

115
autopkgtest.py
Unescape View File

@ -351,7 +351,7 @@ class AutoPackageTest(object):
self.requested_tests.setdefault(src, {}).setdefault( self.requested_tests.setdefault(src, {}).setdefault(
ver, {}).setdefault(arch, set()).add((trigsrc, trigver)) ver, {}).setdefault(arch, set()).add((trigsrc, trigver))
def fetch_swift_results(self, swift_url, src, arch, trigger=None): def fetch_swift_results(self, swift_url, src, arch):
'''Download new results for source package/arch from swift''' '''Download new results for source package/arch from swift'''
# prepare query: get all runs with a timestamp later than latest_stamp # prepare query: get all runs with a timestamp later than latest_stamp
@ -387,14 +387,12 @@ class AutoPackageTest(object):
for p in result_paths: for p in result_paths:
self.fetch_one_result( self.fetch_one_result(
os.path.join(swift_url, 'autopkgtest-' + self.series, p, 'result.tar'), os.path.join(swift_url, 'autopkgtest-' + self.series, p, 'result.tar'), src, arch)
src, arch, trigger)
def fetch_one_result(self, url, src, arch, trigger=None): def fetch_one_result(self, url, src, arch):
'''Download one result URL for source/arch '''Download one result URL for source/arch
Remove matching pending_tests entries. If trigger is given (src, ver) Remove matching pending_tests entries.
it is added to the triggers of that result.
''' '''
try: try:
f = urlopen(url) f = urlopen(url)
@ -413,11 +411,7 @@ class AutoPackageTest(object):
exitcode = int(tar.extractfile('exitcode').read().strip()) exitcode = int(tar.extractfile('exitcode').read().strip())
srcver = tar.extractfile('testpkg-version').read().decode().strip() srcver = tar.extractfile('testpkg-version').read().decode().strip()
(ressrc, ver) = srcver.split() (ressrc, ver) = srcver.split()
try: testinfo = json.loads(tar.extractfile('testinfo.json').read().decode())
testinfo = json.loads(tar.extractfile('testinfo.json').read().decode())
except KeyError:
self.log_error('warning: %s does not have a testinfo.json' % url)
testinfo = {}
except (KeyError, ValueError, tarfile.TarError) as e: except (KeyError, ValueError, tarfile.TarError) as e:
self.log_error('%s is damaged, ignoring: %s' % (url, str(e))) self.log_error('%s is damaged, ignoring: %s' % (url, str(e)))
# ignore this; this will leave an orphaned request in pending.txt # ignore this; this will leave an orphaned request in pending.txt
@ -431,13 +425,13 @@ class AutoPackageTest(object):
return return
# parse recorded triggers in test result # parse recorded triggers in test result
if 'custom_environment' in testinfo: for e in testinfo.get('custom_environment', []):
for e in testinfo['custom_environment']: if e.startswith('ADT_TEST_TRIGGERS='):
if e.startswith('ADT_TEST_TRIGGERS='): result_triggers = [tuple(i.split('/', 1)) for i in e.split('=', 1)[1].split() if '/' in i]
result_triggers = [tuple(i.split('/', 1)) for i in e.split('=', 1)[1].split() if '/' in i] break
break
else: else:
result_triggers = None self.log_error('%s result has no ADT_TEST_TRIGGERS, ignoring')
return
stamp = os.path.basename(os.path.dirname(url)) stamp = os.path.basename(os.path.dirname(url))
# allow some skipped tests, but nothing else # allow some skipped tests, but nothing else
@ -446,68 +440,49 @@ class AutoPackageTest(object):
self.log_verbose('Fetched test result for %s/%s/%s %s (triggers: %s): %s' % ( self.log_verbose('Fetched test result for %s/%s/%s %s (triggers: %s): %s' % (
src, ver, arch, stamp, result_triggers, passed and 'pass' or 'fail')) src, ver, arch, stamp, result_triggers, passed and 'pass' or 'fail'))
# remove matching test requests, remember triggers # remove matching test requests
satisfied_triggers = set()
for request_map in [self.requested_tests, self.pending_tests]: for request_map in [self.requested_tests, self.pending_tests]:
for pending_ver, pending_archinfo in request_map.get(src, {}).copy().items(): for pending_ver, pending_archinfo in request_map.get(src, {}).copy().items():
# don't consider newer requested versions # don't consider newer requested versions
if apt_pkg.version_compare(pending_ver, ver) > 0: if apt_pkg.version_compare(pending_ver, ver) > 0:
continue continue
if result_triggers: for result_trigger in result_triggers:
# explicitly recording/retrieving test triggers is the
# preferred (and robust) way of matching results to pending
# requests
for result_trigger in result_triggers:
satisfied_triggers.add(result_trigger)
try:
request_map[src][pending_ver][arch].remove(result_trigger)
self.log_verbose('-> matches pending request %s/%s/%s for trigger %s' %
(src, pending_ver, arch, str(result_trigger)))
except (KeyError, ValueError):
self.log_verbose('-> does not match any pending request for %s/%s/%s' %
(src, pending_ver, arch))
else:
# ... but we still need to support results without
# testinfo.json and recorded triggers until we stop caring about
# existing wily and trusty results; match the latest result to all
# triggers for src that have at least the requested version
try: try:
t = pending_archinfo[arch] request_map[src][pending_ver][arch].remove(result_trigger)
self.log_verbose('-> matches pending request %s/%s for triggers %s' % self.log_verbose('-> matches pending request %s/%s/%s for trigger %s' %
(src, pending_ver, str(t))) (src, pending_ver, arch, str(result_trigger)))
satisfied_triggers.update(t) except (KeyError, ValueError):
del request_map[src][pending_ver][arch] self.log_verbose('-> does not match any pending request for %s/%s/%s' %
except KeyError: (src, pending_ver, arch))
self.log_verbose('-> does not match any pending request for %s/%s' %
(src, pending_ver))
# FIXME: this is a hack that mostly applies to re-running tests
# manually without giving a trigger. Tests which don't get
# triggered by a particular kernel version are fine with that, so
# add some heuristic once we drop the above code.
if trigger:
satisfied_triggers.add(trigger)
# add this result # add this result
src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}, False]) src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}, False])
if passed: trigmap = src_arch_results[1].setdefault(ver, {})
# update ever_passed field, unless we got triggered from for trig in result_triggers:
# linux-meta*: we trigger separate per-kernel tests for reverse trig_idx = trig[0] + '/' + trig[1]
# test dependencies, and we don't want to track per-trigger
# ever_passed. This would be wrong for everything except the # If a test runs because of its own package (newer version), ensure
# kernel, and the kernel team tracks per-kernel regressions already # that we got a new enough version; FIXME: this should be done more
if not result_triggers or not result_triggers[0][0].startswith('linux-meta'): # generically by matching against testpkg-versions
src_arch_results[2] = True if trig[0] == src and apt_pkg.version_compare(ver, trig[1]) < 0:
if satisfied_triggers: self.log_error('test trigger %s, but run for older version %s, ignoring' %
for trig in satisfied_triggers: (trig_idx, ver))
src_arch_results[1].setdefault(ver, {})[trig[0] + '/' + trig[1]] = passed continue
else:
# this result did not match any triggers? then we are in backwards # passed results are always good, but don't clobber existing passed
# compat mode for results without recorded triggers; update all # results with failures from re-runs
# results if passed or trig_idx not in trigmap:
for trig in src_arch_results[1].setdefault(ver, {}): trigmap[trig_idx] = passed
src_arch_results[1][ver][trig] = passed
# update ever_passed field, unless we got triggered from
# linux-meta*: we trigger separate per-kernel tests for reverse
# test dependencies, and we don't want to track per-trigger
# ever_passed. This would be wrong for everything except the
# kernel, and the kernel team tracks per-kernel regressions already
if passed and not result_triggers[0][0].startswith('linux-meta'):
src_arch_results[2] = True
# update latest_stamp # update latest_stamp
if stamp > src_arch_results[0]: if stamp > src_arch_results[0]:
src_arch_results[0] = stamp src_arch_results[0] = stamp
@ -640,7 +615,7 @@ class AutoPackageTest(object):
if arch not in self.pending_tests.get(trigpkg, {}).get(trigver, {}): if arch not in self.pending_tests.get(trigpkg, {}).get(trigver, {}):
self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' % self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' %
(pkg, arch, trigpkg, trigver)) (pkg, arch, trigpkg, trigver))
self.fetch_swift_results(self.britney.options.adt_swift_url, pkg, arch, (trigpkg, trigver)) self.fetch_swift_results(self.britney.options.adt_swift_url, pkg, arch)
# update the results cache # update the results cache
with open(self.results_cache_file + '.new', 'w') as f: with open(self.results_cache_file + '.new', 'w') as f:

45
tests/test_autopkgtest.py
Unescape View File

@ -303,7 +303,7 @@ lightgreen 1 i386 green 2
res = json.load(f) res = json.load(f)
self.assertEqual(res['green']['i386'], self.assertEqual(res['green']['i386'],
['20150101_100200@', ['20150101_100200@',
{'1': {}, '2': {'green/2': True}}, {'1': {'passedbefore/1': True}, '2': {'green/2': True}},
True]) True])
self.assertEqual(res['lightgreen']['amd64'], self.assertEqual(res['lightgreen']['amd64'],
['20150101_100101@', ['20150101_100101@',
@ -369,47 +369,31 @@ lightgreen 1 i386 green 2
self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
def test_multi_rdepends_with_tests_mixed_no_recorded_triggers(self): def test_results_without_triggers(self):
'''Multiple reverse dependencies with tests (mixed results), no recorded triggers''' '''Old results without recorded triggers'''
# green has passed before on i386 only, therefore ALWAYSFAILED on amd64
self.swift.set_results({'autopkgtest-series': {
'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')),
}})
# first run requests tests and marks them as pending
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (False, {'green 2': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'RUNNING'},
'lightgreen 1': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'RUNNING-ALWAYSFAILED'},
'darkgreen 1': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'RUNNING-ALWAYSFAILED'},
})
},
{'green': [('old-version', '1'), ('new-version', '2')]})
# second run collects the results
self.swift.set_results({'autopkgtest-series': { self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/i386/g/green/20150101_100100@': (0, 'green 1', tr('passedbefore/1')),
'series/i386/g/green/20150101_100200@': (0, 'green 2'), 'series/i386/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100201@': (4, 'green 2'), 'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
}}) }})
out = self.do_test( # none of the above results should be accepted
[], self.do_test(
{'green': (False, {'green 2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'}, [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'RUNNING-ALWAYSFAILED'}, {'green': (False, {'green 2': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'RUNNING'},
'darkgreen 1': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'PASS'}, 'lightgreen 1': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'RUNNING-ALWAYSFAILED'},
'darkgreen 1': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'RUNNING-ALWAYSFAILED'},
}) })
}) })
# not expecting any failures to retrieve from swift
self.assertNotIn('Failure', out, out)
# there should be some pending ones # there should be some pending ones
self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
self.assertIn('green 2 i386 green 2', self.pending_requests)
def test_multi_rdepends_with_tests_regression(self): def test_multi_rdepends_with_tests_regression(self):
'''Multiple reverse dependencies with tests (regression)''' '''Multiple reverse dependencies with tests (regression)'''
@ -1452,12 +1436,13 @@ fancy 1 i386 linux-meta-lts-grumpy 1
('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None),
], ],
{'linux-meta': (True, {'fancy 1': {'amd64': 'PASS', 'i386': 'PASS'}}), {'linux-meta': (True, {'fancy 1': {'amd64': 'PASS', 'i386': 'PASS'}}),
# we don't have an explicit result for amd64, so the old one counts # we don't have an explicit result for amd64
'linux-meta-lts-grumpy': (True, {'fancy 1': {'amd64': 'ALWAYSFAIL', 'i386': 'ALWAYSFAIL'}}), 'linux-meta-lts-grumpy': (True, {'fancy 1': {'amd64': 'RUNNING-ALWAYSFAILED', 'i386': 'ALWAYSFAIL'}}),
'linux-meta-64only': (True, {'fancy 1': {'amd64': 'PASS'}}), 'linux-meta-64only': (True, {'fancy 1': {'amd64': 'PASS'}}),
}) })
self.assertEqual(self.pending_requests, '') self.assertEqual(self.pending_requests,
'fancy 1 amd64 linux-meta-lts-grumpy 1\n')
def test_kernel_triggered_tests(self): def test_kernel_triggered_tests(self):
'''linux, lxc, glibc tests get triggered by linux-meta* uploads''' '''linux, lxc, glibc tests get triggered by linux-meta* uploads'''

Write Preview
Loading…
Cancel
Save