Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-03-12 20:01:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement swift free usage of autopkgtest

Browse Source
This commit is contained in:
Paul Gevers 2017-09-24 15:23:28 +02:00
parent ce41819e05
commit 648feb71d3
No known key found for this signature in database
GPG Key ID: 9C5C99EB05BD750A
4 changed files with 124 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
britney.conf
View File

@ -89,3 +89,6 @@ ADT_PPAS =
# set this to the path of a (r/o) results.cache for running many parallel
# britney instances for PPAs without updating the cache
ADT_SHARED_RESULTS_CACHE =
# Swift base URL with the results (must be publicly readable and browsable)
# or file location if results are pre-fetched
ADT_SWIFT_URL = file://cache/debci.json

75
britney2/policies/autopkgtest.py
View File

@ -118,6 +118,21 @@ class AutopkgtestPolicy(BasePolicy):
self.log('%s does not exist, re-downloading all results from swift' %
self.results_cache_file)
# read in the new results
if self.options.adt_swift_url.startswith('file://'):
debci_file = self.options.adt_swift_url[7:]
if os.path.exists(debci_file):
with open(debci_file) as f:
test_results = json.load(f)
self.log('Read new results from %s' % debci_file)
for result in test_results:
(trigger, src, arch, ver, passed, stamp) = result
self.remove_from_pending(trigger, src, arch)
self.add_trigger_to_results(trigger, src, ver, arch, stamp, passed)
else:
self.log('%s does not exist, no new data will be processed' %
debci_file)
# we need sources, binaries, and installability tester, so for now
# remember the whole britney object
self.britney = britney
@ -557,35 +572,41 @@ class AutopkgtestPolicy(BasePolicy):
# remove matching test requests
for trigger in result_triggers:
try:
arch_list = self.pending_tests[trigger][src]
arch_list.remove(arch)
if not arch_list:
del self.pending_tests[trigger][src]
if not self.pending_tests[trigger]:
del self.pending_tests[trigger]
self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger))
except (KeyError, ValueError):
self.log('-> does not match any pending request for %s/%s' % (src, arch))
self.remove_from_pending(trigger, src, arch)
# add this result
for trigger in result_triggers:
# If a test runs because of its own package (newer version), ensure
# that we got a new enough version; FIXME: this should be done more
# generically by matching against testpkg-versions
(trigsrc, trigver) = trigger.split('/', 1)
if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0:
self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E')
continue
self.add_trigger_to_results(trigger, src, ver, arch, stamp, passed)
result = self.test_results.setdefault(trigger, {}).setdefault(
src, {}).setdefault(arch, [False, None, ''])
def remove_from_pending(self, trigger, src, arch):
try:
arch_list = self.pending_tests[trigger][src]
arch_list.remove(arch)
if not arch_list:
del self.pending_tests[trigger][src]
if not self.pending_tests[trigger]:
del self.pending_tests[trigger]
self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger))
except (KeyError, ValueError):
self.log('-> does not match any pending request for %s/%s' % (src, arch))
# don't clobber existing passed results with failures from re-runs
if passed or not result[0]:
result[0] = passed
result[1] = ver
result[2] = stamp
def add_trigger_to_results(self, trigger, src, ver, arch, stamp, passed):
# If a test runs because of its own package (newer version), ensure
# that we got a new enough version; FIXME: this should be done more
# generically by matching against testpkg-versions
(trigsrc, trigver) = trigger.split('/', 1)
if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0:
self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E')
return
result = self.test_results.setdefault(trigger, {}).setdefault(
src, {}).setdefault(arch, [False, None, ''])
# don't clobber existing passed results with failures from re-runs
if passed or not result[0]:
result[0] = passed
result[1] = ver
result[2] = stamp
def send_test_request(self, src, arch, trigger, huge=False):
'''Send out AMQP request for testing src/arch for trigger
@ -628,6 +649,8 @@ class AutopkgtestPolicy(BasePolicy):
# Don't re-request if we already have a result
try:
passed = self.test_results[trigger][src][arch][0]
if self.options.adt_swift_url.startswith('file://'):
return
if passed:
self.log('%s/%s triggered by %s already passed' % (src, arch, trigger))
return
@ -635,6 +658,10 @@ class AutopkgtestPolicy(BasePolicy):
(src, arch, trigger))
raise KeyError # fall through
except KeyError:
# Without swift we don't expect new results
if self.options.adt_swift_url.startswith('file://'):
pass
self.fetch_swift_results(self.options.adt_swift_url, src, arch)
# do we have one now?
try:

3
tests/__init__.py
View File

@ -388,8 +388,7 @@ ADT_AMQP = file://output/debci.input
ADT_PPAS =
ADT_SHARED_RESULTS_CACHE =
# TODO: remove next line
ADT_SWIFT_URL = overwritten_by_the_test_anyways
ADT_SWIFT_URL = http://localhost:18085
''')
assert os.path.exists(self.britney)

73
tests/test_autopkgtest.py
View File

@ -48,10 +48,6 @@ class T(TestBase):
for line in fileinput.input(self.britney_conf, inplace=True):
if 'ADT_AMQP' in line:
print('ADT_AMQP = file://%s' % self.fake_amqp)
elif 'ADT_SWIFT_URL' in line:
print('ADT_SWIFT_URL = http://localhost:18085')
elif 'ADT_ARCHES' in line:
print('ADT_ARCHES = amd64 i386')
else:
sys.stdout.write(line)
@ -2394,5 +2390,74 @@ class T(TestBase):
### self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': ppa, 'green': ppa})
def test_swift_url_is_file(self):
'''Run without swift but with debci file (as Debian does)'''
'''Based on test_multi_rdepends_with_tests_regression'''
'''Multiple reverse dependencies with tests (regression)'''
debci_file = os.path.join(self.data.path, 'debci.output')
# Don't use swift but debci output file
for line in fileinput.input(self.britney_conf, inplace=True):
if line.startswith('ADT_SWIFT_URL'):
print('ADT_SWIFT_URL = file://%s' % debci_file)
else:
sys.stdout.write(line)
with open(debci_file, 'w') as f:
f.write('''
[
["green/2", "darkgreen", "i386", "1", true, "20170917_100000"],
["green/2", "darkgreen", "amd64", "1", true, "20170917_100000"],
["green/1", "lightgreen", "i386", "1", true, "20170917_101000"],
["green/2", "lightgreen", "i386", "1", false, "20170917_101001"],
["green/1", "lightgreen", "amd64", "1", true, "20170917_101000"],
["green/2", "lightgreen", "amd64", "1", false, "20170917_101001"],
["green/2", "green", "i386", "2", true, "20170917_102000"],
["green/1", "green", "amd64", "2", true, "20170917_102000"],
["green/2", "green", "amd64", "2", false, "20170917_102001"]
]
''')
self.data.add_default_packages(green=False)
out, exc, _ = self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'},
'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'},
})
},
{'green': [('old-version', '1'), ('new-version', '2')]}
)
# should have links to log and history, but no artifacts (as this is
# not a PPA)
self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][0],
'REGRESSION')
link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][1])
self.assertEqual(link.path, os.path.join(debci_file, 'autopkgtest-testing/testing/amd64/l/lightgreen/20170917_101001/log.gz'))
self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][2:4],
['http://autopkgtest.ubuntu.com/packages/l/lightgreen/testing/amd64',
None])
# should have retry link for the regressions (not a stable URL, test
# separately)
link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][4])
self.assertEqual(link.netloc, 'autopkgtest.ubuntu.com')
self.assertEqual(link.path, '/request.cgi')
self.assertEqual(urllib.parse.parse_qs(link.query),
{'release': ['testing'], 'arch': ['amd64'],
'package': ['lightgreen'], 'trigger': ['green/2']})
# we already had all results before the run, so this should not trigger
# any new requests
self.assertEqual(self.amqp_requests, set())
self.assertEqual(self.pending_requests, {})
# not expecting any failures to retrieve from swift
self.assertNotIn('Failure', out, out)
if __name__ == '__main__':
unittest.main()