From ce775eeb5d374e45ad0068128698eab7a67c937f Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 10 Jul 2015 06:21:46 +0200 Subject: [PATCH] Add test results from swift Until now, autopkgtest results were triggered via an external "adt-britney" command from lp:auto-package-testing. This required a lot of state files and duplicated effort, uses hardcoded absolute paths to these external tools, and is quite hard to understand and maintain. We also want to move away from Jenkins and rsyncing state files. Directly retrieve autopkgtest results from a publicly readable and browsable Swift container, with a debci-compatible layout (https://wiki.debian.org/debci/DistributedSpec). This now tracks both requests and results on a per-architecture granularity, so that we can track per-architecture regressions/always-failed. Introduce a new ADT_SWIFT_URL config option that sets the swift base URL. If this key is not set, the behaviour does not change compared to previous versions, and no results will be retrieved from the cloud. This still keeps the old adt-britney requests/results as the authoritative data and for now merely shows the swift results in addition. With that we can compare the results and run the cloud testing in parallel to find/fix problems until we switch over. Due to that, the code to britney.py is temporary, does *not* use AutoPackageTest.results(), and instead just reads the internal results map. --- autopkgtest.py | 203 +++++++++++++++++++++++++++++++++++++- britney.conf | 2 + britney.py | 30 +++++- britney_nobreakall.conf | 3 +- tests/test_autopkgtest.py | 135 +++++++++++++++++++++++-- 5 files changed, 360 insertions(+), 13 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index 5252a53..5ab9c8d 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -24,8 +24,14 @@ import subprocess import tempfile from textwrap import dedent import time -import apt_pkg +import json +import tarfile +import io +import copy +import itertools +from urllib import urlencode, urlopen +import apt_pkg import kombu from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION) @@ -42,6 +48,28 @@ ADT_EXCUSES_LABELS = { } +def srchash(src): + '''archive hash prefix for source package''' + + if src.startswith('lib'): + return src[:4] + else: + return src[0] + + +def merge_triggers(trigs1, trigs2): + '''Merge two (pkg, ver) trigger iterables + + Return [(pkg, ver), ...] list with only the highest version for each + package. + ''' + pkgvers = {} + for pkg, ver in itertools.chain(trigs1, trigs2): + if apt_pkg.version_compare(ver, pkgvers.setdefault(pkg, '0')) >= 0: + pkgvers[pkg] = ver + return list(pkgvers.items()) + + class AutoPackageTest(object): """autopkgtest integration @@ -71,6 +99,28 @@ class AutoPackageTest(object): os.mkdir(self.test_state_dir) self.read_pending_tests() + # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers)] + # - "passed" is a bool + # - It's tempting to just use a global "latest" time stamp, but due to + # swift's "eventual consistency" we might miss results with older time + # stamps from other packages that we don't see in the current run, but + # will in the next one. This doesn't hurt for older results of the same + # package. + # - triggers is a list of (source, version) pairs which unstable + # packages triggered this test run. We need to track this to avoid + # unnecessarily re-running tests. + self.test_results = {} + self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') + + # read the cached results that we collected so far + if os.path.exists(self.results_cache_file): + with open(self.results_cache_file) as f: + self.test_results = json.load(f) + self.log_verbose('Read previous results from %s' % self.results_cache_file) + else: + self.log_verbose('%s does not exist, re-downloading all results ' + 'from swift' % self.results_cache_file) + def log_verbose(self, msg): if self.britney.options.verbose: print('I: [%s] - %s' % (time.asctime(), msg)) @@ -175,16 +225,124 @@ class AutoPackageTest(object): '''Add one test request to the local self.requested_tests queue This will only be done if that test wasn't already requested in a - previous run, i. e. it is already in self.pending_tests. + previous run (i. e. not already in self.pending_tests) or there already + is a result for it. ''' + try: + for (tsrc, tver) in self.test_results[src][arch][1][ver][1]: + if tsrc == trigsrc and apt_pkg.version_compare(tver, trigver) >= 0: + self.log_verbose('There already is a result for %s/%s/%s triggered by %s/%s' % + (src, ver, arch, tsrc, tver)) + return + except KeyError: + pass + if (trigsrc, trigver) in self.pending_tests.get(src, {}).get( - ver, {}).get(arch, set()): + ver, {}).get(arch, set()): self.log_verbose('test %s/%s/%s for %s/%s is already pending, not queueing' % (src, ver, arch, trigsrc, trigver)) return self.requested_tests.setdefault(src, {}).setdefault( ver, {}).setdefault(arch, set()).add((trigsrc, trigver)) + def fetch_swift_results(self, swift_url, src, arch): + '''Download new results for source package/arch from swift''' + + # prepare query: get all runs with a timestamp later than latest_stamp + # for this package/arch; '@' is at the end of each run timestamp, to + # mark the end of a test run directory path + # example: wily/amd64/libp/libpng/20150630_054517@/result.tar + query = {'delimiter': '@', + 'prefix': '%s/%s/%s/%s/' % (self.series, arch, srchash(src), src)} + try: + # don't include the last run again, so make the marker + # "infinitesimally later" by appending 'zz' + query['marker'] = self.test_results[src][arch][0] + 'zz' + except KeyError: + # no stamp yet, download all results + pass + + # request new results from swift + url = os.path.join(swift_url, 'autopkgtest-' + self.series) + url += '?' + urlencode(query) + try: + f = urlopen(url) + if f.getcode() == 200: + result_paths = f.read().strip().splitlines() + else: + self.log_error('Failure to fetch swift results from %s: %u' % + (url, f.getcode())) + f.close() + return + f.close() + except IOError as e: + self.log_error('Failure to fetch swift results from %s: %s' % (url, str(e))) + return + + for p in result_paths: + self.fetch_one_result(os.path.join( + swift_url, 'autopkgtest-' + self.series, p, 'result.tar'), src, arch) + + def fetch_one_result(self, url, src, arch): + '''Download one result URL for source/arch + + Remove matching pending_tests entries. + ''' + try: + f = urlopen(url) + if f.getcode() == 200: + tar_bytes = io.BytesIO(f.read()) + f.close() + else: + self.log_error('Failure to fetch %s: %u' % (url, f.getcode())) + return + except IOError as e: + self.log_error('Failure to fetch %s: %s' % (url, str(e))) + return + + try: + with tarfile.open(None, 'r', tar_bytes) as tar: + exitcode = int(tar.extractfile('exitcode').read().strip()) + srcver = tar.extractfile('testpkg-version').read().decode().strip() + (ressrc, ver) = srcver.split() + except (KeyError, ValueError, tarfile.TarError) as e: + self.log_error('%s is damaged: %s' % (url, str(e))) + return + + if src != ressrc: + self.log_error('%s is a result for package %s, but expected package %s' % + (url, ressrc, src)) + return + + stamp = os.path.basename(os.path.dirname(url)) + # allow some skipped tests, but nothing else + passed = exitcode in [0, 2] + + self.log_verbose('Fetched test result for %s/%s on %s: %s' % ( + src, ver, arch, passed and 'pass' or 'fail')) + + # remove matching test requests, remember triggers + satisfied_triggers = set() + for pending_ver, pending_archinfo in self.pending_tests.get(src, {}).copy().items(): + # don't consider newer requested versions + if apt_pkg.version_compare(pending_ver, ver) <= 0: + try: + t = pending_archinfo[arch] + self.log_verbose('-> matches pending request for triggers %s' % str(t)) + satisfied_triggers.update(t) + del self.pending_tests[src][pending_ver][arch] + except KeyError: + self.log_error('-> does not match any pending request!') + pass + + # add this result + src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}]) + src_arch_results[1][ver] = (passed, merge_triggers( + src_arch_results[1].get(ver, (None, []))[1], satisfied_triggers)) + # update latest_stamp + if stamp > src_arch_results[0]: + src_arch_results[0] = stamp + # # obsolete adt-britney helpers # @@ -397,6 +555,38 @@ class AutoPackageTest(object): self._adt_britney("submit", request_path) def collect(self): + # fetch results from swift + try: + swift_url = self.britney.options.adt_swift_url + except AttributeError: + self.log_error('ADT_SWIFT_URL not set, cannot collect results') + swift_url = None + try: + self.britney.options.adt_amqp + except AttributeError: + self.log_error('ADT_AMQP not set, not collecting results from swift') + swift_url = None + + if swift_url: + # update results from swift for all packages that we are waiting + # for, and remove pending tests that we have results for on all + # arches + for pkg, verinfo in copy.deepcopy(self.pending_tests.items()): + for archinfo in verinfo.values(): + for arch in archinfo: + self.fetch_swift_results(swift_url, pkg, arch) + + # update the results cache + with open(self.results_cache_file + '.new', 'w') as f: + json.dump(self.test_results, f, indent=2) + os.rename(self.results_cache_file + '.new', self.results_cache_file) + self.log_verbose('Updated results cache') + + # new results remove pending requests, update the on-disk cache + self.update_pending_tests() + + # deprecated results for old Jenkins/lp:auto-package-testing, will go + # away self._ensure_rc_file() result_path = self._result_path self._adt_britney("collect", "-O", result_path) @@ -413,6 +603,13 @@ class AutoPackageTest(object): (src, ver, trigsrc, trigver, status)) def results(self, trigsrc, trigver): + '''Return test results for triggering package + + Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests + that got triggered by trigsrc/trigver. + ''' + # deprecated results for old Jenkins/lp:auto-package-testing, will go + # away for status, src, ver in self.pkgcauses[trigsrc][trigver]: # Check for regression if status == 'FAIL': diff --git a/britney.conf b/britney.conf index 0e13542..28e6be9 100644 --- a/britney.conf +++ b/britney.conf @@ -67,6 +67,8 @@ ADT_DEBUG = no ADT_ARCHES = amd64 i386 # comment this to disable autopkgtest requests ADT_AMQP = ampq://user:pwd@amqp.example.com +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.mycloud.example.com/v1/AUTH_autopkgtest BOOTTEST_ENABLE = yes BOOTTEST_DEBUG = yes diff --git a/britney.py b/britney.py index c35cb16..48e51b7 100755 --- a/britney.py +++ b/britney.py @@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes, from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) -from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS +from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash from boottest import BootTest @@ -1853,6 +1853,7 @@ class Britney(object): jenkins_private = ( "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" % self.options.series.title()) + cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" for e in autopkgtest_excuses: adtpass = True for status, adtsrc, adtver in autopkgtest.results( @@ -1883,6 +1884,33 @@ class Britney(object): "%s" % (adtsrc, adtver, forces[0].user)) else: adtpass = False + + # temporary: also show results from cloud based tests, + # until that becomes the primary mechanism + for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]): + msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver) + archmsg = [] + for arch in self.options.adt_arches.split(): + url = cloud_url % {'h': srchash(testsrc), 's': testsrc, + 'r': self.options.series, 'a': arch} + try: + r = autopkgtest.test_results[testsrc][arch][1][testver][0] + status = r and 'PASS' or 'REGRESSION' + except KeyError: + try: + autopkgtest.pending_tests[testsrc][testver][arch] + status = 'RUNNING' + except KeyError: + # neither done nor pending -> exclusion, or disabled + continue + + archmsg.append('%s: %s' % + (url, arch, ADT_EXCUSES_LABELS[status])) + + if archmsg: + e.addhtml(msg + ', '.join(archmsg)) + # end of temporary code + if not adtpass and e.is_valid: hints = self.hints.search('force-skiptest', package=e.name) hints.extend(self.hints.search('force', package=e.name)) diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index 5aa483d..8db5a1f 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -67,4 +67,5 @@ ADT_DEBUG = no ADT_ARCHES = amd64 i386 # comment this to disable autopkgtest requests ADT_AMQP = ampq://user:pwd@amqp.example.com - +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.mycloud.example.com/v1/AUTH_autopkgtest diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index a46c4a6..42f477f 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -13,12 +13,13 @@ import sys import subprocess import fileinput import unittest +import json PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, PROJECT_DIR) from autopkgtest import ADT_EXCUSES_LABELS -from tests import TestBase +from tests import TestBase, mock_swift NOT_CONSIDERED = False VALID_CANDIDATE = True @@ -34,12 +35,14 @@ class TestAutoPkgTest(TestBase): super(TestAutoPkgTest, self).setUp() self.fake_amqp = os.path.join(self.data.path, 'amqp') - # Disable boottests and set fake AMQP server + # Disable boottests and set fake AMQP and Swift server for line in fileinput.input(self.britney_conf, inplace=True): if line.startswith('BOOTTEST_ENABLE'): print('BOOTTEST_ENABLE = no') elif line.startswith('ADT_AMQP'): print('ADT_AMQP = file://%s' % self.fake_amqp) + elif line.startswith('ADT_SWIFT_URL'): + print('ADT_SWIFT_URL = http://localhost:18085') else: sys.stdout.write(line) @@ -70,11 +73,22 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) testsuite='specialtest') self.data.add('justdata', False, {'Architecture': 'all'}) + # create mock Swift server (but don't start it yet, as tests first need + # to poke in results) + self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085) + self.swift.set_results({}) + + def tearDown(self): + del self.swift + def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None): for (pkg, fields, testsuite) in unstable_add: self.data.add(pkg, True, fields, True, testsuite) + self.swift.start() (excuses, out) = self.run_britney() + self.swift.stop() + #print('-------\nexcuses: %s\n-----' % excuses) #print('-------\nout: %s\n-----' % out) #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) @@ -105,6 +119,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) except IOError: self.pending_requests = None + return out + def test_multi_rdepends_with_tests_all_running(self): '''Multiple reverse dependencies with tests (all running)''' @@ -113,7 +129,10 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) # FIXME: while we only submit requests through AMQP, but don't consider # their results, we don't expect this to hold back stuff. VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<']) + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) # we expect the package's and its reverse dependencies' tests to get # triggered @@ -121,8 +140,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) self.amqp_requests, set(['debci-series-i386:green', 'debci-series-amd64:green', 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', - 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen', - ])) + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) os.unlink(self.fake_amqp) # ... and that they get recorded as pending @@ -141,6 +159,108 @@ lightgreen 1 i386 green 2 # but the set of pending tests doesn't change self.assertEqual(self.pending_requests, expected_pending) + def test_multi_rdepends_with_tests_all_pass(self): + '''Multiple reverse dependencies with tests (all pass)''' + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1'), + # version in testing fails + 'series/i386/g/green/20150101_020000@': (4, 'green 1'), + 'series/amd64/g/green/20150101_020000@': (4, 'green 1'), + # version in unstable succeeds + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2'), + }}) + + out = self.do_test( + [], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + # all tests ran, there should be no more pending ones + self.assertEqual(self.pending_requests, '') + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + # caches the results and triggers + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: + res = json.load(f) + self.assertEqual(res['green']['i386'], + ['20150101_100200@', {'1': [False, []], + '2': [True, [['green', '2']]]}]) + self.assertEqual(res['lightgreen']['amd64'], + ['20150101_100101@', {'1': [True, [['green', '2']]]}]) + + # third run should not trigger any new tests, should all be in the + # cache + os.unlink(self.fake_amqp) + self.swift.set_results({}) + out = self.do_test( + [], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, '') + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_mixed(self): + '''Multiple reverse dependencies with tests (mixed results)''' + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + self.do_test( + [], + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass']) + + # there should be some pending ones + self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) + self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) + def test_package_pair_running(self): '''Two packages in unstable that need to go in together (running)''' @@ -159,8 +279,7 @@ lightgreen 1 i386 green 2 self.amqp_requests, set(['debci-series-i386:green', 'debci-series-amd64:green', 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', - 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen', - ])) + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) os.unlink(self.fake_amqp) # ... and that they get recorded as pending @@ -180,7 +299,7 @@ lightgreen 2 i386 lightgreen 2 # Disable AMQP server config for line in fileinput.input(self.britney_conf, inplace=True): - if not line.startswith('ADT_AMQP'): + if not line.startswith('ADT_AMQP') and not line.startswith('ADT_SWIFT_URL'): sys.stdout.write(line) self.do_test(