diff --git a/autopkgtest.py b/autopkgtest.py index 5252a53..5ab9c8d 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -24,8 +24,14 @@ import subprocess import tempfile from textwrap import dedent import time -import apt_pkg +import json +import tarfile +import io +import copy +import itertools +from urllib import urlencode, urlopen +import apt_pkg import kombu from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION) @@ -42,6 +48,28 @@ ADT_EXCUSES_LABELS = { } +def srchash(src): + '''archive hash prefix for source package''' + + if src.startswith('lib'): + return src[:4] + else: + return src[0] + + +def merge_triggers(trigs1, trigs2): + '''Merge two (pkg, ver) trigger iterables + + Return [(pkg, ver), ...] list with only the highest version for each + package. + ''' + pkgvers = {} + for pkg, ver in itertools.chain(trigs1, trigs2): + if apt_pkg.version_compare(ver, pkgvers.setdefault(pkg, '0')) >= 0: + pkgvers[pkg] = ver + return list(pkgvers.items()) + + class AutoPackageTest(object): """autopkgtest integration @@ -71,6 +99,28 @@ class AutoPackageTest(object): os.mkdir(self.test_state_dir) self.read_pending_tests() + # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers)] + # - "passed" is a bool + # - It's tempting to just use a global "latest" time stamp, but due to + # swift's "eventual consistency" we might miss results with older time + # stamps from other packages that we don't see in the current run, but + # will in the next one. This doesn't hurt for older results of the same + # package. + # - triggers is a list of (source, version) pairs which unstable + # packages triggered this test run. We need to track this to avoid + # unnecessarily re-running tests. + self.test_results = {} + self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') + + # read the cached results that we collected so far + if os.path.exists(self.results_cache_file): + with open(self.results_cache_file) as f: + self.test_results = json.load(f) + self.log_verbose('Read previous results from %s' % self.results_cache_file) + else: + self.log_verbose('%s does not exist, re-downloading all results ' + 'from swift' % self.results_cache_file) + def log_verbose(self, msg): if self.britney.options.verbose: print('I: [%s] - %s' % (time.asctime(), msg)) @@ -175,16 +225,124 @@ class AutoPackageTest(object): '''Add one test request to the local self.requested_tests queue This will only be done if that test wasn't already requested in a - previous run, i. e. it is already in self.pending_tests. + previous run (i. e. not already in self.pending_tests) or there already + is a result for it. ''' + try: + for (tsrc, tver) in self.test_results[src][arch][1][ver][1]: + if tsrc == trigsrc and apt_pkg.version_compare(tver, trigver) >= 0: + self.log_verbose('There already is a result for %s/%s/%s triggered by %s/%s' % + (src, ver, arch, tsrc, tver)) + return + except KeyError: + pass + if (trigsrc, trigver) in self.pending_tests.get(src, {}).get( - ver, {}).get(arch, set()): + ver, {}).get(arch, set()): self.log_verbose('test %s/%s/%s for %s/%s is already pending, not queueing' % (src, ver, arch, trigsrc, trigver)) return self.requested_tests.setdefault(src, {}).setdefault( ver, {}).setdefault(arch, set()).add((trigsrc, trigver)) + def fetch_swift_results(self, swift_url, src, arch): + '''Download new results for source package/arch from swift''' + + # prepare query: get all runs with a timestamp later than latest_stamp + # for this package/arch; '@' is at the end of each run timestamp, to + # mark the end of a test run directory path + # example: wily/amd64/libp/libpng/20150630_054517@/result.tar + query = {'delimiter': '@', + 'prefix': '%s/%s/%s/%s/' % (self.series, arch, srchash(src), src)} + try: + # don't include the last run again, so make the marker + # "infinitesimally later" by appending 'zz' + query['marker'] = self.test_results[src][arch][0] + 'zz' + except KeyError: + # no stamp yet, download all results + pass + + # request new results from swift + url = os.path.join(swift_url, 'autopkgtest-' + self.series) + url += '?' + urlencode(query) + try: + f = urlopen(url) + if f.getcode() == 200: + result_paths = f.read().strip().splitlines() + else: + self.log_error('Failure to fetch swift results from %s: %u' % + (url, f.getcode())) + f.close() + return + f.close() + except IOError as e: + self.log_error('Failure to fetch swift results from %s: %s' % (url, str(e))) + return + + for p in result_paths: + self.fetch_one_result(os.path.join( + swift_url, 'autopkgtest-' + self.series, p, 'result.tar'), src, arch) + + def fetch_one_result(self, url, src, arch): + '''Download one result URL for source/arch + + Remove matching pending_tests entries. + ''' + try: + f = urlopen(url) + if f.getcode() == 200: + tar_bytes = io.BytesIO(f.read()) + f.close() + else: + self.log_error('Failure to fetch %s: %u' % (url, f.getcode())) + return + except IOError as e: + self.log_error('Failure to fetch %s: %s' % (url, str(e))) + return + + try: + with tarfile.open(None, 'r', tar_bytes) as tar: + exitcode = int(tar.extractfile('exitcode').read().strip()) + srcver = tar.extractfile('testpkg-version').read().decode().strip() + (ressrc, ver) = srcver.split() + except (KeyError, ValueError, tarfile.TarError) as e: + self.log_error('%s is damaged: %s' % (url, str(e))) + return + + if src != ressrc: + self.log_error('%s is a result for package %s, but expected package %s' % + (url, ressrc, src)) + return + + stamp = os.path.basename(os.path.dirname(url)) + # allow some skipped tests, but nothing else + passed = exitcode in [0, 2] + + self.log_verbose('Fetched test result for %s/%s on %s: %s' % ( + src, ver, arch, passed and 'pass' or 'fail')) + + # remove matching test requests, remember triggers + satisfied_triggers = set() + for pending_ver, pending_archinfo in self.pending_tests.get(src, {}).copy().items(): + # don't consider newer requested versions + if apt_pkg.version_compare(pending_ver, ver) <= 0: + try: + t = pending_archinfo[arch] + self.log_verbose('-> matches pending request for triggers %s' % str(t)) + satisfied_triggers.update(t) + del self.pending_tests[src][pending_ver][arch] + except KeyError: + self.log_error('-> does not match any pending request!') + pass + + # add this result + src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}]) + src_arch_results[1][ver] = (passed, merge_triggers( + src_arch_results[1].get(ver, (None, []))[1], satisfied_triggers)) + # update latest_stamp + if stamp > src_arch_results[0]: + src_arch_results[0] = stamp + # # obsolete adt-britney helpers # @@ -397,6 +555,38 @@ class AutoPackageTest(object): self._adt_britney("submit", request_path) def collect(self): + # fetch results from swift + try: + swift_url = self.britney.options.adt_swift_url + except AttributeError: + self.log_error('ADT_SWIFT_URL not set, cannot collect results') + swift_url = None + try: + self.britney.options.adt_amqp + except AttributeError: + self.log_error('ADT_AMQP not set, not collecting results from swift') + swift_url = None + + if swift_url: + # update results from swift for all packages that we are waiting + # for, and remove pending tests that we have results for on all + # arches + for pkg, verinfo in copy.deepcopy(self.pending_tests.items()): + for archinfo in verinfo.values(): + for arch in archinfo: + self.fetch_swift_results(swift_url, pkg, arch) + + # update the results cache + with open(self.results_cache_file + '.new', 'w') as f: + json.dump(self.test_results, f, indent=2) + os.rename(self.results_cache_file + '.new', self.results_cache_file) + self.log_verbose('Updated results cache') + + # new results remove pending requests, update the on-disk cache + self.update_pending_tests() + + # deprecated results for old Jenkins/lp:auto-package-testing, will go + # away self._ensure_rc_file() result_path = self._result_path self._adt_britney("collect", "-O", result_path) @@ -413,6 +603,13 @@ class AutoPackageTest(object): (src, ver, trigsrc, trigver, status)) def results(self, trigsrc, trigver): + '''Return test results for triggering package + + Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests + that got triggered by trigsrc/trigver. + ''' + # deprecated results for old Jenkins/lp:auto-package-testing, will go + # away for status, src, ver in self.pkgcauses[trigsrc][trigver]: # Check for regression if status == 'FAIL': diff --git a/britney.conf b/britney.conf index 0e13542..28e6be9 100644 --- a/britney.conf +++ b/britney.conf @@ -67,6 +67,8 @@ ADT_DEBUG = no ADT_ARCHES = amd64 i386 # comment this to disable autopkgtest requests ADT_AMQP = ampq://user:pwd@amqp.example.com +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.mycloud.example.com/v1/AUTH_autopkgtest BOOTTEST_ENABLE = yes BOOTTEST_DEBUG = yes diff --git a/britney.py b/britney.py index c35cb16..48e51b7 100755 --- a/britney.py +++ b/britney.py @@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes, from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) -from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS +from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash from boottest import BootTest @@ -1853,6 +1853,7 @@ class Britney(object): jenkins_private = ( "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" % self.options.series.title()) + cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" for e in autopkgtest_excuses: adtpass = True for status, adtsrc, adtver in autopkgtest.results( @@ -1883,6 +1884,33 @@ class Britney(object): "%s" % (adtsrc, adtver, forces[0].user)) else: adtpass = False + + # temporary: also show results from cloud based tests, + # until that becomes the primary mechanism + for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]): + msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver) + archmsg = [] + for arch in self.options.adt_arches.split(): + url = cloud_url % {'h': srchash(testsrc), 's': testsrc, + 'r': self.options.series, 'a': arch} + try: + r = autopkgtest.test_results[testsrc][arch][1][testver][0] + status = r and 'PASS' or 'REGRESSION' + except KeyError: + try: + autopkgtest.pending_tests[testsrc][testver][arch] + status = 'RUNNING' + except KeyError: + # neither done nor pending -> exclusion, or disabled + continue + + archmsg.append('%s: %s' % + (url, arch, ADT_EXCUSES_LABELS[status])) + + if archmsg: + e.addhtml(msg + ', '.join(archmsg)) + # end of temporary code + if not adtpass and e.is_valid: hints = self.hints.search('force-skiptest', package=e.name) hints.extend(self.hints.search('force', package=e.name)) diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index 5aa483d..8db5a1f 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -67,4 +67,5 @@ ADT_DEBUG = no ADT_ARCHES = amd64 i386 # comment this to disable autopkgtest requests ADT_AMQP = ampq://user:pwd@amqp.example.com - +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.mycloud.example.com/v1/AUTH_autopkgtest diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index a46c4a6..42f477f 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -13,12 +13,13 @@ import sys import subprocess import fileinput import unittest +import json PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, PROJECT_DIR) from autopkgtest import ADT_EXCUSES_LABELS -from tests import TestBase +from tests import TestBase, mock_swift NOT_CONSIDERED = False VALID_CANDIDATE = True @@ -34,12 +35,14 @@ class TestAutoPkgTest(TestBase): super(TestAutoPkgTest, self).setUp() self.fake_amqp = os.path.join(self.data.path, 'amqp') - # Disable boottests and set fake AMQP server + # Disable boottests and set fake AMQP and Swift server for line in fileinput.input(self.britney_conf, inplace=True): if line.startswith('BOOTTEST_ENABLE'): print('BOOTTEST_ENABLE = no') elif line.startswith('ADT_AMQP'): print('ADT_AMQP = file://%s' % self.fake_amqp) + elif line.startswith('ADT_SWIFT_URL'): + print('ADT_SWIFT_URL = http://localhost:18085') else: sys.stdout.write(line) @@ -70,11 +73,22 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) testsuite='specialtest') self.data.add('justdata', False, {'Architecture': 'all'}) + # create mock Swift server (but don't start it yet, as tests first need + # to poke in results) + self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085) + self.swift.set_results({}) + + def tearDown(self): + del self.swift + def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None): for (pkg, fields, testsuite) in unstable_add: self.data.add(pkg, True, fields, True, testsuite) + self.swift.start() (excuses, out) = self.run_britney() + self.swift.stop() + #print('-------\nexcuses: %s\n-----' % excuses) #print('-------\nout: %s\n-----' % out) #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) @@ -105,6 +119,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) except IOError: self.pending_requests = None + return out + def test_multi_rdepends_with_tests_all_running(self): '''Multiple reverse dependencies with tests (all running)''' @@ -113,7 +129,10 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) # FIXME: while we only submit requests through AMQP, but don't consider # their results, we don't expect this to hold back stuff. VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<']) + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) # we expect the package's and its reverse dependencies' tests to get # triggered @@ -121,8 +140,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) self.amqp_requests, set(['debci-series-i386:green', 'debci-series-amd64:green', 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', - 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen', - ])) + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) os.unlink(self.fake_amqp) # ... and that they get recorded as pending @@ -141,6 +159,108 @@ lightgreen 1 i386 green 2 # but the set of pending tests doesn't change self.assertEqual(self.pending_requests, expected_pending) + def test_multi_rdepends_with_tests_all_pass(self): + '''Multiple reverse dependencies with tests (all pass)''' + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1'), + # version in testing fails + 'series/i386/g/green/20150101_020000@': (4, 'green 1'), + 'series/amd64/g/green/20150101_020000@': (4, 'green 1'), + # version in unstable succeeds + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2'), + }}) + + out = self.do_test( + [], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + # all tests ran, there should be no more pending ones + self.assertEqual(self.pending_requests, '') + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + # caches the results and triggers + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: + res = json.load(f) + self.assertEqual(res['green']['i386'], + ['20150101_100200@', {'1': [False, []], + '2': [True, [['green', '2']]]}]) + self.assertEqual(res['lightgreen']['amd64'], + ['20150101_100101@', {'1': [True, [['green', '2']]]}]) + + # third run should not trigger any new tests, should all be in the + # cache + os.unlink(self.fake_amqp) + self.swift.set_results({}) + out = self.do_test( + [], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, '') + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_mixed(self): + '''Multiple reverse dependencies with tests (mixed results)''' + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + self.do_test( + [], + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass']) + + # there should be some pending ones + self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) + self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) + def test_package_pair_running(self): '''Two packages in unstable that need to go in together (running)''' @@ -159,8 +279,7 @@ lightgreen 1 i386 green 2 self.amqp_requests, set(['debci-series-i386:green', 'debci-series-amd64:green', 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', - 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen', - ])) + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) os.unlink(self.fake_amqp) # ... and that they get recorded as pending @@ -180,7 +299,7 @@ lightgreen 2 i386 lightgreen 2 # Disable AMQP server config for line in fileinput.input(self.britney_conf, inplace=True): - if not line.startswith('ADT_AMQP'): + if not line.startswith('ADT_AMQP') and not line.startswith('ADT_SWIFT_URL'): sys.stdout.write(line) self.do_test(