diff --git a/autopkgtest.py b/autopkgtest.py index c8e9728..b817aa9 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -26,6 +26,10 @@ from textwrap import dedent import time import apt_pkg +import kombu + +from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE) + adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") @@ -53,7 +57,139 @@ class AutoPackageTest(object): self.series = series self.debug = debug self.read() - self.rc_path = None + self.rc_path = None # for adt-britney, obsolete + self.test_state_dir = os.path.join(britney.options.unstable, + 'autopkgtest') + # map of requested tests from request() + # src -> ver -> {(triggering-src1, ver1), ...} + self.requested_tests = {} + # same map for tests requested in previous runs + self.pending_tests = None + self.pending_tests_file = os.path.join(self.test_state_dir, 'pending.txt') + + if not os.path.isdir(self.test_state_dir): + os.mkdir(self.test_state_dir) + self.read_pending_tests() + + def log_verbose(self, msg): + if self.britney.options.verbose: + print('I: [%s] - %s' % (time.asctime(), msg)) + + def log_error(self, msg): + print('E: [%s] - %s' % (time.asctime(), msg)) + + def tests_for_source(self, src, ver): + '''Iterate over all tests that should be run for given source''' + + sources_info = self.britney.sources['unstable'] + # FIXME: For now assume that amd64 has all binaries that we are + # interested in for reverse dependency checking + binaries_info = self.britney.binaries['unstable']['amd64'][0] + + srcinfo = sources_info[src] + # we want to test the package itself, if it still has a test in + # unstable + if srcinfo[AUTOPKGTEST]: + yield (src, ver) + + # plus all direct reverse dependencies of its binaries which have + # an autopkgtest + for binary in srcinfo[BINARIES]: + binary = binary.split('/')[0] # chop off arch + try: + rdeps = binaries_info[binary][RDEPENDS] + except KeyError: + self.log_verbose('Ignoring nonexistant binary %s (FTBFS/NBS)?' % binary) + continue + for rdep in rdeps: + rdep_src = binaries_info[rdep][SOURCE] + if sources_info[rdep_src][AUTOPKGTEST]: + # we don't care about the version of rdep + yield (rdep_src, None) + + # + # AMQP/cloud interface helpers + # + + def read_pending_tests(self): + '''Read pending test requests from previous britney runs + + Read UNSTABLE/autopkgtest/requested.txt with the format: + srcpkg srcver triggering-srcpkg triggering-srcver + + Initialize self.pending_tests with that data. + ''' + assert self.pending_tests is None, 'already initialized' + self.pending_tests = {} + if not os.path.exists(self.pending_tests_file): + self.log_verbose('No %s, starting with no pending tests' % + self.pending_tests_file) + return + with open(self.pending_tests_file) as f: + for l in f: + l = l.strip() + if not l: + continue + try: + (src, ver, trigsrc, trigver) = l.split() + except ValueError: + self.log_error('ignoring malformed line in %s: %s' % + (self.pending_tests_file, l)) + continue + if ver == '-': + ver = None + if trigver == '-': + trigver = None + self.pending_tests.setdefault(src, {}).setdefault( + ver, set()).add((trigsrc, trigver)) + self.log_verbose('Read pending requested tests from %s: %s' % + (self.pending_tests_file, self.pending_tests)) + + def update_pending_tests(self): + '''Update pending tests after submitting requested tests + + Update UNSTABLE/autopkgtest/requested.txt, see read_pending_tests() for + the format. + ''' + # merge requested_tests into pending_tests + for src, verinfo in self.requested_tests.items(): + for ver, triggers in verinfo.items(): + self.pending_tests.setdefault(src, {}).setdefault( + ver, set()).update(triggers) + self.requested_tests = {} + + # write it + with open(self.pending_tests_file + '.new', 'w') as f: + for src in sorted(self.pending_tests): + for ver in sorted(self.pending_tests[src]): + for (trigsrc, trigver) in sorted(self.pending_tests[src][ver]): + if ver is None: + ver = '-' + if trigver is None: + trigver = '-' + f.write('%s %s %s %s\n' % (src, ver, trigsrc, trigver)) + os.rename(self.pending_tests_file + '.new', self.pending_tests_file) + self.log_verbose('Updated pending requested tests in %s' % + self.pending_tests_file) + + def add_test_request(self, src, ver, trigsrc, trigver): + '''Add one test request to the local self.requested_tests queue + + This will only be done if that test wasn't already requested in a + previous run, i. e. it is already in self.pending_tests. + + versions can be None if you don't care about the particular version. + ''' + if (trigsrc, trigver) in self.pending_tests.get(src, {}).get(ver, set()): + self.log_verbose('test %s/%s for %s/%s is already pending, not queueing' % + (src, ver, trigsrc, trigver)) + return + self.requested_tests.setdefault(src, {}).setdefault( + ver, set()).add((trigsrc, trigver)) + + # + # obsolete adt-britney helpers + # def _ensure_rc_file(self): if self.rc_path: @@ -137,10 +273,30 @@ class AutoPackageTest(object): command.extend(args) subprocess.check_call(command) + # + # Public API + # + def request(self, packages, excludes=None): if excludes is None: excludes = [] + self.log_verbose('Requested autopkgtests for %s, exclusions: %s' % + (['%s/%s' % i for i in packages], str(excludes))) + for src, ver in packages: + for (testsrc, testver) in self.tests_for_source(src, ver): + if testsrc not in excludes: + self.add_test_request(testsrc, testver, src, ver) + + if self.britney.options.verbose: + for src, verinfo in self.requested_tests.items(): + for ver, triggers in verinfo.items(): + self.log_verbose('Requesting %s/%s autopkgtest to verify %s' % + (src, ver, ', '.join(['%s/%s' % i for i in triggers]))) + + # deprecated requests for old Jenkins/lp:auto-package-testing, will go + # away + self._ensure_rc_file() request_path = self._request_path if os.path.exists(request_path): @@ -164,9 +320,8 @@ class AutoPackageTest(object): request_file.write(line) else: if self.britney.options.verbose: - print("I: [%s] - Requested autopkgtest for %s but " - "run_autopkgtest set to False" % - (time.asctime(), src)) + self.log_verbose("Requested autopkgtest for %s but " + "run_autopkgtest set to False" % src) for linebits in self._parse(request_path): # Make sure that there's an entry in pkgcauses for each new @@ -176,8 +331,8 @@ class AutoPackageTest(object): src = linebits.pop(0) ver = linebits.pop(0) if self.britney.options.verbose: - print("I: [%s] - Requested autopkgtest for %s_%s (%s)" % - (time.asctime(), src, ver, " ".join(linebits))) + self.log_verbose("Requested autopkgtest for %s_%s (%s)" % + (src, ver, " ".join(linebits))) try: status = linebits.pop(0).upper() while True: @@ -194,6 +349,40 @@ class AutoPackageTest(object): pass def submit(self): + # send AMQP requests for new test requests + # TODO: Once we support version constraints in AMQP requests, add them + queues = ['debci-%s-%s' % (self.series, arch) + for arch in self.britney.options.adt_arches.split()] + + try: + amqp_url = self.britney.options.adt_amqp + except AttributeError: + self.log_error('ADT_AMQP not set, cannot submit requests') + return + + if amqp_url.startswith('amqp://'): + with kombu.Connection(amqp_url) as conn: + for q in queues: + # don't use SimpleQueue here as it always declares queues; + # ACLs might not allow that + with kombu.Producer(conn, routing_key=q, auto_declare=False) as p: + for pkg in self.requested_tests: + p.publish(pkg) + elif amqp_url.startswith('file://'): + # in testing mode, adt_amqp will be a file:// URL + with open(amqp_url[7:], 'a') as f: + for pkg in self.requested_tests: + for q in queues: + f.write('%s:%s\n' % (q, pkg)) + else: + self.log_error('Unknown ADT_AMQP schema in %s' % + self.britney.options.adt_amqp) + + # mark them as pending now + self.update_pending_tests() + + # deprecated requests for old Jenkins/lp:auto-package-testing, will go + # away self._ensure_rc_file() request_path = self._request_path if os.path.exists(request_path): @@ -211,10 +400,9 @@ class AutoPackageTest(object): for trigsrc in sorted(self.pkglist[src][ver]['causes']): for trigver, status \ in self.pkglist[src][ver]['causes'][trigsrc]: - print("I: [%s] - Collected autopkgtest status " - "for %s_%s/%s_%s: " "%s" % ( - time.asctime(), src, ver, trigsrc, - trigver, status)) + self.log_verbose("Collected autopkgtest status " + "for %s_%s/%s_%s: " "%s" % + (src, ver, trigsrc, trigver, status)) def results(self, trigsrc, trigver): for status, src, ver in self.pkgcauses[trigsrc][trigver]: diff --git a/britney.conf b/britney.conf index 8d9b285..0e13542 100644 --- a/britney.conf +++ b/britney.conf @@ -65,6 +65,8 @@ REMOVE_OBSOLETE = no ADT_ENABLE = yes ADT_DEBUG = no ADT_ARCHES = amd64 i386 +# comment this to disable autopkgtest requests +ADT_AMQP = ampq://user:pwd@amqp.example.com BOOTTEST_ENABLE = yes BOOTTEST_DEBUG = yes diff --git a/britney.py b/britney.py index becd35c..c35cb16 100755 --- a/britney.py +++ b/britney.py @@ -564,6 +564,7 @@ class Britney(object): [], get_field('Maintainer'), False, + get_field('Testsuite', '').startswith('autopkgtest'), ] return sources diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index eb76959..5aa483d 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -65,3 +65,6 @@ REMOVE_OBSOLETE = no ADT_ENABLE = yes ADT_DEBUG = no ADT_ARCHES = amd64 i386 +# comment this to disable autopkgtest requests +ADT_AMQP = ampq://user:pwd@amqp.example.com + diff --git a/consts.py b/consts.py index c72bb8b..493a538 100644 --- a/consts.py +++ b/consts.py @@ -24,6 +24,7 @@ SECTION = 1 BINARIES = 2 MAINTAINER = 3 FAKESRC = 4 +AUTOPKGTEST = 5 # binary package SOURCE = 2 diff --git a/tests/__init__.py b/tests/__init__.py index 81c0316..f254765 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -63,7 +63,7 @@ class TestData: def __del__(self): shutil.rmtree(self.path) - def add(self, name, unstable, fields={}, add_src=True): + def add(self, name, unstable, fields={}, add_src=True, testsuite=None): '''Add a binary package to the index file. You need to specify at least the package name and in which list to put @@ -73,7 +73,8 @@ class TestData: fields. Unless add_src is set to False, this will also automatically create a - source record, based on fields['Source'] and name. + source record, based on fields['Source'] and name. In that case, the + "Testsuite:" field is set to the testsuite argument. ''' assert (name not in self.added_binaries[unstable]) self.added_binaries[unstable].add(name) @@ -93,8 +94,11 @@ class TestData: if add_src: src = fields.get('Source', name) if src not in self.added_sources[unstable]: - self.add_src(src, unstable, {'Version': fields['Version'], - 'Section': fields['Section']}) + srcfields = {'Version': fields['Version'], + 'Section': fields['Section']} + if testsuite: + srcfields['Testsuite'] = testsuite + self.add_src(src, unstable, srcfields) def add_src(self, name, unstable, fields={}): '''Add a source package to the index file. @@ -102,7 +106,8 @@ class TestData: You need to specify at least the package name and in which list to put it (unstable==True for unstable/proposed, or False for testing/release). fields specifies all additional entries, which can be - Version (default: 1), Section (default: devel), and Extra-Source-Only. + Version (default: 1), Section (default: devel), Testsuite (default: + none), and Extra-Source-Only. ''' assert (name not in self.added_sources[unstable]) self.added_sources[unstable].add(name) @@ -128,18 +133,14 @@ class TestBase(unittest.TestCase): super(TestBase, self).setUp() self.data = TestData() self.britney = os.path.join(PROJECT_DIR, 'britney.py') - self.britney_conf = os.path.join(PROJECT_DIR, 'britney.conf') + # create temporary config so that tests can hack it + self.britney_conf = os.path.join(self.data.path, 'britney.conf') + shutil.copy(os.path.join(PROJECT_DIR, 'britney.conf'), self.britney_conf) assert os.path.exists(self.britney) - assert os.path.exists(self.britney_conf) def tearDown(self): del self.data - def restore_config(self, content): - """Helper for restoring configuration contents on cleanup.""" - with open(self.britney_conf, 'w') as fp: - fp.write(content) - def run_britney(self, args=[]): '''Run britney. diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 6112098..dd371e0 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -11,6 +11,7 @@ import operator import os import sys import subprocess +import fileinput import unittest PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -27,9 +28,139 @@ apt_pkg.init() class TestAutoPkgTest(TestBase): + '''AMQP/cloud interface''' def setUp(self): super(TestAutoPkgTest, self).setUp() + self.fake_amqp = os.path.join(self.data.path, 'amqp') + + # Disable boottests and set fake AMQP server + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('BOOTTEST_ENABLE'): + print('BOOTTEST_ENABLE = no') + elif line.startswith('ADT_AMQP'): + print('ADT_AMQP = file://%s' % self.fake_amqp) + else: + sys.stdout.write(line) + + # fake adt-britney script; necessary until we drop that code + self.adt_britney = os.path.join( + self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney') + os.makedirs(os.path.dirname(self.adt_britney)) + with open(self.adt_britney, 'w') as f: + f.write('''#!/bin/sh -e +touch $HOME/proposed-migration/autopkgtest/work/adt.request.series +echo "$@" >> /%s/adt-britney.log ''' % self.data.path) + os.chmod(self.adt_britney, 0o755) + + # add a bunch of packages to testing to avoid repetition + self.data.add('libc6', False) + self.data.add('libgreen1', False, {'Source': 'green', + 'Depends': 'libc6 (>= 0.9)'}) + self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Conflicts': 'blue'}, + testsuite='autopkgtest') + self.data.add('lightgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest') + # autodep8 or similar test + self.data.add('darkgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest-pkg-foo') + self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', + 'Conflicts': 'green'}, + testsuite='specialtest') + self.data.add('justdata', False, {'Architecture': 'all'}) + + def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None): + for (pkg, fields, testsuite) in unstable_add: + self.data.add(pkg, True, fields, True, testsuite) + + (excuses, out) = self.run_britney() + #print('-------\nexcuses: %s\n-----' % excuses) + #print('-------\nout: %s\n-----' % out) + #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) + #subprocess.call(['bash', '-i'], cwd=self.data.path) + if considered: + self.assertIn('Valid candidate', excuses) + else: + self.assertIn('Not considered', excuses) + + if excuses_expect: + for re in excuses_expect: + self.assertRegexpMatches(excuses, re) + if excuses_no_expect: + for re in excuses_no_expect: + self.assertNotRegexpMatches(excuses, re) + + self.amqp_requests = set() + try: + with open(self.fake_amqp) as f: + for line in f: + self.amqp_requests.add(line.strip()) + except IOError: + pass + + try: + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/pending.txt')) as f: + self.pending_requests = f.read() + except IOError: + self.pending_requests = None + + def test_multi_rdepends_with_tests(self): + '''Multiple reverse dependencies with tests''' + + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<']) + + # we expect the package's and its reverse dependencies' tests to get + # triggered + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green', 'debci-series-amd64:green', + 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen', + ])) + os.unlink(self.fake_amqp) + + expected_pending = '''darkgreen - green 2 +green 2 green 2 +lightgreen - green 2 +''' + + # ... and that they get recorded as pending + self.assertEqual(self.pending_requests, expected_pending) + + # if we run britney again this should *not* trigger any new tests + self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<']) + self.assertEqual(self.amqp_requests, set()) + # but the set of pending tests doesn't change + self.assertEqual(self.pending_requests, expected_pending) + + def test_no_amqp_config(self): + '''Run without autopkgtest requests''' + + # Disable AMQP server config + for line in fileinput.input(self.britney_conf, inplace=True): + if not line.startswith('ADT_AMQP'): + sys.stdout.write(line) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<'], ['autopkgtest']) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, None) + + +class TestAdtBritney(TestBase): + '''Legacy adt-britney/lp:auto-package-testing interface''' + + def setUp(self): + super(TestAdtBritney, self).setUp() # Mofify configuration according to the test context. with open(self.britney_conf, 'r') as fp: @@ -39,7 +170,6 @@ class TestAutoPkgTest(TestBase): 'BOOTTEST_ENABLE = yes', 'BOOTTEST_ENABLE = no') with open(self.britney_conf, 'w') as fp: fp.write(new_config) - self.addCleanup(self.restore_config, original_config) # fake adt-britney script self.adt_britney = os.path.join( diff --git a/tests/test_boottest.py b/tests/test_boottest.py index 9e4014a..770f501 100644 --- a/tests/test_boottest.py +++ b/tests/test_boottest.py @@ -143,7 +143,6 @@ class TestBoottestEnd2End(TestBase): 'BOOTTEST_FETCH = yes', 'BOOTTEST_FETCH = no') with open(self.britney_conf, 'w') as fp: fp.write(new_config) - self.addCleanup(self.restore_config, original_config) self.data.add('libc6', False, {'Architecture': 'armhf'}),