From add46d2904ff4b8061ef0df9dde51296d8c66e22 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 1 Jul 2015 15:49:06 +0200 Subject: [PATCH 01/10] Store autopkgtest flag in sources Extend read_sources to store a new AUTOPKGTEST boolean flag, which is true if the Testsuite: field exists and starts with "autopkgtest" (this covers autodep8 cases like autopkgtest-pkg-perl). Extend TestData.add() to take a new testsuite argument which specifies the source's Testsuite: field. --- britney.py | 1 + consts.py | 1 + tests/__init__.py | 15 ++++++++++----- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/britney.py b/britney.py index becd35c..c35cb16 100755 --- a/britney.py +++ b/britney.py @@ -564,6 +564,7 @@ class Britney(object): [], get_field('Maintainer'), False, + get_field('Testsuite', '').startswith('autopkgtest'), ] return sources diff --git a/consts.py b/consts.py index c72bb8b..493a538 100644 --- a/consts.py +++ b/consts.py @@ -24,6 +24,7 @@ SECTION = 1 BINARIES = 2 MAINTAINER = 3 FAKESRC = 4 +AUTOPKGTEST = 5 # binary package SOURCE = 2 diff --git a/tests/__init__.py b/tests/__init__.py index 81c0316..07eb564 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -63,7 +63,7 @@ class TestData: def __del__(self): shutil.rmtree(self.path) - def add(self, name, unstable, fields={}, add_src=True): + def add(self, name, unstable, fields={}, add_src=True, testsuite=None): '''Add a binary package to the index file. You need to specify at least the package name and in which list to put @@ -73,7 +73,8 @@ class TestData: fields. Unless add_src is set to False, this will also automatically create a - source record, based on fields['Source'] and name. + source record, based on fields['Source'] and name. In that case, the + "Testsuite:" field is set to the testsuite argument. ''' assert (name not in self.added_binaries[unstable]) self.added_binaries[unstable].add(name) @@ -93,8 +94,11 @@ class TestData: if add_src: src = fields.get('Source', name) if src not in self.added_sources[unstable]: - self.add_src(src, unstable, {'Version': fields['Version'], - 'Section': fields['Section']}) + srcfields = {'Version': fields['Version'], + 'Section': fields['Section']} + if testsuite: + srcfields['Testsuite'] = testsuite + self.add_src(src, unstable, srcfields) def add_src(self, name, unstable, fields={}): '''Add a source package to the index file. @@ -102,7 +106,8 @@ class TestData: You need to specify at least the package name and in which list to put it (unstable==True for unstable/proposed, or False for testing/release). fields specifies all additional entries, which can be - Version (default: 1), Section (default: devel), and Extra-Source-Only. + Version (default: 1), Section (default: devel), Testsuite (default: + none), and Extra-Source-Only. ''' assert (name not in self.added_sources[unstable]) self.added_sources[unstable].add(name) From 65a1d73164d9cb402cfac1bd270b67472941535f Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 2 Jul 2015 16:24:53 +0200 Subject: [PATCH 02/10] tests: Don't touch original britney.conf Don't change britney.conf from the source tree and restore it back after the tests; this is prone to leave a broken config behind if a test gets interrupted. Instead, copy the file to our temp dir, and let tests hack on it there. (Introduced in lp:~canonical-ci-engineering/britney/boottesting-support) --- tests/__init__.py | 10 +++------- tests/test_autopkgtest.py | 1 - tests/test_boottest.py | 1 - 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 07eb564..f254765 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -133,18 +133,14 @@ class TestBase(unittest.TestCase): super(TestBase, self).setUp() self.data = TestData() self.britney = os.path.join(PROJECT_DIR, 'britney.py') - self.britney_conf = os.path.join(PROJECT_DIR, 'britney.conf') + # create temporary config so that tests can hack it + self.britney_conf = os.path.join(self.data.path, 'britney.conf') + shutil.copy(os.path.join(PROJECT_DIR, 'britney.conf'), self.britney_conf) assert os.path.exists(self.britney) - assert os.path.exists(self.britney_conf) def tearDown(self): del self.data - def restore_config(self, content): - """Helper for restoring configuration contents on cleanup.""" - with open(self.britney_conf, 'w') as fp: - fp.write(content) - def run_britney(self, args=[]): '''Run britney. diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 6112098..a9106ee 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -39,7 +39,6 @@ class TestAutoPkgTest(TestBase): 'BOOTTEST_ENABLE = yes', 'BOOTTEST_ENABLE = no') with open(self.britney_conf, 'w') as fp: fp.write(new_config) - self.addCleanup(self.restore_config, original_config) # fake adt-britney script self.adt_britney = os.path.join( diff --git a/tests/test_boottest.py b/tests/test_boottest.py index 9e4014a..770f501 100644 --- a/tests/test_boottest.py +++ b/tests/test_boottest.py @@ -143,7 +143,6 @@ class TestBoottestEnd2End(TestBase): 'BOOTTEST_FETCH = yes', 'BOOTTEST_FETCH = no') with open(self.britney_conf, 'w') as fp: fp.write(new_config) - self.addCleanup(self.restore_config, original_config) self.data.add('libc6', False, {'Architecture': 'armhf'}), From a3a523b4ed01633ba1c5004456113661d9c9e584 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 2 Jul 2015 17:36:23 +0200 Subject: [PATCH 03/10] autopkgtest.py: Factorize logging into helper methods --- autopkgtest.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index c8e9728..b6d7173 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -55,6 +55,13 @@ class AutoPackageTest(object): self.read() self.rc_path = None + def log_verbose(self, msg): + if self.britney.options.verbose: + print('I: [%s] - %s' % (time.asctime(), msg)) + + def log_error(self, msg): + print('E: [%s] - %s' % (time.asctime(), msg)) + def _ensure_rc_file(self): if self.rc_path: return @@ -164,9 +171,8 @@ class AutoPackageTest(object): request_file.write(line) else: if self.britney.options.verbose: - print("I: [%s] - Requested autopkgtest for %s but " - "run_autopkgtest set to False" % - (time.asctime(), src)) + self.log_verbose("Requested autopkgtest for %s but " + "run_autopkgtest set to False" % src) for linebits in self._parse(request_path): # Make sure that there's an entry in pkgcauses for each new @@ -176,8 +182,8 @@ class AutoPackageTest(object): src = linebits.pop(0) ver = linebits.pop(0) if self.britney.options.verbose: - print("I: [%s] - Requested autopkgtest for %s_%s (%s)" % - (time.asctime(), src, ver, " ".join(linebits))) + self.log_verbose("Requested autopkgtest for %s_%s (%s)" % + (src, ver, " ".join(linebits))) try: status = linebits.pop(0).upper() while True: @@ -211,10 +217,9 @@ class AutoPackageTest(object): for trigsrc in sorted(self.pkglist[src][ver]['causes']): for trigver, status \ in self.pkglist[src][ver]['causes'][trigsrc]: - print("I: [%s] - Collected autopkgtest status " - "for %s_%s/%s_%s: " "%s" % ( - time.asctime(), src, ver, trigsrc, - trigver, status)) + self.log_verbose("Collected autopkgtest status " + "for %s_%s/%s_%s: " "%s" % + (src, ver, trigsrc, trigver, status)) def results(self, trigsrc, trigver): for status, src, ver in self.pkgcauses[trigsrc][trigver]: From 335073e9011e78c93e9096b85f07edcfabd28c36 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 2 Jul 2015 17:41:49 +0200 Subject: [PATCH 04/10] Add requesting tests through AMQP Until now, autopkgtests were triggered via an external "adt-britney" command from lp:auto-package-testing. This duplicated a lot of effort (such as its own chdist and reverse dependency calculation), maintains a lot of state files, uses hardcoded absolute paths to these external tools, and is quite hard to understand and maintain. We want to get rid of all this. Add logic to AutoPackageTest.request() to use britney's existing reverse dependency maps and figure out the set of tests to run for packages in unstable. These are being tracked in "requested_tests". Add logic to AutoPackageTest.submit() to send test requests to the AMQP server specified in the new "ADT_AMQP" config key. For testing this can be a file:// URL, and if not set no test requests will be sent at all. The set of tests which were requested in previous runs are tracked in UNSTABLE/autopkgtest/pending.txt, so that we don't re-request tests in subsequent runs. There is no implementation for collect() and results() yet, these will be done in a separate commit. Add initial test cases. --- autopkgtest.py | 171 +++++++++++++++++++++++++++++++++++++- britney.conf | 2 + tests/test_autopkgtest.py | 131 +++++++++++++++++++++++++++++ 3 files changed, 303 insertions(+), 1 deletion(-) diff --git a/autopkgtest.py b/autopkgtest.py index b6d7173..1c8acff 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -26,6 +26,10 @@ from textwrap import dedent import time import apt_pkg +import kombu + +from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE) + adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") @@ -53,7 +57,19 @@ class AutoPackageTest(object): self.series = series self.debug = debug self.read() - self.rc_path = None + self.rc_path = None # for adt-britney, obsolete + self.test_state_dir = os.path.join(britney.options.unstable, + 'autopkgtest') + # map of requested tests from request() + # src -> ver -> {(triggering-src1, ver1), ...} + self.requested_tests = {} + # same map for tests requested in previous runs + self.pending_tests = None + self.pending_tests_file = os.path.join(self.test_state_dir, 'pending.txt') + + if not os.path.isdir(self.test_state_dir): + os.mkdir(self.test_state_dir) + self.read_pending_tests() def log_verbose(self, msg): if self.britney.options.verbose: @@ -62,6 +78,88 @@ class AutoPackageTest(object): def log_error(self, msg): print('E: [%s] - %s' % (time.asctime(), msg)) + # + # AMQP/cloud interface helpers + # + + def read_pending_tests(self): + '''Read pending test requests from previous britney runs + + Read test_state_dir/requested.txt with the format: + srcpkg srcver triggering-srcpkg triggering-srcver + + Initialize self.pending_tests with that data. + ''' + assert self.pending_tests is None, 'already initialized' + self.pending_tests = {} + if not os.path.exists(self.pending_tests_file): + self.log_verbose('No %s, starting with no pending tests' % + self.pending_tests_file) + return + with open(self.pending_tests_file) as f: + for l in f: + l = l.strip() + if not l: + continue + try: + (src, ver, trigsrc, trigver) = l.split() + except ValueError: + self.log_error('ignoring malformed line in %s: %s' % + (self.pending_tests_file, l)) + continue + if ver == '-': + ver = None + if trigver == '-': + trigver = None + self.pending_tests.setdefault(src, {}).setdefault( + ver, set()).add((trigsrc, trigver)) + self.log_verbose('Read pending requested tests from %s: %s' % + (self.pending_tests_file, self.pending_tests)) + + def update_pending_tests(self): + '''Update pending tests after submitting requested tests + + Update test_state_dir/requested.txt, see read_pending_tests() for the + format. + ''' + # merge requested_tests into pending_tests + for src, verinfo in self.requested_tests.items(): + for ver, triggers in verinfo.items(): + self.pending_tests.setdefault(src, {}).setdefault( + ver, set()).update(triggers) + # write it + with open(self.pending_tests_file + '.new', 'w') as f: + for src in sorted(self.pending_tests): + for ver in sorted(self.pending_tests[src]): + for (trigsrc, trigver) in sorted(self.pending_tests[src][ver]): + if ver is None: + ver = '-' + if trigver is None: + trigver = '-' + f.write('%s %s %s %s\n' % (src, ver, trigsrc, trigver)) + os.rename(self.pending_tests_file + '.new', self.pending_tests_file) + self.log_verbose('Updated pending requested tests in %s' % + self.pending_tests_file) + + def add_test_request(self, src, ver, trigsrc, trigver): + '''Add one test request to the local self.requested_tests queue + + This will only be done if that test wasn't already requested in a + previous run, i. e. it is already in self.pending_tests. + + versions can be None if you don't care about the particular version. + ''' + if (trigsrc, trigver) in self.pending_tests.get(src, {}).get(ver, set()): + self.log_verbose('test %s/%s for %s/%s is already pending, not queueing' % + (src, ver, trigsrc, trigver)) + return + self.requested_tests.setdefault(src, {}).setdefault( + ver, set()).add((trigsrc, trigver)) + + # + # obsolete adt-britney helpers + # + def _ensure_rc_file(self): if self.rc_path: return @@ -144,10 +242,48 @@ class AutoPackageTest(object): command.extend(args) subprocess.check_call(command) + # + # Public API + # + def request(self, packages, excludes=None): if excludes is None: excludes = [] + self.log_verbose('Requested autopkgtests for %s, exclusions: %s' % + (['%s/%s' % i for i in packages], str(excludes))) + # add new test requests to self.requested_tests from reverse + # dependencies, unless they are already in self.pending_tests + # build a source -> {triggering-source1, ...} map from reverse + # dependencies + sources_info = self.britney.sources['unstable'] + # FIXME: For now assume that amd64 has all binaries that we are + binaries_info = self.britney.binaries['unstable']['amd64'][0] + # interested in for reverse dependency checking + for src, ver in packages: + srcinfo = sources_info[src] + # we want to test the package itself, if it still has a test in + # unstable + if srcinfo[AUTOPKGTEST] and src not in excludes: + self.add_test_request(src, ver, src, ver) + # plus all direct reverse dependencies of its binaries which have + # an autopkgtest + for binary in srcinfo[BINARIES]: + binary = binary.split('/')[0] # chop off arch + for rdep in binaries_info[binary][RDEPENDS]: + rdep_src = binaries_info[rdep][SOURCE] + if sources_info[rdep_src][AUTOPKGTEST] and rdep_src not in excludes: + # we don't care about the version of rdep + self.add_test_request(rdep_src, None, src, ver) + + for src, verinfo in self.requested_tests.items(): + for ver, triggers in verinfo.items(): + self.log_verbose('Requesting %s/%s autopkgtest to verify %s' % + (src, ver, ', '.join(['%s/%s' % i for i in triggers]))) + + # deprecated requests for old Jenkins/lp:auto-package-testing, will go + # away + self._ensure_rc_file() request_path = self._request_path if os.path.exists(request_path): @@ -200,6 +336,39 @@ class AutoPackageTest(object): pass def submit(self): + # send AMQP requests for new test requests + # TODO: Once we support version constraints in AMQP requests, add them + queues = ['debci-%s-%s' % (self.series, arch) + for arch in self.britney.options.adt_arches.split()] + + try: + amqp_url = self.britney.options.adt_amqp + except AttributeError: + self.log_error('ADT_AMQP not set, cannot submit requests') + return + + if amqp_url.startswith('amqp://'): + with kombu.Connection(amqp_url) as conn: + for q in queues: + amqp_queue = conn.SimpleQueue(q) + for pkg in self.requested_tests: + amqp_queue.put(pkg) + amqp_queue.close() + elif amqp_url.startswith('file://'): + # in testing mode, adt_amqp will be a file:// URL + with open(amqp_url[7:], 'a') as f: + for pkg in self.requested_tests: + for q in queues: + f.write('%s:%s\n' % (q, pkg)) + else: + self.log_error('Unknown ADT_AMQP schema in %s' % + self.britney.options.adt_amqp) + + # mark them as pending now + self.update_pending_tests() + + # deprecated requests for old Jenkins/lp:auto-package-testing, will go + # away self._ensure_rc_file() request_path = self._request_path if os.path.exists(request_path): diff --git a/britney.conf b/britney.conf index 8d9b285..0e13542 100644 --- a/britney.conf +++ b/britney.conf @@ -65,6 +65,8 @@ REMOVE_OBSOLETE = no ADT_ENABLE = yes ADT_DEBUG = no ADT_ARCHES = amd64 i386 +# comment this to disable autopkgtest requests +ADT_AMQP = ampq://user:pwd@amqp.example.com BOOTTEST_ENABLE = yes BOOTTEST_DEBUG = yes diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index a9106ee..dd371e0 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -11,6 +11,7 @@ import operator import os import sys import subprocess +import fileinput import unittest PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -27,9 +28,139 @@ apt_pkg.init() class TestAutoPkgTest(TestBase): + '''AMQP/cloud interface''' def setUp(self): super(TestAutoPkgTest, self).setUp() + self.fake_amqp = os.path.join(self.data.path, 'amqp') + + # Disable boottests and set fake AMQP server + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('BOOTTEST_ENABLE'): + print('BOOTTEST_ENABLE = no') + elif line.startswith('ADT_AMQP'): + print('ADT_AMQP = file://%s' % self.fake_amqp) + else: + sys.stdout.write(line) + + # fake adt-britney script; necessary until we drop that code + self.adt_britney = os.path.join( + self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney') + os.makedirs(os.path.dirname(self.adt_britney)) + with open(self.adt_britney, 'w') as f: + f.write('''#!/bin/sh -e +touch $HOME/proposed-migration/autopkgtest/work/adt.request.series +echo "$@" >> /%s/adt-britney.log ''' % self.data.path) + os.chmod(self.adt_britney, 0o755) + + # add a bunch of packages to testing to avoid repetition + self.data.add('libc6', False) + self.data.add('libgreen1', False, {'Source': 'green', + 'Depends': 'libc6 (>= 0.9)'}) + self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Conflicts': 'blue'}, + testsuite='autopkgtest') + self.data.add('lightgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest') + # autodep8 or similar test + self.data.add('darkgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest-pkg-foo') + self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', + 'Conflicts': 'green'}, + testsuite='specialtest') + self.data.add('justdata', False, {'Architecture': 'all'}) + + def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None): + for (pkg, fields, testsuite) in unstable_add: + self.data.add(pkg, True, fields, True, testsuite) + + (excuses, out) = self.run_britney() + #print('-------\nexcuses: %s\n-----' % excuses) + #print('-------\nout: %s\n-----' % out) + #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) + #subprocess.call(['bash', '-i'], cwd=self.data.path) + if considered: + self.assertIn('Valid candidate', excuses) + else: + self.assertIn('Not considered', excuses) + + if excuses_expect: + for re in excuses_expect: + self.assertRegexpMatches(excuses, re) + if excuses_no_expect: + for re in excuses_no_expect: + self.assertNotRegexpMatches(excuses, re) + + self.amqp_requests = set() + try: + with open(self.fake_amqp) as f: + for line in f: + self.amqp_requests.add(line.strip()) + except IOError: + pass + + try: + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/pending.txt')) as f: + self.pending_requests = f.read() + except IOError: + self.pending_requests = None + + def test_multi_rdepends_with_tests(self): + '''Multiple reverse dependencies with tests''' + + # FIXME: while we only submit requests through AMQP, but don't consider + # their results, we don't expect this to hold back stuff. + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<']) + + # we expect the package's and its reverse dependencies' tests to get + # triggered + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green', 'debci-series-amd64:green', + 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen', + ])) + os.unlink(self.fake_amqp) + + expected_pending = '''darkgreen - green 2 +green 2 green 2 +lightgreen - green 2 +''' + + # ... and that they get recorded as pending + self.assertEqual(self.pending_requests, expected_pending) + + # if we run britney again this should *not* trigger any new tests + self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<']) + self.assertEqual(self.amqp_requests, set()) + # but the set of pending tests doesn't change + self.assertEqual(self.pending_requests, expected_pending) + + def test_no_amqp_config(self): + '''Run without autopkgtest requests''' + + # Disable AMQP server config + for line in fileinput.input(self.britney_conf, inplace=True): + if not line.startswith('ADT_AMQP'): + sys.stdout.write(line) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<'], ['autopkgtest']) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, None) + + +class TestAdtBritney(TestBase): + '''Legacy adt-britney/lp:auto-package-testing interface''' + + def setUp(self): + super(TestAdtBritney, self).setUp() # Mofify configuration according to the test context. with open(self.britney_conf, 'r') as fp: From d5181ef32a171578035ef72e1a293d03e17567f0 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 3 Jul 2015 07:35:00 +0200 Subject: [PATCH 05/10] autopkgtest AMQP: Don't declare queues Don't use kombu's SimpleQueue() as that always declares queues and we neither want that (we want to know when we try to talk to a nonexisting queue), nor might the RabbitMQ ACL allow us to do that. So use kombu.Producer without queue declaration. --- autopkgtest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index 1c8acff..4ed789f 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -350,10 +350,11 @@ class AutoPackageTest(object): if amqp_url.startswith('amqp://'): with kombu.Connection(amqp_url) as conn: for q in queues: - amqp_queue = conn.SimpleQueue(q) - for pkg in self.requested_tests: - amqp_queue.put(pkg) - amqp_queue.close() + # don't use SimpleQueue here as it always declares queues; + # ACLs might not allow that + with kombu.Producer(conn, routing_key=q, auto_declare=False) as p: + for pkg in self.requested_tests: + p.publish(pkg) elif amqp_url.startswith('file://'): # in testing mode, adt_amqp will be a file:// URL with open(amqp_url[7:], 'a') as f: From 73bb510389742062a3b56cbbe13357a28ca67d22 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 3 Jul 2015 07:35:41 +0200 Subject: [PATCH 06/10] autopkgtest.py: Clarify comments where the "pending tests" state file lives --- autopkgtest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index 4ed789f..2cef5b1 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -85,7 +85,7 @@ class AutoPackageTest(object): def read_pending_tests(self): '''Read pending test requests from previous britney runs - Read test_state_dir/requested.txt with the format: + Read UNSTABLE/autopkgtest/requested.txt with the format: srcpkg srcver triggering-srcpkg triggering-srcver Initialize self.pending_tests with that data. @@ -119,8 +119,8 @@ class AutoPackageTest(object): def update_pending_tests(self): '''Update pending tests after submitting requested tests - Update test_state_dir/requested.txt, see read_pending_tests() for the - format. + Update UNSTABLE/autopkgtest/requested.txt, see read_pending_tests() for + the format. ''' # merge requested_tests into pending_tests for src, verinfo in self.requested_tests.items(): From 4dc0d388d3572f4612c42f5f748db28a70e108c7 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 6 Jul 2015 08:03:49 +0200 Subject: [PATCH 07/10] add ADT_AMQP option to britney_nobreakall.conf too --- britney_nobreakall.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index eb76959..5aa483d 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -65,3 +65,6 @@ REMOVE_OBSOLETE = no ADT_ENABLE = yes ADT_DEBUG = no ADT_ARCHES = amd64 i386 +# comment this to disable autopkgtest requests +ADT_AMQP = ampq://user:pwd@amqp.example.com + From cc1bd60d19088551f3bc0b4bb419c34bdb7553f2 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 6 Jul 2015 10:24:51 +0200 Subject: [PATCH 08/10] Factor out "tests for source package" calculation --- autopkgtest.py | 58 ++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index 2cef5b1..c45e29c 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -78,6 +78,30 @@ class AutoPackageTest(object): def log_error(self, msg): print('E: [%s] - %s' % (time.asctime(), msg)) + def tests_for_source(self, src, ver): + '''Iterate over all tests that should be run for given source''' + + sources_info = self.britney.sources['unstable'] + # FIXME: For now assume that amd64 has all binaries that we are + # interested in for reverse dependency checking + binaries_info = self.britney.binaries['unstable']['amd64'][0] + + srcinfo = sources_info[src] + # we want to test the package itself, if it still has a test in + # unstable + if srcinfo[AUTOPKGTEST]: + yield (src, ver) + + # plus all direct reverse dependencies of its binaries which have + # an autopkgtest + for binary in srcinfo[BINARIES]: + binary = binary.split('/')[0] # chop off arch + for rdep in binaries_info[binary][RDEPENDS]: + rdep_src = binaries_info[rdep][SOURCE] + if sources_info[rdep_src][AUTOPKGTEST]: + # we don't care about the version of rdep + yield (rdep_src, None) + # # AMQP/cloud interface helpers # @@ -252,34 +276,16 @@ class AutoPackageTest(object): self.log_verbose('Requested autopkgtests for %s, exclusions: %s' % (['%s/%s' % i for i in packages], str(excludes))) - # add new test requests to self.requested_tests from reverse - # dependencies, unless they are already in self.pending_tests - # build a source -> {triggering-source1, ...} map from reverse - # dependencies - sources_info = self.britney.sources['unstable'] - # FIXME: For now assume that amd64 has all binaries that we are - binaries_info = self.britney.binaries['unstable']['amd64'][0] - # interested in for reverse dependency checking for src, ver in packages: - srcinfo = sources_info[src] - # we want to test the package itself, if it still has a test in - # unstable - if srcinfo[AUTOPKGTEST] and src not in excludes: - self.add_test_request(src, ver, src, ver) - # plus all direct reverse dependencies of its binaries which have - # an autopkgtest - for binary in srcinfo[BINARIES]: - binary = binary.split('/')[0] # chop off arch - for rdep in binaries_info[binary][RDEPENDS]: - rdep_src = binaries_info[rdep][SOURCE] - if sources_info[rdep_src][AUTOPKGTEST] and rdep_src not in excludes: - # we don't care about the version of rdep - self.add_test_request(rdep_src, None, src, ver) + for (testsrc, testver) in self.tests_for_source(src, ver): + if testsrc not in excludes: + self.add_test_request(testsrc, testver, src, ver) - for src, verinfo in self.requested_tests.items(): - for ver, triggers in verinfo.items(): - self.log_verbose('Requesting %s/%s autopkgtest to verify %s' % - (src, ver, ', '.join(['%s/%s' % i for i in triggers]))) + if self.britney.options.verbose: + for src, verinfo in self.requested_tests.items(): + for ver, triggers in verinfo.items(): + self.log_verbose('Requesting %s/%s autopkgtest to verify %s' % + (src, ver, ', '.join(['%s/%s' % i for i in triggers]))) # deprecated requests for old Jenkins/lp:auto-package-testing, will go # away From e0c4ec15b62ff00b71eee3d703e2cc1c8bdee3ee Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 6 Jul 2015 11:31:21 +0200 Subject: [PATCH 09/10] AutoPkgTest.update_pending_tests(): Reset self.requested_tests after merging into self.pending_tests --- autopkgtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autopkgtest.py b/autopkgtest.py index c45e29c..2a6cc2f 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -151,6 +151,8 @@ class AutoPackageTest(object): for ver, triggers in verinfo.items(): self.pending_tests.setdefault(src, {}).setdefault( ver, set()).update(triggers) + self.requested_tests = {} + # write it with open(self.pending_tests_file + '.new', 'w') as f: for src in sorted(self.pending_tests): From 49500104ae9f11d14fb7e6466e23d1675d9c00f3 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 6 Jul 2015 15:02:21 +0200 Subject: [PATCH 10/10] AutoPkgTest.tests_for_source(): Don't trip over NBS binaries --- autopkgtest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/autopkgtest.py b/autopkgtest.py index 2a6cc2f..b817aa9 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -96,7 +96,12 @@ class AutoPackageTest(object): # an autopkgtest for binary in srcinfo[BINARIES]: binary = binary.split('/')[0] # chop off arch - for rdep in binaries_info[binary][RDEPENDS]: + try: + rdeps = binaries_info[binary][RDEPENDS] + except KeyError: + self.log_verbose('Ignoring nonexistant binary %s (FTBFS/NBS)?' % binary) + continue + for rdep in rdeps: rdep_src = binaries_info[rdep][SOURCE] if sources_info[rdep_src][AUTOPKGTEST]: # we don't care about the version of rdep