Merge lp:~pitti/britney/britney2-ubuntu-amqp

autopkgtest.py

@@ -26,6 +26,10 @@ from textwrap import dedent
 import time
 import apt_pkg
 
+import kombu
+
+from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE)
+
 
 adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney")
 
@@ -53,7 +57,139 @@ class AutoPackageTest(object):
         self.series = series
         self.debug = debug
         self.read()
-        self.rc_path = None
+        self.rc_path = None  # for adt-britney, obsolete
+        self.test_state_dir = os.path.join(britney.options.unstable,
+                                           'autopkgtest')
+        # map of requested tests from request()
+        # src -> ver -> {(triggering-src1, ver1), ...}
+        self.requested_tests = {}
+        # same map for tests requested in previous runs
+        self.pending_tests = None
+        self.pending_tests_file = os.path.join(self.test_state_dir, 'pending.txt')
+
+        if not os.path.isdir(self.test_state_dir):
+            os.mkdir(self.test_state_dir)
+        self.read_pending_tests()
+
+    def log_verbose(self, msg):
+        if self.britney.options.verbose:
+            print('I: [%s] - %s' % (time.asctime(), msg))
+
+    def log_error(self, msg):
+        print('E: [%s] - %s' % (time.asctime(), msg))
+
+    def tests_for_source(self, src, ver):
+        '''Iterate over all tests that should be run for given source'''
+
+        sources_info = self.britney.sources['unstable']
+        # FIXME: For now assume that amd64 has all binaries that we are
+        # interested in for reverse dependency checking
+        binaries_info = self.britney.binaries['unstable']['amd64'][0]
+
+        srcinfo = sources_info[src]
+        # we want to test the package itself, if it still has a test in
+        # unstable
+        if srcinfo[AUTOPKGTEST]:
+            yield (src, ver)
+
+        # plus all direct reverse dependencies of its binaries which have
+        # an autopkgtest
+        for binary in srcinfo[BINARIES]:
+            binary = binary.split('/')[0]  # chop off arch
+            try:
+                rdeps = binaries_info[binary][RDEPENDS]
+            except KeyError:
+                self.log_verbose('Ignoring nonexistant binary %s (FTBFS/NBS)?' % binary)
+                continue
+            for rdep in rdeps:
+                rdep_src = binaries_info[rdep][SOURCE]
+                if sources_info[rdep_src][AUTOPKGTEST]:
+                    # we don't care about the version of rdep
+                    yield (rdep_src, None)
+
+    #
+    # AMQP/cloud interface helpers
+    #
+
+    def read_pending_tests(self):
+        '''Read pending test requests from previous britney runs
+
+        Read UNSTABLE/autopkgtest/requested.txt with the format:
+            srcpkg srcver triggering-srcpkg triggering-srcver
+
+        Initialize self.pending_tests with that data.
+        '''
+        assert self.pending_tests is None, 'already initialized'
+        self.pending_tests = {}
+        if not os.path.exists(self.pending_tests_file):
+            self.log_verbose('No %s, starting with no pending tests' %
+                             self.pending_tests_file)
+            return
+        with open(self.pending_tests_file) as f:
+            for l in f:
+                l = l.strip()
+                if not l:
+                    continue
+                try:
+                    (src, ver, trigsrc, trigver) = l.split()
+                except ValueError:
+                    self.log_error('ignoring malformed line in %s: %s' %
+                                   (self.pending_tests_file, l))
+                    continue
+                if ver == '-':
+                    ver = None
+                if trigver == '-':
+                    trigver = None
+                self.pending_tests.setdefault(src, {}).setdefault(
+                    ver, set()).add((trigsrc, trigver))
+        self.log_verbose('Read pending requested tests from %s: %s' %
+                         (self.pending_tests_file, self.pending_tests))
+
+    def update_pending_tests(self):
+        '''Update pending tests after submitting requested tests
+
+        Update UNSTABLE/autopkgtest/requested.txt, see read_pending_tests() for
+        the format.
+        '''
+        # merge requested_tests into pending_tests
+        for src, verinfo in self.requested_tests.items():
+            for ver, triggers in verinfo.items():
+                self.pending_tests.setdefault(src, {}).setdefault(
+                    ver, set()).update(triggers)
+        self.requested_tests = {}
+
+        # write it
+        with open(self.pending_tests_file + '.new', 'w') as f:
+            for src in sorted(self.pending_tests):
+                for ver in sorted(self.pending_tests[src]):
+                    for (trigsrc, trigver) in sorted(self.pending_tests[src][ver]):
+                        if ver is None:
+                            ver = '-'
+                        if trigver is None:
+                            trigver = '-'
+                        f.write('%s %s %s %s\n' % (src, ver, trigsrc, trigver))
+        os.rename(self.pending_tests_file + '.new', self.pending_tests_file)
+        self.log_verbose('Updated pending requested tests in %s' %
+                         self.pending_tests_file)
+
+    def add_test_request(self, src, ver, trigsrc, trigver):
+        '''Add one test request to the local self.requested_tests queue
+
+        This will only be done if that test wasn't already requested in a
+        previous run, i. e. it is already in self.pending_tests.
+
+        versions can be None if you don't care about the particular version.
+        '''
+        if (trigsrc, trigver) in self.pending_tests.get(src, {}).get(ver, set()):
+            self.log_verbose('test %s/%s for %s/%s is already pending, not queueing' %
+                             (src, ver, trigsrc, trigver))
+            return
+        self.requested_tests.setdefault(src, {}).setdefault(
+            ver, set()).add((trigsrc, trigver))
+
+    #
+    # obsolete adt-britney helpers
+    #
 
     def _ensure_rc_file(self):
         if self.rc_path:
@@ -137,10 +273,30 @@ class AutoPackageTest(object):
         command.extend(args)
         subprocess.check_call(command)
 
+    #
+    # Public API
+    #
+
     def request(self, packages, excludes=None):
         if excludes is None:
             excludes = []
 
+        self.log_verbose('Requested autopkgtests for %s, exclusions: %s' %
+                         (['%s/%s' % i for i in packages], str(excludes)))
+        for src, ver in packages:
+            for (testsrc, testver) in self.tests_for_source(src, ver):
+                if testsrc not in excludes:
+                    self.add_test_request(testsrc, testver, src, ver)
+
+        if self.britney.options.verbose:
+            for src, verinfo in self.requested_tests.items():
+                for ver, triggers in verinfo.items():
+                    self.log_verbose('Requesting %s/%s autopkgtest to verify %s' %
+                                     (src, ver, ', '.join(['%s/%s' % i for i in triggers])))
+
+        # deprecated requests for old Jenkins/lp:auto-package-testing, will go
+        # away
+
         self._ensure_rc_file()
         request_path = self._request_path
         if os.path.exists(request_path):
@@ -164,9 +320,8 @@ class AutoPackageTest(object):
                     request_file.write(line)
                 else:
                     if self.britney.options.verbose:
-                        print("I: [%s] - Requested autopkgtest for %s but "
-                              "run_autopkgtest set to False" %
-                              (time.asctime(), src))
+                        self.log_verbose("Requested autopkgtest for %s but "
+                                         "run_autopkgtest set to False" % src)
 
         for linebits in self._parse(request_path):
             # Make sure that there's an entry in pkgcauses for each new
@@ -176,8 +331,8 @@ class AutoPackageTest(object):
             src = linebits.pop(0)
             ver = linebits.pop(0)
             if self.britney.options.verbose:
-                print("I: [%s] - Requested autopkgtest for %s_%s (%s)" %
-                      (time.asctime(), src, ver, " ".join(linebits)))
+                self.log_verbose("Requested autopkgtest for %s_%s (%s)" %
+                                 (src, ver, " ".join(linebits)))
             try:
                 status = linebits.pop(0).upper()
                 while True:
@@ -194,6 +349,40 @@ class AutoPackageTest(object):
                 pass
 
     def submit(self):
+        # send AMQP requests for new test requests
+        # TODO: Once we support version constraints in AMQP requests, add them
+        queues = ['debci-%s-%s' % (self.series, arch)
+                  for arch in self.britney.options.adt_arches.split()]
+
+        try:
+            amqp_url = self.britney.options.adt_amqp
+        except AttributeError:
+            self.log_error('ADT_AMQP not set, cannot submit requests')
+            return
+
+        if amqp_url.startswith('amqp://'):
+            with kombu.Connection(amqp_url) as conn:
+                for q in queues:
+                    # don't use SimpleQueue here as it always declares queues;
+                    # ACLs might not allow that
+                    with kombu.Producer(conn, routing_key=q, auto_declare=False) as p:
+                        for pkg in self.requested_tests:
+                            p.publish(pkg)
+        elif amqp_url.startswith('file://'):
+            # in testing mode, adt_amqp will be a file:// URL
+            with open(amqp_url[7:], 'a') as f:
+                for pkg in self.requested_tests:
+                    for q in queues:
+                        f.write('%s:%s\n' % (q, pkg))
+        else:
+            self.log_error('Unknown ADT_AMQP schema in %s' %
+                           self.britney.options.adt_amqp)
+
+        # mark them as pending now
+        self.update_pending_tests()
+
+        # deprecated requests for old Jenkins/lp:auto-package-testing, will go
+        # away
         self._ensure_rc_file()
         request_path = self._request_path
         if os.path.exists(request_path):
@@ -211,10 +400,9 @@ class AutoPackageTest(object):
                     for trigsrc in sorted(self.pkglist[src][ver]['causes']):
                         for trigver, status \
                                 in self.pkglist[src][ver]['causes'][trigsrc]:
-                            print("I: [%s] - Collected autopkgtest status "
-                                  "for %s_%s/%s_%s: " "%s" % (
-                                      time.asctime(), src, ver, trigsrc,
-                                      trigver, status))
+                            self.log_verbose("Collected autopkgtest status "
+                                             "for %s_%s/%s_%s: " "%s" %
+                                             (src, ver, trigsrc, trigver, status))
 
     def results(self, trigsrc, trigver):
         for status, src, ver in self.pkgcauses[trigsrc][trigver]:

britney.conf

@@ -65,6 +65,8 @@ REMOVE_OBSOLETE   = no
 ADT_ENABLE        = yes
 ADT_DEBUG         = no
 ADT_ARCHES        = amd64 i386
+# comment this to disable autopkgtest requests
+ADT_AMQP          = ampq://user:pwd@amqp.example.com
 
 BOOTTEST_ENABLE   = yes
 BOOTTEST_DEBUG    = yes

britney.py

@@ -564,6 +564,7 @@ class Britney(object):
                             [],
                             get_field('Maintainer'),
                             False,
+                            get_field('Testsuite', '').startswith('autopkgtest'),
                            ]
         return sources
 

britney_nobreakall.conf

@@ -65,3 +65,6 @@ REMOVE_OBSOLETE   = no
 ADT_ENABLE        = yes
 ADT_DEBUG         = no
 ADT_ARCHES        = amd64 i386
+# comment this to disable autopkgtest requests
+ADT_AMQP          = ampq://user:pwd@amqp.example.com
+

consts.py

@@ -24,6 +24,7 @@ SECTION = 1
 BINARIES = 2
 MAINTAINER = 3
 FAKESRC = 4
+AUTOPKGTEST = 5
 
 # binary package
 SOURCE = 2

tests/__init__.py

@@ -63,7 +63,7 @@ class TestData:
     def __del__(self):
         shutil.rmtree(self.path)
 
-    def add(self, name, unstable, fields={}, add_src=True):
+    def add(self, name, unstable, fields={}, add_src=True, testsuite=None):
         '''Add a binary package to the index file.
 
         You need to specify at least the package name and in which list to put
@@ -73,7 +73,8 @@ class TestData:
         fields.
 
         Unless add_src is set to False, this will also automatically create a
-        source record, based on fields['Source'] and name.
+        source record, based on fields['Source'] and name. In that case, the
+        "Testsuite:" field is set to the testsuite argument.
         '''
         assert (name not in self.added_binaries[unstable])
         self.added_binaries[unstable].add(name)
@@ -93,8 +94,11 @@ class TestData:
         if add_src:
             src = fields.get('Source', name)
             if src not in self.added_sources[unstable]:
-                self.add_src(src, unstable, {'Version': fields['Version'],
-                                             'Section': fields['Section']})
+                srcfields = {'Version': fields['Version'],
+                             'Section': fields['Section']}
+                if testsuite:
+                    srcfields['Testsuite'] = testsuite
+                self.add_src(src, unstable, srcfields)
 
     def add_src(self, name, unstable, fields={}):
         '''Add a source package to the index file.
@@ -102,7 +106,8 @@ class TestData:
         You need to specify at least the package name and in which list to put
         it (unstable==True for unstable/proposed, or False for
         testing/release). fields specifies all additional entries, which can be
-        Version (default: 1), Section (default: devel), and Extra-Source-Only.
+        Version (default: 1), Section (default: devel), Testsuite (default:
+        none), and Extra-Source-Only.
         '''
         assert (name not in self.added_sources[unstable])
         self.added_sources[unstable].add(name)
@@ -128,18 +133,14 @@ class TestBase(unittest.TestCase):
         super(TestBase, self).setUp()
         self.data = TestData()
         self.britney = os.path.join(PROJECT_DIR, 'britney.py')
-        self.britney_conf = os.path.join(PROJECT_DIR, 'britney.conf')
+        # create temporary config so that tests can hack it
+        self.britney_conf = os.path.join(self.data.path, 'britney.conf')
+        shutil.copy(os.path.join(PROJECT_DIR, 'britney.conf'), self.britney_conf)
         assert os.path.exists(self.britney)
-        assert os.path.exists(self.britney_conf)
 
     def tearDown(self):
         del self.data
 
-    def restore_config(self, content):
-        """Helper for restoring configuration contents on cleanup."""
-        with open(self.britney_conf, 'w') as fp:
-            fp.write(content)
-
     def run_britney(self, args=[]):
         '''Run britney.
 

tests/test_autopkgtest.py

@@ -11,6 +11,7 @@ import operator
 import os
 import sys
 import subprocess
+import fileinput
 import unittest
 
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -27,9 +28,139 @@ apt_pkg.init()
 
 
 class TestAutoPkgTest(TestBase):
+    '''AMQP/cloud interface'''
 
     def setUp(self):
         super(TestAutoPkgTest, self).setUp()
+        self.fake_amqp = os.path.join(self.data.path, 'amqp')
+
+        # Disable boottests and set fake AMQP server
+        for line in fileinput.input(self.britney_conf, inplace=True):
+            if line.startswith('BOOTTEST_ENABLE'):
+                print('BOOTTEST_ENABLE   = no')
+            elif line.startswith('ADT_AMQP'):
+                print('ADT_AMQP = file://%s' % self.fake_amqp)
+            else:
+                sys.stdout.write(line)
+
+        # fake adt-britney script; necessary until we drop that code
+        self.adt_britney = os.path.join(
+            self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney')
+        os.makedirs(os.path.dirname(self.adt_britney))
+        with open(self.adt_britney, 'w') as f:
+            f.write('''#!/bin/sh -e
+touch $HOME/proposed-migration/autopkgtest/work/adt.request.series
+echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
+        os.chmod(self.adt_britney, 0o755)
+
+        # add a bunch of packages to testing to avoid repetition
+        self.data.add('libc6', False)
+        self.data.add('libgreen1', False, {'Source': 'green',
+                                           'Depends': 'libc6 (>= 0.9)'})
+        self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1',
+                                       'Conflicts': 'blue'},
+                      testsuite='autopkgtest')
+        self.data.add('lightgreen', False, {'Depends': 'libgreen1'},
+                      testsuite='autopkgtest')
+        # autodep8 or similar test
+        self.data.add('darkgreen', False, {'Depends': 'libgreen1'},
+                      testsuite='autopkgtest-pkg-foo')
+        self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)',
+                                      'Conflicts': 'green'},
+                      testsuite='specialtest')
+        self.data.add('justdata', False, {'Architecture': 'all'})
+
+    def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None):
+        for (pkg, fields, testsuite) in unstable_add:
+            self.data.add(pkg, True, fields, True, testsuite)
+
+        (excuses, out) = self.run_britney()
+        #print('-------\nexcuses: %s\n-----' % excuses)
+        #print('-------\nout: %s\n-----' % out)
+        #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf))
+        #subprocess.call(['bash', '-i'], cwd=self.data.path)
+        if considered:
+            self.assertIn('Valid candidate', excuses)
+        else:
+            self.assertIn('Not considered', excuses)
+
+        if excuses_expect:
+            for re in excuses_expect:
+                self.assertRegexpMatches(excuses, re)
+        if excuses_no_expect:
+            for re in excuses_no_expect:
+                self.assertNotRegexpMatches(excuses, re)
+
+        self.amqp_requests = set()
+        try:
+            with open(self.fake_amqp) as f:
+                for line in f:
+                    self.amqp_requests.add(line.strip())
+        except IOError:
+            pass
+
+        try:
+            with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/pending.txt')) as f:
+                self.pending_requests = f.read()
+        except IOError:
+                self.pending_requests = None
+
+    def test_multi_rdepends_with_tests(self):
+        '''Multiple reverse dependencies with tests'''
+
+        # FIXME: while we only submit requests through AMQP, but don't consider
+        # their results, we don't expect this to hold back stuff.
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<'])
+
+        # we expect the package's and its reverse dependencies' tests to get
+        # triggered
+        self.assertEqual(
+            self.amqp_requests,
+            set(['debci-series-i386:green', 'debci-series-amd64:green',
+                 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
+                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen',
+                ]))
+        os.unlink(self.fake_amqp)
+
+        expected_pending = '''darkgreen - green 2
+green 2 green 2
+lightgreen - green 2
+'''
+
+        # ... and that they get recorded as pending
+        self.assertEqual(self.pending_requests, expected_pending)
+
+        # if we run britney again this should *not* trigger any new tests
+        self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1</a> to .*>2<'])
+        self.assertEqual(self.amqp_requests, set())
+        # but the set of pending tests doesn't change
+        self.assertEqual(self.pending_requests, expected_pending)
+
+    def test_no_amqp_config(self):
+        '''Run without autopkgtest requests'''
+
+        # Disable AMQP server config
+        for line in fileinput.input(self.britney_conf, inplace=True):
+            if not line.startswith('ADT_AMQP'):
+                sys.stdout.write(line)
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<'], ['autopkgtest'])
+
+        self.assertEqual(self.amqp_requests, set())
+        self.assertEqual(self.pending_requests, None)
+
+
+class TestAdtBritney(TestBase):
+    '''Legacy adt-britney/lp:auto-package-testing interface'''
+
+    def setUp(self):
+        super(TestAdtBritney, self).setUp()
 
         # Mofify configuration according to the test context.
         with open(self.britney_conf, 'r') as fp:
@@ -39,7 +170,6 @@ class TestAutoPkgTest(TestBase):
             'BOOTTEST_ENABLE   = yes', 'BOOTTEST_ENABLE   = no')
         with open(self.britney_conf, 'w') as fp:
             fp.write(new_config)
-        self.addCleanup(self.restore_config, original_config)
 
         # fake adt-britney script
         self.adt_britney = os.path.join(

tests/test_boottest.py

@@ -143,7 +143,6 @@ class TestBoottestEnd2End(TestBase):
             'BOOTTEST_FETCH    = yes', 'BOOTTEST_FETCH    = no')
         with open(self.britney_conf, 'w') as fp:
             fp.write(new_config)
-        self.addCleanup(self.restore_config, original_config)
 
         self.data.add('libc6', False, {'Architecture': 'armhf'}),