Add test results from swift

Until now, autopkgtest results were triggered via an external "adt-britney"
command from lp:auto-package-testing. This required a lot of state files and
duplicated effort, uses hardcoded absolute paths to these external tools, and
is quite hard to understand and maintain. We also want to move away from
Jenkins and rsyncing state files.

Directly retrieve autopkgtest results from a publicly readable and browsable
Swift container, with a debci-compatible layout
(https://wiki.debian.org/debci/DistributedSpec). This now tracks both requests
and results on a per-architecture granularity, so that we can track
per-architecture regressions/always-failed.

Introduce a new ADT_SWIFT_URL config option that sets the swift base URL. If
this key is not set, the behaviour does not change compared to previous
versions, and no results will be retrieved from the cloud.

This still keeps the old adt-britney requests/results as the authoritative
data and for now merely shows the swift results in addition. With that we can
compare the results and run the cloud testing in parallel to find/fix problems
until we switch over. Due to that, the code to britney.py is temporary, does
*not* use AutoPackageTest.results(), and instead just reads the internal
results map.

autopkgtest.py

@@ -24,8 +24,14 @@ import subprocess
 import tempfile
 from textwrap import dedent
 import time
-import apt_pkg
+import json
+import tarfile
+import io
+import copy
+import itertools
+from urllib import urlencode, urlopen
 
+import apt_pkg
 import kombu
 
 from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION)
@@ -42,6 +48,28 @@ ADT_EXCUSES_LABELS = {
 }
 
 
+def srchash(src):
+    '''archive hash prefix for source package'''
+
+    if src.startswith('lib'):
+        return src[:4]
+    else:
+        return src[0]
+
+
+def merge_triggers(trigs1, trigs2):
+    '''Merge two (pkg, ver) trigger iterables
+
+    Return [(pkg, ver), ...] list with only the highest version for each
+    package.
+    '''
+    pkgvers = {}
+    for pkg, ver in itertools.chain(trigs1, trigs2):
+        if apt_pkg.version_compare(ver, pkgvers.setdefault(pkg, '0')) >= 0:
+            pkgvers[pkg] = ver
+    return list(pkgvers.items())
+
+
 class AutoPackageTest(object):
     """autopkgtest integration
 
@@ -71,6 +99,28 @@ class AutoPackageTest(object):
             os.mkdir(self.test_state_dir)
         self.read_pending_tests()
 
+        # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers)]
+        # - "passed" is a bool
+        # - It's tempting to just use a global "latest" time stamp, but due to
+        #   swift's "eventual consistency" we might miss results with older time
+        #   stamps from other packages that we don't see in the current run, but
+        #   will in the next one. This doesn't hurt for older results of the same
+        #   package.
+        # - triggers is a list of (source, version) pairs which unstable
+        #   packages triggered this test run. We need to track this to avoid
+        #   unnecessarily re-running tests.
+        self.test_results = {}
+        self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache')
+
+        # read the cached results that we collected so far
+        if os.path.exists(self.results_cache_file):
+            with open(self.results_cache_file) as f:
+                self.test_results = json.load(f)
+            self.log_verbose('Read previous results from %s' % self.results_cache_file)
+        else:
+            self.log_verbose('%s does not exist, re-downloading all results '
+                             'from swift' % self.results_cache_file)
+
     def log_verbose(self, msg):
         if self.britney.options.verbose:
             print('I: [%s] - %s' % (time.asctime(), msg))
@@ -175,16 +225,124 @@ class AutoPackageTest(object):
         '''Add one test request to the local self.requested_tests queue
 
         This will only be done if that test wasn't already requested in a
-        previous run, i. e. it is already in self.pending_tests.
+        previous run (i. e. not already in self.pending_tests) or there already
+        is a result for it.
         '''
+        try:
+            for (tsrc, tver) in self.test_results[src][arch][1][ver][1]:
+                if tsrc == trigsrc and apt_pkg.version_compare(tver, trigver) >= 0:
+                    self.log_verbose('There already is a result for %s/%s/%s triggered by %s/%s' %
+                                     (src, ver, arch, tsrc, tver))
+                    return
+        except KeyError:
+            pass
+
         if (trigsrc, trigver) in self.pending_tests.get(src, {}).get(
-            ver, {}).get(arch, set()):
+                ver, {}).get(arch, set()):
             self.log_verbose('test %s/%s/%s for %s/%s is already pending, not queueing' %
                              (src, ver, arch, trigsrc, trigver))
             return
         self.requested_tests.setdefault(src, {}).setdefault(
             ver, {}).setdefault(arch, set()).add((trigsrc, trigver))
 
+    def fetch_swift_results(self, swift_url, src, arch):
+        '''Download new results for source package/arch from swift'''
+
+        # prepare query: get all runs with a timestamp later than latest_stamp
+        # for this package/arch; '@' is at the end of each run timestamp, to
+        # mark the end of a test run directory path
+        # example: <autopkgtest-wily>wily/amd64/libp/libpng/20150630_054517@/result.tar
+        query = {'delimiter': '@',
+                 'prefix': '%s/%s/%s/%s/' % (self.series, arch, srchash(src), src)}
+        try:
+            # don't include the last run again, so make the marker
+            # "infinitesimally later" by appending 'zz'
+            query['marker'] = self.test_results[src][arch][0] + 'zz'
+        except KeyError:
+            # no stamp yet, download all results
+            pass
+
+        # request new results from swift
+        url = os.path.join(swift_url, 'autopkgtest-' + self.series)
+        url += '?' + urlencode(query)
+        try:
+            f = urlopen(url)
+            if f.getcode() == 200:
+                result_paths = f.read().strip().splitlines()
+            else:
+                self.log_error('Failure to fetch swift results from %s: %u' %
+                               (url, f.getcode()))
+                f.close()
+                return
+            f.close()
+        except IOError as e:
+            self.log_error('Failure to fetch swift results from %s: %s' % (url, str(e)))
+            return
+
+        for p in result_paths:
+            self.fetch_one_result(os.path.join(
+                swift_url, 'autopkgtest-' + self.series, p, 'result.tar'), src, arch)
+
+    def fetch_one_result(self, url, src, arch):
+        '''Download one result URL for source/arch
+
+        Remove matching pending_tests entries.
+        '''
+        try:
+            f = urlopen(url)
+            if f.getcode() == 200:
+                tar_bytes = io.BytesIO(f.read())
+                f.close()
+            else:
+                self.log_error('Failure to fetch %s: %u' % (url, f.getcode()))
+                return
+        except IOError as e:
+            self.log_error('Failure to fetch %s: %s' % (url, str(e)))
+            return
+
+        try:
+            with tarfile.open(None, 'r', tar_bytes) as tar:
+                exitcode = int(tar.extractfile('exitcode').read().strip())
+                srcver = tar.extractfile('testpkg-version').read().decode().strip()
+                (ressrc, ver) = srcver.split()
+        except (KeyError, ValueError, tarfile.TarError) as e:
+            self.log_error('%s is damaged: %s' % (url, str(e)))
+            return
+
+        if src != ressrc:
+            self.log_error('%s is a result for package %s, but expected package %s' %
+                           (url, ressrc, src))
+            return
+
+        stamp = os.path.basename(os.path.dirname(url))
+        # allow some skipped tests, but nothing else
+        passed = exitcode in [0, 2]
+
+        self.log_verbose('Fetched test result for %s/%s on %s: %s' % (
+            src, ver, arch, passed and 'pass' or 'fail'))
+
+        # remove matching test requests, remember triggers
+        satisfied_triggers = set()
+        for pending_ver, pending_archinfo in self.pending_tests.get(src, {}).copy().items():
+            # don't consider newer requested versions
+            if apt_pkg.version_compare(pending_ver, ver) <= 0:
+                try:
+                    t = pending_archinfo[arch]
+                    self.log_verbose('-> matches pending request for triggers %s' % str(t))
+                    satisfied_triggers.update(t)
+                    del self.pending_tests[src][pending_ver][arch]
+                except KeyError:
+                    self.log_error('-> does not match any pending request!')
+                    pass
+
+        # add this result
+        src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}])
+        src_arch_results[1][ver] = (passed, merge_triggers(
+            src_arch_results[1].get(ver, (None, []))[1], satisfied_triggers))
+        # update latest_stamp
+        if stamp > src_arch_results[0]:
+            src_arch_results[0] = stamp
+
     #
     # obsolete adt-britney helpers
     #
@@ -397,6 +555,38 @@ class AutoPackageTest(object):
             self._adt_britney("submit", request_path)
 
     def collect(self):
+        # fetch results from swift
+        try:
+            swift_url = self.britney.options.adt_swift_url
+        except AttributeError:
+            self.log_error('ADT_SWIFT_URL not set, cannot collect results')
+            swift_url = None
+        try:
+            self.britney.options.adt_amqp
+        except AttributeError:
+            self.log_error('ADT_AMQP not set, not collecting results from swift')
+            swift_url = None
+
+        if swift_url:
+            # update results from swift for all packages that we are waiting
+            # for, and remove pending tests that we have results for on all
+            # arches
+            for pkg, verinfo in copy.deepcopy(self.pending_tests.items()):
+                for archinfo in verinfo.values():
+                    for arch in archinfo:
+                        self.fetch_swift_results(swift_url, pkg, arch)
+
+            # update the results cache
+            with open(self.results_cache_file + '.new', 'w') as f:
+                json.dump(self.test_results, f, indent=2)
+            os.rename(self.results_cache_file + '.new', self.results_cache_file)
+            self.log_verbose('Updated results cache')
+
+            # new results remove pending requests, update the on-disk cache
+            self.update_pending_tests()
+
+        # deprecated results for old Jenkins/lp:auto-package-testing, will go
+        # away
         self._ensure_rc_file()
         result_path = self._result_path
         self._adt_britney("collect", "-O", result_path)
@@ -413,6 +603,13 @@ class AutoPackageTest(object):
                                              (src, ver, trigsrc, trigver, status))
 
     def results(self, trigsrc, trigver):
+        '''Return test results for triggering package
+
+        Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests
+        that got triggered by trigsrc/trigver.
+        '''
+        # deprecated results for old Jenkins/lp:auto-package-testing, will go
+        # away
         for status, src, ver in self.pkgcauses[trigsrc][trigver]:
             # Check for regression
             if status == 'FAIL':

britney.conf

@@ -67,6 +67,8 @@ ADT_DEBUG         = no
 ADT_ARCHES        = amd64 i386
 # comment this to disable autopkgtest requests
 ADT_AMQP          = ampq://user:pwd@amqp.example.com
+# Swift base URL with the results (must be publicly readable and browsable)
+ADT_SWIFT_URL     = https://objectstorage.mycloud.example.com/v1/AUTH_autopkgtest
 
 BOOTTEST_ENABLE   = yes
 BOOTTEST_DEBUG    = yes

britney.py

@@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes,
 from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
                    SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
                    PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL)
-from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS
+from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash
 from boottest import BootTest
 
 
@@ -1853,6 +1853,7 @@ class Britney(object):
             jenkins_private = (
                 "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %
                 self.options.series.title())
+            cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s"
             for e in autopkgtest_excuses:
                 adtpass = True
                 for status, adtsrc, adtver in autopkgtest.results(
@@ -1883,6 +1884,33 @@ class Britney(object):
                                 "%s" % (adtsrc, adtver, forces[0].user))
                         else:
                             adtpass = False
+
+                # temporary: also show results from cloud based tests,
+                # until that becomes the primary mechanism
+                for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]):
+                    msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver)
+                    archmsg = []
+                    for arch in self.options.adt_arches.split():
+                        url = cloud_url % {'h': srchash(testsrc), 's': testsrc,
+                                           'r': self.options.series, 'a': arch}
+                        try:
+                            r = autopkgtest.test_results[testsrc][arch][1][testver][0]
+                            status = r and 'PASS' or 'REGRESSION'
+                        except KeyError:
+                            try:
+                                autopkgtest.pending_tests[testsrc][testver][arch]
+                                status = 'RUNNING'
+                            except KeyError:
+                                # neither done nor pending -> exclusion, or disabled
+                                continue
+
+                        archmsg.append('<a href="%s">%s: %s</a>' %
+                                       (url, arch, ADT_EXCUSES_LABELS[status]))
+
+                    if archmsg:
+                        e.addhtml(msg + ', '.join(archmsg))
+                # end of temporary code
+
                 if not adtpass and e.is_valid:
                     hints = self.hints.search('force-skiptest', package=e.name)
                     hints.extend(self.hints.search('force', package=e.name))

britney_nobreakall.conf

@@ -67,4 +67,5 @@ ADT_DEBUG         = no
 ADT_ARCHES        = amd64 i386
 # comment this to disable autopkgtest requests
 ADT_AMQP          = ampq://user:pwd@amqp.example.com
-
+# Swift base URL with the results (must be publicly readable and browsable)
+ADT_SWIFT_URL     = https://objectstorage.mycloud.example.com/v1/AUTH_autopkgtest

tests/test_autopkgtest.py

@@ -13,12 +13,13 @@ import sys
 import subprocess
 import fileinput
 import unittest
+import json
 
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, PROJECT_DIR)
 
 from autopkgtest import ADT_EXCUSES_LABELS
-from tests import TestBase
+from tests import TestBase, mock_swift
 
 NOT_CONSIDERED = False
 VALID_CANDIDATE = True
@@ -34,12 +35,14 @@ class TestAutoPkgTest(TestBase):
         super(TestAutoPkgTest, self).setUp()
         self.fake_amqp = os.path.join(self.data.path, 'amqp')
 
-        # Disable boottests and set fake AMQP server
+        # Disable boottests and set fake AMQP and Swift server
         for line in fileinput.input(self.britney_conf, inplace=True):
             if line.startswith('BOOTTEST_ENABLE'):
                 print('BOOTTEST_ENABLE   = no')
             elif line.startswith('ADT_AMQP'):
                 print('ADT_AMQP = file://%s' % self.fake_amqp)
+            elif line.startswith('ADT_SWIFT_URL'):
+                print('ADT_SWIFT_URL = http://localhost:18085')
             else:
                 sys.stdout.write(line)
 
@@ -70,11 +73,22 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
                       testsuite='specialtest')
         self.data.add('justdata', False, {'Architecture': 'all'})
 
+        # create mock Swift server (but don't start it yet, as tests first need
+        # to poke in results)
+        self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085)
+        self.swift.set_results({})
+
+    def tearDown(self):
+        del self.swift
+
     def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None):
         for (pkg, fields, testsuite) in unstable_add:
             self.data.add(pkg, True, fields, True, testsuite)
 
+        self.swift.start()
         (excuses, out) = self.run_britney()
+        self.swift.stop()
+
         #print('-------\nexcuses: %s\n-----' % excuses)
         #print('-------\nout: %s\n-----' % out)
         #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf))
@@ -105,6 +119,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
         except IOError:
                 self.pending_requests = None
 
+        return out
+
     def test_multi_rdepends_with_tests_all_running(self):
         '''Multiple reverse dependencies with tests (all running)'''
 
@@ -113,7 +129,10 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
             # FIXME: while we only submit requests through AMQP, but don't consider
             # their results, we don't expect this to hold back stuff.
             VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<'])
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
 
         # we expect the package's and its reverse dependencies' tests to get
         # triggered
@@ -121,8 +140,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
             self.amqp_requests,
             set(['debci-series-i386:green', 'debci-series-amd64:green',
                  'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
-                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen',
-                ]))
+                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
         os.unlink(self.fake_amqp)
 
         # ... and that they get recorded as pending
@@ -141,6 +159,108 @@ lightgreen 1 i386 green 2
         # but the set of pending tests doesn't change
         self.assertEqual(self.pending_requests, expected_pending)
 
+    def test_multi_rdepends_with_tests_all_pass(self):
+        '''Multiple reverse dependencies with tests (all pass)'''
+
+        # first run requests tests and marks them as pending
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            # FIXME: while we only submit requests through AMQP, but don't consider
+            # their results, we don't expect this to hold back stuff.
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+
+        # second run collects the results
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1'),
+            # version in testing fails
+            'series/i386/g/green/20150101_020000@': (4, 'green 1'),
+            'series/amd64/g/green/20150101_020000@': (4, 'green 1'),
+            # version in unstable succeeds
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (0, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        # all tests ran, there should be no more pending ones
+        self.assertEqual(self.pending_requests, '')
+
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+        # caches the results and triggers
+        with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f:
+            res = json.load(f)
+        self.assertEqual(res['green']['i386'],
+                         ['20150101_100200@', {'1': [False, []],
+                                               '2': [True, [['green', '2']]]}])
+        self.assertEqual(res['lightgreen']['amd64'],
+                         ['20150101_100101@', {'1': [True, [['green', '2']]]}])
+
+        # third run should not trigger any new tests, should all be in the
+        # cache
+        os.unlink(self.fake_amqp)
+        self.swift.set_results({})
+        out = self.do_test(
+            [],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+        self.assertEqual(self.amqp_requests, set())
+        self.assertEqual(self.pending_requests, '')
+        self.assertNotIn('Failure', out, out)
+
+    def test_multi_rdepends_with_tests_mixed(self):
+        '''Multiple reverse dependencies with tests (mixed results)'''
+
+        # first run requests tests and marks them as pending
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            # FIXME: while we only submit requests through AMQP, but don't consider
+            # their results, we don't expect this to hold back stuff.
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+
+        # second run collects the results
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        self.do_test(
+            [],
+            # FIXME: while we only submit requests through AMQP, but don't consider
+            # their results, we don't expect this to hold back stuff.
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass'])
+
+        # there should be some pending ones
+        self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
+        self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
+
     def test_package_pair_running(self):
         '''Two packages in unstable that need to go in together (running)'''
 
@@ -159,8 +279,7 @@ lightgreen 1 i386 green 2
             self.amqp_requests,
             set(['debci-series-i386:green', 'debci-series-amd64:green',
                  'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
-                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen',
-                ]))
+                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
         os.unlink(self.fake_amqp)
 
         # ... and that they get recorded as pending
@@ -180,7 +299,7 @@ lightgreen 2 i386 lightgreen 2
 
         # Disable AMQP server config
         for line in fileinput.input(self.britney_conf, inplace=True):
-            if not line.startswith('ADT_AMQP'):
+            if not line.startswith('ADT_AMQP') and not line.startswith('ADT_SWIFT_URL'):
                 sys.stdout.write(line)
 
         self.do_test(