Switch autopkgtest evaluation to cloud results

Change AutoPackageTest.results() to evaluate the Swift results instead of the
adt-britney ones.

TODO:
 - Add more tests (like for adt-britney)
 - Drop triggering of adt-britney tests
 - Drop adt-britney tests (which fail now)

autopkgtest.py

@@ -39,7 +39,6 @@ from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION)
 
 adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney")
 
-ADT_PASS = ["PASS", "ALWAYSFAIL"]
 ADT_EXCUSES_LABELS = {
     "PASS": '<span style="background:#87d96c">Pass</span>',
     "ALWAYSFAIL": '<span style="background:#e5c545">Always failed</span>',
@@ -647,23 +646,33 @@ class AutoPackageTest(object):
     def results(self, trigsrc, trigver):
         '''Return test results for triggering package
 
-        Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests
+        Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING)
-        that got triggered by trigsrc/trigver.
+        iterator for all package tests that got triggered by trigsrc/trigver.
         '''
-        # deprecated results for old Jenkins/lp:auto-package-testing, will go
+        for testsrc, testver in self.tests_for_source(trigsrc, trigver):
-        # away
+            passed = True
-        for status, src, ver in self.pkgcauses[trigsrc][trigver]:
+            arch_status = {}
-            # Check for regression
+            for arch in self.britney.options.adt_arches.split():
-            if status == 'FAIL':
+                try:
-                passed_once = False
+                    if self.test_results[testsrc][arch][1][testver][0]:
-                for lver in self.pkglist[src]:
+                        arch_status[arch] = 'PASS'
-                    for trigsrc in self.pkglist[src][lver]['causes']:
+                    else:
-                        for trigver, status \
+                        if self.test_results[testsrc][arch][2]:
-                                in self.pkglist[src][lver]['causes'][trigsrc]:
+                            arch_status[arch] = 'REGRESSION'
-                            if status == 'PASS':
+                            passed = False
-                                passed_once = True
+                        else:
-                if not passed_once:
+                            arch_status[arch] = 'ALWAYSFAIL'
-                    status = 'ALWAYSFAIL'
+                except KeyError:
-                else:
+                    try:
-                    status = 'REGRESSION'
+                        self.pending_tests[testsrc][testver][arch]
-            yield status, src, ver
+                        arch_status[arch] = 'RUNNING'
+                        passed = False
+                    except KeyError:
+                        # neither done nor pending -> exclusion, or disabled
+                        continue
+
+            # disabled or ignored?
+            if not arch_status:
+                continue
+
+            yield (passed, testsrc, testver, arch_status)

britney.py

@@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes,
 from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
                    SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
                    PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL)
-from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash
+from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash
 from boottest import BootTest
 
 
@@ -1849,28 +1849,21 @@ class Britney(object):
             if not self.options.dry_run:
                 autopkgtest.submit()
                 autopkgtest.collect(autopkgtest_packages)
-            jenkins_public = "https://jenkins.qa.ubuntu.com/job"
-            jenkins_private = (
-                "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %
-                self.options.series.title())
             cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s"
+            adtpass = True
             for e in autopkgtest_excuses:
-                adtpass = True
+                for adtpass, adtsrc, adtver, arch_status in autopkgtest.results(
-                for status, adtsrc, adtver in autopkgtest.results(
                         e.name, e.ver[1]):
-                    public_url = "%s/%s-adt-%s/lastBuild" % (
+                    archmsg = []
-                        jenkins_public, self.options.series,
+                    for arch in sorted(arch_status):
-                        adtsrc.replace("+", "-"))
+                        url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc,
-                    private_url = "%s/%s-adt-%s/lastBuild" % (
+                                           'r': self.options.series, 'a': arch}
-                        jenkins_private, self.options.series,
+                        archmsg.append('<a href="%s">%s: %s</a>' %
-                        adtsrc.replace("+", "-"))
+                                       (url, arch, ADT_EXCUSES_LABELS[arch_status[arch]]))
-                    adt_label = ADT_EXCUSES_LABELS.get(status, status)
+                    e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg)))
-                    e.addhtml(
+
-                        "autopkgtest for %s %s: %s (Jenkins: "
+                    # hints can override failures
-                        "<a href=\"%s\">public</a>, "
+                    if not adtpass:
-                        "<a href=\"%s\">private</a>)" %
-                        (adtsrc, adtver, adt_label, public_url, private_url))
-                    if status not in ADT_PASS:
                         hints = self.hints.search(
                             'force-badtest', package=adtsrc)
                         hints.extend(
@@ -1882,34 +1875,7 @@ class Britney(object):
                             e.addhtml(
                                 "Should wait for %s %s test, but forced by "
                                 "%s" % (adtsrc, adtver, forces[0].user))
-                        else:
+                            adtpass = True
-                            adtpass = False
-
-                # temporary: also show results from cloud based tests,
-                # until that becomes the primary mechanism
-                for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]):
-                    msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver)
-                    archmsg = []
-                    for arch in self.options.adt_arches.split():
-                        url = cloud_url % {'h': srchash(testsrc), 's': testsrc,
-                                           'r': self.options.series, 'a': arch}
-                        try:
-                            r = autopkgtest.test_results[testsrc][arch][1][testver][0]
-                            status = r and 'PASS' or 'REGRESSION'
-                        except KeyError:
-                            try:
-                                autopkgtest.pending_tests[testsrc][testver][arch]
-                                status = 'RUNNING'
-                            except KeyError:
-                                # neither done nor pending -> exclusion, or disabled
-                                continue
-
-                        archmsg.append('<a href="%s">%s: %s</a>' %
-                                       (url, arch, ADT_EXCUSES_LABELS[status]))
-
-                    if archmsg:
-                        e.addhtml(msg + ', '.join(archmsg))
-                # end of temporary code
 
                 if not adtpass and e.is_valid:
                     hints = self.hints.search('force-skiptest', package=e.name)

tests/test_autopkgtest.py

@@ -125,9 +125,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
 
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -153,7 +151,7 @@ lightgreen 1 i386 green 2
         self.assertEqual(self.pending_requests, expected_pending)
 
         # if we run britney again this should *not* trigger any new tests
-        self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1</a> to .*>2<'])
+        self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1</a> to .*>2<'])
         self.assertEqual(self.amqp_requests, set())
         # but the set of pending tests doesn't change
         self.assertEqual(self.pending_requests, expected_pending)
@@ -164,9 +162,7 @@ lightgreen 1 i386 green 2
         # first run requests tests and marks them as pending
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -233,9 +229,7 @@ lightgreen 1 i386 green 2
         # first run requests tests and marks them as pending
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -252,11 +246,9 @@ lightgreen 1 i386 green 2
 
         out = self.do_test(
             [],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
              r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
              r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass'])
 
@@ -267,15 +259,67 @@ lightgreen 1 i386 green 2
         self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
         self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
 
+    def test_multi_rdepends_with_tests_regression(self):
+        '''Multiple reverse dependencies with tests (regression)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+    def test_multi_rdepends_with_tests_always_failed(self):
+        '''Multiple reverse dependencies with tests (always failed)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (4, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
     def test_package_pair_running(self):
         '''Two packages in unstable that need to go in together (running)'''
 
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
              ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'\blightgreen\b.*>1</a> to .*>2<'])
 
@@ -313,9 +357,7 @@ lightgreen 2 i386 lightgreen 2
 
         self.do_test(
             [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\blightgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
             ['in progress'])
@@ -341,9 +383,7 @@ lightgreen 2 i386 lightgreen 2
 
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
              r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
@@ -392,9 +432,7 @@ lightgreen 2 i386 lightgreen 2
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
              ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'\blightgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',