Switch autopkgtest evaluation to cloud results

Change AutoPackageTest.results() to evaluate the Swift results instead of the adt-britney ones. TODO: - Add more tests (like for adt-britney) - Drop triggering of adt-britney tests - Drop adt-britney tests (which fail now)
10 years ago · 2bf6eb5652
parent 76287b50ca
commit 2bf6eb5652
3 changed files with 107 additions and 94 deletions
--- a/autopkgtest.py
+++ b/autopkgtest.py
@ -39,7 +39,6 @@ from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION)

 adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney")

-ADT_PASS = ["PASS", "ALWAYSFAIL"]
 ADT_EXCUSES_LABELS = {
    "PASS": '<span style="background:#87d96c">Pass</span>',
    "ALWAYSFAIL": '<span style="background:#e5c545">Always failed</span>',
@ -647,23 +646,33 @@ class AutoPackageTest(object):
    def results(self, trigsrc, trigver):
        '''Return test results for triggering package

-        Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests
-        that got triggered by trigsrc/trigver.
+        Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING)
+        iterator for all package tests that got triggered by trigsrc/trigver.
        '''
-        # deprecated results for old Jenkins/lp:auto-package-testing, will go
-        # away
-        for status, src, ver in self.pkgcauses[trigsrc][trigver]:
-            # Check for regression
-            if status == 'FAIL':
-                passed_once = False
-                for lver in self.pkglist[src]:
-                    for trigsrc in self.pkglist[src][lver]['causes']:
-                        for trigver, status \
-                                in self.pkglist[src][lver]['causes'][trigsrc]:
-                            if status == 'PASS':
-                                passed_once = True
-                if not passed_once:
-                    status = 'ALWAYSFAIL'
-                else:
-                    status = 'REGRESSION'
-            yield status, src, ver
+        for testsrc, testver in self.tests_for_source(trigsrc, trigver):
+            passed = True
+            arch_status = {}
+            for arch in self.britney.options.adt_arches.split():
+                try:
+                    if self.test_results[testsrc][arch][1][testver][0]:
+                        arch_status[arch] = 'PASS'
+                    else:
+                        if self.test_results[testsrc][arch][2]:
+                            arch_status[arch] = 'REGRESSION'
+                            passed = False
+                        else:
+                            arch_status[arch] = 'ALWAYSFAIL'
+                except KeyError:
+                    try:
+                        self.pending_tests[testsrc][testver][arch]
+                        arch_status[arch] = 'RUNNING'
+                        passed = False
+                    except KeyError:
+                        # neither done nor pending -> exclusion, or disabled
+                        continue
+
+            # disabled or ignored?
+            if not arch_status:
+                continue
+
+            yield (passed, testsrc, testver, arch_status)
--- a/britney.py
+++ b/britney.py
@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes,
 from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
                   SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
                   PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL)
-from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash
+from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash
 from boottest import BootTest


@ -1849,28 +1849,21 @@ class Britney(object):
            if not self.options.dry_run:
                autopkgtest.submit()
                autopkgtest.collect(autopkgtest_packages)
-            jenkins_public = "https://jenkins.qa.ubuntu.com/job"
-            jenkins_private = (
-                "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %
-                self.options.series.title())
            cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s"
+            adtpass = True
            for e in autopkgtest_excuses:
-                adtpass = True
-                for status, adtsrc, adtver in autopkgtest.results(
+                for adtpass, adtsrc, adtver, arch_status in autopkgtest.results(
                        e.name, e.ver[1]):
-                    public_url = "%s/%s-adt-%s/lastBuild" % (
-                        jenkins_public, self.options.series,
-                        adtsrc.replace("+", "-"))
-                    private_url = "%s/%s-adt-%s/lastBuild" % (
-                        jenkins_private, self.options.series,
-                        adtsrc.replace("+", "-"))
-                    adt_label = ADT_EXCUSES_LABELS.get(status, status)
-                    e.addhtml(
-                        "autopkgtest for %s %s: %s (Jenkins: "
-                        "<a href=\"%s\">public</a>, "
-                        "<a href=\"%s\">private</a>)" %
-                        (adtsrc, adtver, adt_label, public_url, private_url))
-                    if status not in ADT_PASS:
+                    archmsg = []
+                    for arch in sorted(arch_status):
+                        url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc,
+                                           'r': self.options.series, 'a': arch}
+                        archmsg.append('<a href="%s">%s: %s</a>' %
+                                       (url, arch, ADT_EXCUSES_LABELS[arch_status[arch]]))
+                    e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg)))
+
+                    # hints can override failures
+                    if not adtpass:
                        hints = self.hints.search(
                            'force-badtest', package=adtsrc)
                        hints.extend(
@ -1882,34 +1875,7 @@ class Britney(object):
                            e.addhtml(
                                "Should wait for %s %s test, but forced by "
                                "%s" % (adtsrc, adtver, forces[0].user))
-                        else:
-                            adtpass = False
-
-                # temporary: also show results from cloud based tests,
-                # until that becomes the primary mechanism
-                for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]):
-                    msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver)
-                    archmsg = []
-                    for arch in self.options.adt_arches.split():
-                        url = cloud_url % {'h': srchash(testsrc), 's': testsrc,
-                                           'r': self.options.series, 'a': arch}
-                        try:
-                            r = autopkgtest.test_results[testsrc][arch][1][testver][0]
-                            status = r and 'PASS' or 'REGRESSION'
-                        except KeyError:
-                            try:
-                                autopkgtest.pending_tests[testsrc][testver][arch]
-                                status = 'RUNNING'
-                            except KeyError:
-                                # neither done nor pending -> exclusion, or disabled
-                                continue
-
-                        archmsg.append('<a href="%s">%s: %s</a>' %
-                                       (url, arch, ADT_EXCUSES_LABELS[status]))
-
-                    if archmsg:
-                        e.addhtml(msg + ', '.join(archmsg))
-                # end of temporary code
+                            adtpass = True

                if not adtpass and e.is_valid:
                    hints = self.hints.search('force-skiptest', package=e.name)
--- a/tests/test_autopkgtest.py
+++ b/tests/test_autopkgtest.py
@ -125,9 +125,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -153,7 +151,7 @@ lightgreen 1 i386 green 2
        self.assertEqual(self.pending_requests, expected_pending)

        # if we run britney again this should *not* trigger any new tests
-        self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1</a> to .*>2<'])
+        self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1</a> to .*>2<'])
        self.assertEqual(self.amqp_requests, set())
        # but the set of pending tests doesn't change
        self.assertEqual(self.pending_requests, expected_pending)
@ -164,9 +162,7 @@ lightgreen 1 i386 green 2
        # first run requests tests and marks them as pending
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -233,9 +229,7 @@ lightgreen 1 i386 green 2
        # first run requests tests and marks them as pending
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -252,11 +246,9 @@ lightgreen 1 i386 green 2

        out = self.do_test(
            [],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass'])

@ -267,15 +259,67 @@ lightgreen 1 i386 green 2
        self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
        self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)

+    def test_multi_rdepends_with_tests_regression(self):
+        '''Multiple reverse dependencies with tests (regression)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+    def test_multi_rdepends_with_tests_always_failed(self):
+        '''Multiple reverse dependencies with tests (always failed)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (4, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
    def test_package_pair_running(self):
        '''Two packages in unstable that need to go in together (running)'''

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
             ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
             r'\blightgreen\b.*>1</a> to .*>2<'])

@ -313,9 +357,7 @@ lightgreen 2 i386 lightgreen 2

        self.do_test(
            [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\blightgreen\b.*>1</a> to .*>2<',
             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
            ['in progress'])
@ -341,9 +383,7 @@ lightgreen 2 i386 lightgreen 2

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
@ -392,9 +432,7 @@ lightgreen 2 i386 lightgreen 2
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
             ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
+            NOT_CONSIDERED,
            [r'\bgreen\b.*>1</a> to .*>2<',
             r'\blightgreen\b.*>1</a> to .*>2<',
             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',