Change autopkgtest tests to check YAML instead of HTML

Matching the HTML for regexps does not work reliably when we have multiple unstable packages of which only some are valid candidates but others aren't. It also imposes a too strict test on the particular formatting when we are only interested in the actual data and structure. So move towards checking the machine parseable YAML instead and read that into a proper Python dict. This exposed a bug uncovered by test_rdepends_unbuilt() which we previously missed because we couldn't check triggered tests per package.
10 years ago · 39dc24ec71
parent 358920c395
commit 39dc24ec71
3 changed files with 337 additions and 261 deletions
--- a/tests/init.py
+++ b/tests/init.py
@ -11,7 +11,6 @@ import subprocess
 import tempfile
 import unittest

-
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

 architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el']
@ -154,7 +153,7 @@ class TestBase(unittest.TestCase):
        '''Run britney.

        Assert that it succeeds and does not produce anything on stderr.
-        Return (excuses.html, britney_out).
+        Return (excuses.yaml, excuses.html, britney_out).
        '''
        britney = subprocess.Popen([self.britney, '-v', '-c', self.britney_conf,
                                    '--distribution=ubuntu',
@ -167,11 +166,14 @@ class TestBase(unittest.TestCase):
        self.assertEqual(britney.returncode, 0, out + err)
        self.assertEqual(err, '')

+        with open(os.path.join(self.data.path, 'output', self.data.series,
+                               'excuses.yaml')) as f:
+            yaml = f.read()
        with open(os.path.join(self.data.path, 'output', self.data.series,
                               'excuses.html')) as f:
-            excuses = f.read()
+            html = f.read()

-        return (excuses, out)
+        return (yaml, html, out)

    def create_hint(self, username, content):
        '''Create a hint file for the given username and content'''
--- a/tests/test_autopkgtest.py
+++ b/tests/test_autopkgtest.py
@ -12,16 +12,15 @@ import sys
 import fileinput
 import unittest
 import json
+import pprint
+
+import yaml

 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, PROJECT_DIR)

 from tests import TestBase, mock_swift

-NOT_CONSIDERED = False
-VALID_CANDIDATE = True
-
-
 apt_pkg.init()


@ -70,29 +69,54 @@ class TestAutoPkgTest(TestBase):
    def tearDown(self):
        del self.swift

-    def do_test(self, unstable_add, considered, excuses_expect=None, excuses_no_expect=None):
+    def do_test(self, unstable_add, expect_status, expect_excuses={}):
+        '''Run britney with some unstable packages and verify excuses.
+
+        unstable_add is a list of (binpkgname, field_dict, testsuite_value)
+        passed to TestData.add for "unstable".
+
+        expect_status is a dict sourcename → (is_candidate, testsrc → arch → status)
+        that is checked against the excuses YAML.
+
+        expect_excuses is a dict sourcename →  [(key, value), ...]
+        matches that are checked against the excuses YAML.
+
+        Return (output, excuses_dict).
+        '''
        for (pkg, fields, testsuite) in unstable_add:
            self.data.add(pkg, True, fields, True, testsuite)

        self.swift.start()
-        (excuses, out) = self.run_britney()
+        (excuses_yaml, excuses_html, out) = self.run_britney()
        self.swift.stop()

+        # convert excuses to source indexed dict
+        excuses_dict = {}
+        for s in yaml.load(excuses_yaml)['sources']:
+            excuses_dict[s['source']] = s
+
        if 'SHOW_EXCUSES' in os.environ:
-            print('-------\nexcuses: %s\n-----' % excuses)
+            print('------- excuses -----')
+            pprint.pprint(excuses_dict, width=200)
+        if 'SHOW_HTML' in os.environ:
+            print('------- excuses.html -----\n%s\n' % excuses_html)
        if 'SHOW_OUTPUT' in os.environ:
-            print('-------\nout: %s\n-----' % out)
-        if considered:
-            self.assertIn('Valid candidate', excuses)
-        else:
-            self.assertIn('Not considered', excuses)
-
-        if excuses_expect:
-            for re in excuses_expect:
-                self.assertRegex(excuses, re, excuses)
-        if excuses_no_expect:
-            for re in excuses_no_expect:
-                self.assertNotRegex(excuses, re, excuses)
+            print('------- output -----\n%s\n' % out)
+
+        for src, (is_candidate, testmap) in expect_status.items():
+            self.assertEqual(excuses_dict[src]['is-candidate'], is_candidate)
+            for testsrc, archmap in testmap.items():
+                for arch, status in archmap.items():
+                    self.assertEqual(excuses_dict[src]['tests']['autopkgtest'][testsrc][arch][0],
+                                     status,
+                                     excuses_dict[src]['tests']['autopkgtest'][testsrc])
+
+        for src, matches in expect_excuses.items():
+            for k, v in matches:
+                if isinstance(excuses_dict[src][k], list):
+                    self.assertIn(v, excuses_dict[src][k])
+                else:
+                    self.assertEqual(excuses_dict[src][k], v)

        self.amqp_requests = set()
        try:
@ -111,19 +135,22 @@ class TestAutoPkgTest(TestBase):

        self.assertNotIn('FIXME', out)

-        return out
+        return (out, excuses_dict)

    def test_no_request_for_uninstallable(self):
        '''Does not request a test for an uninstallable package'''

-        self.do_test(
+        exc = self.do_test(
            # uninstallable unstable version
            [('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\blightgreen\b.*>1</a> to .*>1.1~beta<',
-             'lightgreen/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'],
-            # autopkgtest should not be triggered for uninstallable pkg
-            ['autopkgtest'])
+            {'lightgreen': (False, {})},
+            {'lightgreen': [('old-version', '1'), ('new-version', '1.1~beta'),
+                            ('reason', 'depends'),
+                            ('excuses', 'lightgreen/amd64 unsatisfiable Depends: libgreen1 (>= 2)')
+                           ]
+            })[1]
+        # autopkgtest should not be triggered for uninstallable pkg
+        self.assertEqual(exc['lightgreen']['tests'], {})

        self.assertEqual(self.pending_requests, '')
        self.assertEqual(self.amqp_requests, set())
@ -133,11 +160,12 @@ class TestAutoPkgTest(TestBase):

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+            {'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]})

        # we expect the package's and its reverse dependencies' tests to get
        # triggered
@ -158,7 +186,7 @@ lightgreen 1 i386 green 2
        self.assertEqual(self.pending_requests, expected_pending)

        # if we run britney again this should *not* trigger any new tests
-        self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1</a> to .*>2<'])
+        self.do_test([], {'green': (False, {})})
        self.assertEqual(self.amqp_requests, set())
        # but the set of pending tests doesn't change
        self.assertEqual(self.pending_requests, expected_pending)
@ -169,11 +197,12 @@ lightgreen 1 i386 green 2
        # first run requests tests and marks them as pending
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+            {'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]})

        # second run collects the results
        self.swift.set_results({'autopkgtest-series': {
@ -191,11 +220,13 @@ lightgreen 1 i386 green 2

        out = self.do_test(
            [],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]}
+        )[0]

        # all tests ran, there should be no more pending ones
        self.assertEqual(self.pending_requests, '')
@ -220,11 +251,11 @@ lightgreen 1 i386 green 2
        self.swift.set_results({})
        out = self.do_test(
            [],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             })
+            })[0]
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')
        self.assertNotIn('Failure', out, out)
@ -235,11 +266,12 @@ lightgreen 1 i386 green 2
        # first run requests tests and marks them as pending
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+            {'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]})

        # second run collects the results
        self.swift.set_results({'autopkgtest-series': {
@ -252,11 +284,11 @@ lightgreen 1 i386 green 2

        out = self.do_test(
            [],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass'])
+            {'green': (False, {'green 2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'PASS'},
+                              })
+            })

        # not expecting any failures to retrieve from swift
        self.assertNotIn('Failure', out, out)
@ -282,11 +314,13 @@ lightgreen 1 i386 green 2

        out = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'green': (False, {'green 2': {'amd64': 'REGRESSION', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]}
+        )[0]

        self.assertEqual(self.pending_requests, '')
        # not expecting any failures to retrieve from swift
@ -311,11 +345,13 @@ lightgreen 1 i386 green 2

        out = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'green': (False, {'green 2': {'amd64': 'REGRESSION', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]}
+        )[0]

        self.assertEqual(self.pending_requests, '')
        # not expecting any failures to retrieve from swift
@ -338,11 +374,13 @@ lightgreen 1 i386 green 2

        out = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'green': (True, {'green 2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'ALWAYSFAIL', 'i386': 'ALWAYSFAIL'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]}
+        )[0]

        self.assertEqual(self.pending_requests, '')
        # not expecting any failures to retrieve from swift
@ -352,12 +390,17 @@ lightgreen 1 i386 green 2
        '''Unbuilt package should not trigger tests or get considered'''

        self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'})
-        self.do_test(
+        exc = self.do_test(
+            # uninstallable unstable version
            [],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'missing build on.*amd64.*green, libgreen1'],
-            ['autopkgtest'])
+            {'green': (False, {})},
+            {'green': [('old-version', '1'), ('new-version', '2'),
+                       ('reason', 'no-binaries'),
+                       ('excuses', 'green has no up-to-date binaries on any arch')
+                      ]
+            })[1]
+        # autopkgtest should not be triggered for unbuilt pkg
+        self.assertEqual(exc['green']['tests'], {})

    def test_rdepends_unbuilt(self):
        '''Unbuilt reverse dependency'''
@ -372,22 +415,25 @@ lightgreen 1 i386 green 2
            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
            'series/i386/g/green/20150101_020000@': (0, 'green 1'),
            'series/amd64/g/green/20150101_020000@': (0, 'green 1'),
-            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
-            'series/amd64/g/green/20150101_100201@': (0, 'green 2'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 1.1'),
+            'series/amd64/g/green/20150101_100201@': (0, 'green 1.1'),
        }})

        # add unbuilt lightgreen; should run tests against the old version
        self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'})
        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
-             r'lightgreen has no up-to-date binaries on any arch'],
-            ['Valid candidate'])
+            [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (False, {'green 1.1': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              }),
+             'lightgreen': (False, {}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '1.1')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2'),
+                            ('excuses', 'lightgreen has no up-to-date binaries on any arch')]
+            }
+        )

        self.assertEqual(
            self.amqp_requests,
@ -397,7 +443,7 @@ lightgreen 1 i386 green 2
        self.assertEqual(self.pending_requests, '')

        # next run should not trigger any new requests
-        self.do_test([], NOT_CONSIDERED, [], ['Valid candidate'])
+        self.do_test([], {'green': (False, {}), 'lightgreen': (False, {})})
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')

@ -408,15 +454,19 @@ lightgreen 1 i386 green 2
        }})
        self.data.remove_all(True)
        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
+            [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
             ('lightgreen', {'Version': '2'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 2: .*amd64.*Pass.*i386.*Pass'],
-            ['Not considered'])
+            {'green': (True, {'green 1.1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              # FIXME: expecting a lightgreen test here
+                              # 'lightgreen 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+             'lightgreen': (True, {'lightgreen 2': {'amd64': 'PASS', 'i386': 'PASS'}}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '1.1')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2')],
+            }
+        )
        self.assertEqual(self.amqp_requests,
                         set(['debci-series-amd64:lightgreen', 'debci-series-i386:lightgreen']))
        self.assertEqual(self.pending_requests, '')
@ -437,19 +487,22 @@ lightgreen 1 i386 green 2
        # run britney once to pick up previous results
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE)
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'}})})

        # add new uninstallable brokengreen; should not run test at all
-        self.do_test(
+        exc = self.do_test(
            [('brokengreen', {'Version': '1', 'Depends': 'libgreen1, nonexisting'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\bbrokengreen\b.*- to .*>1<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             'Not considered',  # for brokengreen
-             r'brokengreen/amd64 unsatisfiable Depends: nonexisting'],
-            ['autopkgtest for brokengreen'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'}}),
+             'brokengreen': (False, {}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'brokengreen': [('old-version', '-'), ('new-version', '1'),
+                             ('reason', 'depends'),
+                             ('excuses', 'brokengreen/amd64 unsatisfiable Depends: nonexisting')],
+            })[1]
+        # autopkgtest should not be triggered for uninstallable pkg
+        self.assertEqual(exc['brokengreen']['tests'], {})
+
        self.assertEqual(self.amqp_requests, set())

    def test_rdepends_unbuilt_new_version_result(self):
@ -470,22 +523,25 @@ lightgreen 1 i386 green 2
            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
            'series/i386/g/green/20150101_020000@': (0, 'green 1'),
            'series/amd64/g/green/20150101_020000@': (0, 'green 1'),
-            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
-            'series/amd64/g/green/20150101_100201@': (0, 'green 2'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 1.1'),
+            'series/amd64/g/green/20150101_100201@': (0, 'green 1.1'),
        }})

        # add unbuilt lightgreen; should run tests against the old version
        self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'})
        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
-             r'lightgreen has no up-to-date binaries on any arch'],
-            ['Valid candidate'])
+            [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (False, {'green 1.1': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              }),
+             'lightgreen': (False, {}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '1.1')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2'),
+                            ('excuses', 'lightgreen has no up-to-date binaries on any arch')]
+            }
+        )
        self.assertEqual(
            self.amqp_requests,
            set(['debci-series-i386:green', 'debci-series-amd64:green',
@ -500,19 +556,22 @@ lightgreen 1 i386 green 2
        }})
        self.do_test(
            [],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 2.*: .*amd64.*Pass.*i386.*Pass',
-             r'lightgreen has no up-to-date binaries on any arch'])
-
+            {'green': (True, {'green 1.1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+             'lightgreen': (False, {}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '1.1')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2'),
+                            ('excuses', 'lightgreen has no up-to-date binaries on any arch')]
+            }
+        )
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')

        # next run should not trigger any new requests
-        self.do_test([], NOT_CONSIDERED, [])
+        self.do_test([], {'green': (True, {}), 'lightgreen': (False, {})})
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')

@ -523,14 +582,17 @@ lightgreen 1 i386 green 2
        self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'})
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'lightgreen has no up-to-date binaries on any arch'],
-            ['Valid candidate'])
+            {'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              }),
+             'lightgreen': (False, {}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2'),
+                            ('excuses', 'lightgreen has no up-to-date binaries on any arch')]
+            }
+        )
        self.assertEqual(
            self.amqp_requests,
            set(['debci-series-i386:green', 'debci-series-amd64:green',
@ -550,20 +612,22 @@ lightgreen 1 i386 green 2
        }})
        self.do_test(
            [],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression',
-             r'lightgreen has no up-to-date binaries on any arch'],
-            ['Valid candidate'])
-
+            {'green': (False, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              }),
+             'lightgreen': (False, {}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2'),
+                            ('excuses', 'lightgreen has no up-to-date binaries on any arch')]
+            }
+        )
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')

        # next run should not trigger any new requests
-        self.do_test([], NOT_CONSIDERED, [], ['Valid candidate'])
+        self.do_test([], {'green': (False, {}), 'lightgreen': (False, {})})
        self.assertEqual(self.pending_requests, '')
        self.assertEqual(self.amqp_requests, set())

@ -585,12 +649,15 @@ lightgreen 1 i386 green 2

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'Should wait for lightgreen 1 test, but forced by pitti'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2'),
+                       ('forced-reason', 'badtest lightgreen 1'),
+                       ('excuses', 'Should wait for lightgreen 1 test, but forced by pitti')]
+            })

    def test_hint_force_badtest_different_version(self):
        '''force-badtest hint with non-matching version'''
@ -608,14 +675,16 @@ lightgreen 1 i386 green 2

        self.create_hint('pitti', 'force-badtest lightgreen/0.1')

-        self.do_test(
+        exc = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'],
-            ['Should wait'])
+            {'green': (False, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              }),
+            },
+            {'green': [('reason', 'autopkgtest')]}
+        )[1]
+        self.assertNotIn('forced-reason', exc['green'])

    def test_hint_force_skiptest(self):
        '''force-skiptest hint'''
@ -624,26 +693,30 @@ lightgreen 1 i386 green 2

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'Should wait for.*tests.*green 2.*forced by pitti'])
+            {'green': (True, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                             }),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2'),
+                       ('forced-reason', 'skiptest'),
+                       ('excuses', 'Should wait for tests relating to green 2, but forced by pitti')]
+            })

    def test_hint_force_skiptest_different_version(self):
        '''force-skiptest hint with non-matching version'''

        self.create_hint('pitti', 'force-skiptest green/1')
-
-        self.do_test(
+        exc = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'],
-            ['Should wait'])
+            {'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              }),
+            },
+            {'green': [('reason', 'autopkgtest')]}
+        )[1]
+        self.assertNotIn('forced-reason', exc['green'])

    def test_package_pair_running(self):
        '''Two packages in unstable that need to go in together (running)'''
@ -651,9 +724,15 @@ lightgreen 1 i386 green 2
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
             ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<'])
+            {'green': (False, {'green 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              }),
+             'lightgreen': (False, {'lightgreen 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'}}),
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')],
+             'lightgreen': [('old-version', '1'), ('new-version', '2')],
+            })

        # we expect the package's and its reverse dependencies' tests to get
        # triggered; lightgreen should be triggered only once
@ -680,11 +759,12 @@ lightgreen 2 i386 lightgreen 2

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             r'autopkgtest for newgreen 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+            {'newgreen': (False, {'newgreen 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                                  'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                                  'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                                 }),
+            },
+            {'newgreen': [('old-version', '-'), ('new-version', '2')]})

        self.assertEqual(
            self.amqp_requests,
@ -714,11 +794,12 @@ newgreen 2 i386 newgreen 2

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             r'autopkgtest for newgreen 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'newgreen': (True, {'newgreen 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                                 'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                                 'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                                }),
+            },
+            {'newgreen': [('old-version', '-'), ('new-version', '2')]})

        self.assertEqual(
            self.amqp_requests,
@ -737,9 +818,7 @@ newgreen 2 i386 newgreen 2

        self.do_test(
            [('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bdarkgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for darkgreen 2: .*amd64.*in progress.*i386.*in progress'])
+            {'darkgreen': (False, {'darkgreen 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})})

        self.assertEqual(
            self.amqp_requests,
@ -756,9 +835,7 @@ newgreen 2 i386 newgreen 2
        }})
        self.do_test(
            [],
-            VALID_CANDIDATE,
-            [r'\bdarkgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for darkgreen 2: .*amd64.*Pass.*i386.*Pass'])
+            {'darkgreen': (True, {'darkgreen 2': {'amd64': 'PASS', 'i386': 'PASS'}})})
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')

@ -766,9 +843,7 @@ newgreen 2 i386 newgreen 2
        self.data.remove_all(True)
        self.do_test(
            [('darkgreen', {'Version': '3', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bdarkgreen\b.*>1</a> to .*>3<',
-             r'autopkgtest for darkgreen 3: .*amd64.*in progress.*i386.*in progress'])
+            {'darkgreen': (False, {'darkgreen 3': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})})
        self.assertEqual(
            self.amqp_requests,
            set(['debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
@ -791,10 +866,11 @@ newgreen 2 i386 newgreen 2

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\green\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+            })

        self.assertEqual(
            self.amqp_requests,
@ -807,11 +883,11 @@ newgreen 2 i386 newgreen 2
        # second run: new version re-triggers all tests
        self.do_test(
            [('libgreen1', {'Version': '3', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\green\b.*>1</a> to .*>3<',
-             r'autopkgtest for green 3: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+            {'green': (False, {'green 3': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'lightgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              }),
+            })

        self.assertEqual(
            self.amqp_requests,
@ -839,11 +915,12 @@ lightgreen 1 i386 green 3
            'series/amd64/l/lightgreen/20150101_100010@': (0, 'lightgreen 1'),
        }})
        self.do_test(
-            [], NOT_CONSIDERED,
-            [r'\green\b.*>1</a> to .*>3<',
-             r'autopkgtest for green 3: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+            [],
+            {'green': (False, {'green 3': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'darkgreen 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                              }),
+            })
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests,
                         'darkgreen 1 amd64 green 3\ndarkgreen 1 i386 green 3\n')
@ -854,11 +931,12 @@ lightgreen 1 i386 green 3
            'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 1'),
        }})
        self.do_test(
-            [], VALID_CANDIDATE,
-            [r'\green\b.*>1</a> to .*>3<',
-             r'autopkgtest for green 3: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            [],
+            {'green': (True, {'green 3': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+            })
        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, '')

@ -875,11 +953,7 @@ lightgreen 1 i386 green 3

        self.do_test(
            [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
-            ['in progress'])
-
+            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}})})
        self.assertEqual(self.pending_requests, '')

        # one more tmpfail result, should not confuse britney with None version
@ -888,10 +962,7 @@ lightgreen 1 i386 green 3
        }})
        self.do_test(
            [],
-            NOT_CONSIDERED,
-            [r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
-            ['in progress'])
+            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}})})
        with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f:
            contents = f.read()
        self.assertNotIn('null', contents)
@ -916,10 +987,11 @@ lightgreen 1 i386 green 3

        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
-             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
+            {'green': (False, {'green 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'lightgreen 1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              }),
+            })
        self.assertEqual(self.pending_requests, '')

        # re-running test manually succeeded (note: darkgreen result should be
@ -940,10 +1012,12 @@ lightgreen 1 i386 green 3
            'series/amd64/l/lightgreen/20150101_100201@': (0, 'lightgreen 1'),
        }})
        self.do_test(
-            [], VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass'])
+            [],
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'darkgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+            })
        self.assertEqual(self.pending_requests, '')

    def test_remove_from_unstable(self):
@ -965,11 +1039,10 @@ lightgreen 1 i386 green 3
        self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
             ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'])
+            {'green': (False, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                               'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                              }),
+            })
        self.assertEqual(self.pending_requests, '')

        # remove new lightgreen by resetting archive indexes, and re-adding
@ -993,13 +1066,13 @@ lightgreen 1 i386 green 3
        }})

        # next run should re-trigger lightgreen 1 to test against green/2
-        self.do_test(
+        exc = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass'],
-            ['lightgreen 2'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+            })[1]
+        self.assertNotIn('lightgreen 2', exc['green']['tests']['autopkgtest'])

        # should not trigger new requests
        self.assertEqual(self.pending_requests, '')
@ -1009,11 +1082,10 @@ lightgreen 1 i386 green 3
        # but the next run should not trigger anything new
        self.do_test(
            [],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
-             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass'],
-            ['lightgreen 2'])
+            {'green': (True, {'green 2': {'amd64': 'PASS', 'i386': 'PASS'},
+                              'lightgreen 1': {'amd64': 'PASS', 'i386': 'PASS'},
+                             }),
+            })
        self.assertEqual(self.pending_requests, '')
        self.assertEqual(self.amqp_requests, set())

@ -1025,10 +1097,12 @@ lightgreen 1 i386 green 3

        self.do_test(
            [('lightgreen', {'Version': '2'}, 'autopkgtest')],
-            NOT_CONSIDERED,
-            [r'\blightgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for lightgreen 2: .*amd64.*in progress.*i386.*in progress',
-             r'autopkgtest for rainbow 1: .*amd64.*in progress.*i386.*in progress'])
+            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                                    'rainbow 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'},
+                                   }),
+            },
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]}
+        )

    def test_dkms(self):
        '''DKMS packages are autopkgtested (via autodep8)'''
@ -1038,9 +1112,8 @@ lightgreen 1 i386 green 3

        self.do_test(
            [('dkms', {'Version': '2'}, None)],
-            NOT_CONSIDERED,
-            [r'\bdkms\b.*>1</a> to .*>2<',
-             r'autopkgtest for fancy 1: .*amd64.*in progress.*i386.*in progress'])
+            {'dkms': (False, {'fancy 1': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})},
+            {'dkms': [('old-version', '1'), ('new-version', '2')]})

    def test_disable_adt(self):
        '''Run without autopkgtest requests'''
@ -1053,10 +1126,11 @@ lightgreen 1 i386 green 3
            elif not line.startswith('ADT_AMQP') and not line.startswith('ADT_SWIFT_URL'):
                sys.stdout.write(line)

-        self.do_test(
+        exc = self.do_test(
            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<'], ['autopkgtest'])
+            {'green': (True, {})},
+            {'green': [('old-version', '1'), ('new-version', '2')]})[1]
+        self.assertNotIn('autopkgtest', exc['green']['tests'])

        self.assertEqual(self.amqp_requests, set())
        self.assertEqual(self.pending_requests, None)
--- a/tests/test_boottest.py
+++ b/tests/test_boottest.py
@ -239,7 +239,7 @@ args.func()
        for (pkg, fields) in context:
            self.data.add(pkg, True, fields, testsuite='autopkgtest')
        self.make_boottest()
-        (excuses, out) = self.run_britney()
+        (excuses_yaml, excuses, out) = self.run_britney()
        # print('-------\nexcuses: %s\n-----' % excuses)
        # print('-------\nout: %s\n-----' % out)
        if expect: