Switch autopkgtest evaluation to cloud results

- Change AutoPackageTest.results() to evaluate the Swift results instead of
   the adt-britney ones.
 - Drop TestAdtBritney tests which now fail as we switched results evaluation
   to swift. Port relevant tests to TestAutoPkgTest.
 - Drop obsolete adt-britney autopkgtest code.
 - Adjust TestBoottestEnd2End.test_with_adt() for cloud results.

autopkgtest.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013 Canonical Ltd.
-# Author: Colin Watson <cjwatson@ubuntu.com>
+# Authors:
-# Partly based on code in auto-package-testing by
+#   Colin Watson <cjwatson@ubuntu.com>
-# Jean-Baptiste Lallement <jean-baptiste.lallement@canonical.com>
+#   Jean-Baptiste Lallement <jean-baptiste.lallement@canonical.com>
+#   Martin Pitt <martin.pitt@ubuntu.com>
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,12 +18,7 @@
 
 from __future__ import print_function
 
-from collections import defaultdict
-from contextlib import closing
 import os
-import subprocess
-import tempfile
-from textwrap import dedent
 import time
 import json
 import tarfile
@@ -37,9 +33,6 @@ import kombu
 from consts import (AUTOPKGTEST, BINARIES, DEPENDS, RDEPENDS, SOURCE, VERSION)
 
 
-adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney")
-
-ADT_PASS = ["PASS", "ALWAYSFAIL"]
 ADT_EXCUSES_LABELS = {
     "PASS": '<span style="background:#87d96c">Pass</span>',
     "ALWAYSFAIL": '<span style="background:#e5c545">Always failed</span>',
@@ -84,8 +77,6 @@ class AutoPackageTest(object):
         self.distribution = distribution
         self.series = series
         self.debug = debug
-        self.read()
-        self.rc_path = None  # for adt-britney, obsolete
         self.test_state_dir = os.path.join(britney.options.unstable,
                                            'autopkgtest')
         # map of requested tests from request()
@@ -397,92 +388,6 @@ class AutoPackageTest(object):
                                 result.add((src, arch))
         return result
 
-    #
-    # obsolete adt-britney helpers
-    #
-
-    def _ensure_rc_file(self):
-        if self.rc_path:
-            return
-        self.rc_path = os.path.expanduser(
-            "~/proposed-migration/autopkgtest/rc.%s" % self.series)
-        with open(self.rc_path, "w") as rc_file:
-            home = os.path.expanduser("~")
-            print(dedent("""\
-                release: %s
-                aptroot: ~/.chdist/%s-proposed-amd64/
-                apturi: file:%s/mirror/%s
-                components: main restricted universe multiverse
-                rsync_host: rsync://tachash.ubuntu-ci/adt/
-                datadir: ~/proposed-migration/autopkgtest/data""" %
-                         (self.series, self.series, home, self.distribution)),
-                         file=rc_file)
-
-    @property
-    def _request_path(self):
-        return os.path.expanduser(
-            "~/proposed-migration/autopkgtest/work/adt.request.%s" %
-            self.series)
-
-    @property
-    def _result_path(self):
-        return os.path.expanduser(
-            "~/proposed-migration/autopkgtest/work/adt.result.%s" %
-            self.series)
-
-    def _parse(self, path):
-        if os.path.exists(path):
-            with open(path) as f:
-                for line in f:
-                    line = line.strip()
-                    if line.startswith("Suite:") or line.startswith("Date:"):
-                        continue
-                    linebits = line.split()
-                    if len(linebits) < 2:
-                        print("W: Invalid line format: '%s', skipped" % line)
-                        continue
-                    yield linebits
-
-    def read(self):
-        '''Loads a list of results
-
-        This function loads a list of results returned by __parse() and builds
-        2 lists:
-            - a list of source package/version with all the causes that
-            triggered a test and the result of the test for this trigger.
-            - a list of packages/version that triggered a test with the source
-            package/version and result triggered by this package.
-        These lists will be used in result() called from britney.py to generate
-        excuses and now which uploads passed, caused regression or which tests
-        have always been failing
-        '''
-        self.pkglist = defaultdict(dict)
-        self.pkgcauses = defaultdict(lambda: defaultdict(list))
-        for linebits in self._parse(self._result_path):
-            (src, ver, status) = linebits[:3]
-
-            if not (src in self.pkglist and ver in self.pkglist[src]):
-                self.pkglist[src][ver] = {
-                    "status": status,
-                    "causes": {}
-                }
-
-            i = iter(linebits[3:])
-            for trigsrc, trigver in zip(i, i):
-                self.pkglist[src][ver]['causes'].setdefault(
-                    trigsrc, []).append((trigver, status))
-                self.pkgcauses[trigsrc][trigver].append((status, src, ver))
-
-    def _adt_britney(self, *args):
-        command = [
-            adt_britney,
-            "-c", self.rc_path, "-r", self.series, "-PU",
-            ]
-        if self.debug:
-            command.append("-d")
-        command.extend(args)
-        subprocess.check_call(command)
-
     #
     # Public API
     #
@@ -506,60 +411,6 @@ class AutoPackageTest(object):
                         self.log_verbose('Requesting %s/%s/%s autopkgtest to verify %s' %
                                          (src, ver, arch, ', '.join(['%s/%s' % i for i in triggers])))
 
-        # deprecated requests for old Jenkins/lp:auto-package-testing, will go
-        # away
-
-        self._ensure_rc_file()
-        request_path = self._request_path
-        if os.path.exists(request_path):
-            os.unlink(request_path)
-        with closing(tempfile.NamedTemporaryFile(mode="w")) as request_file:
-            for src, ver in packages:
-                if src in self.pkglist and ver in self.pkglist[src]:
-                    continue
-                print("%s %s" % (src, ver), file=request_file)
-            request_file.flush()
-            self._adt_britney("request", "-O", request_path, request_file.name)
-
-        # Remove packages that have been identified as invalid candidates for
-        # testing from the request file i.e run_autopkgtest = False
-        with open(request_path, 'r') as request_file:
-            lines = request_file.readlines()
-        with open(request_path, 'w') as request_file:
-            for line in lines:
-                src = line.split()[0]
-                if src not in excludes:
-                    request_file.write(line)
-                else:
-                    if self.britney.options.verbose:
-                        self.log_verbose("Requested autopkgtest for %s but "
-                                         "run_autopkgtest set to False" % src)
-
-        for linebits in self._parse(request_path):
-            # Make sure that there's an entry in pkgcauses for each new
-            # request, so that results() gives useful information without
-            # relying on the submit/collect cycle.  This improves behaviour
-            # in dry-run mode.
-            src = linebits.pop(0)
-            ver = linebits.pop(0)
-            if self.britney.options.verbose:
-                self.log_verbose("Requested autopkgtest for %s_%s (%s)" %
-                                 (src, ver, " ".join(linebits)))
-            try:
-                status = linebits.pop(0).upper()
-                while True:
-                    trigsrc = linebits.pop(0)
-                    trigver = linebits.pop(0)
-                    for status, csrc, cver in self.pkgcauses[trigsrc][trigver]:
-                        if csrc == trigsrc and cver == trigver:
-                            break
-                    else:
-                        self.pkgcauses[trigsrc][trigver].append(
-                            (status, src, ver))
-            except IndexError:
-                # End of the list
-                pass
-
     def submit(self):
         # send AMQP requests for new test requests
         # TODO: Once we support version constraints in AMQP requests, add them
@@ -601,89 +452,81 @@ class AutoPackageTest(object):
         # mark them as pending now
         self.update_pending_tests()
 
-        # deprecated requests for old Jenkins/lp:auto-package-testing, will go
-        # away
-        self._ensure_rc_file()
-        request_path = self._request_path
-        if os.path.exists(request_path):
-            self._adt_britney("submit", request_path)
-
     def collect(self, packages):
         # fetch results from swift
         try:
             swift_url = self.britney.options.adt_swift_url
         except AttributeError:
             self.log_error('ADT_SWIFT_URL not set, cannot collect results')
-            swift_url = None
+            return
         try:
             self.britney.options.adt_amqp
         except AttributeError:
             self.log_error('ADT_AMQP not set, not collecting results from swift')
-            swift_url = None
+            return
-
+
-        if swift_url:
+        # update results from swift for all packages that we are waiting
-            # update results from swift for all packages that we are waiting
+        # for, and remove pending tests that we have results for on all
-            # for, and remove pending tests that we have results for on all
+        # arches
-            # arches
+        for pkg, verinfo in copy.deepcopy(self.pending_tests.items()):
-            for pkg, verinfo in copy.deepcopy(self.pending_tests.items()):
+            for archinfo in verinfo.values():
-                for archinfo in verinfo.values():
+                for arch in archinfo:
-                    for arch in archinfo:
+                    self.fetch_swift_results(swift_url, pkg, arch)
-                        self.fetch_swift_results(swift_url, pkg, arch)
+        # also update results for excuses whose tests failed, in case a
-            # also update results for excuses whose tests failed, in case a
+        # manual retry worked
-            # manual retry worked
+        for (trigpkg, trigver) in packages:
-            for (trigpkg, trigver) in packages:
+            if trigpkg not in self.pending_tests:
-                if trigpkg not in self.pending_tests:
+                for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver):
-                    for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver):
+                    self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' %
-                        self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' %
+                                     (pkg, arch, trigpkg, trigver))
-                                         (pkg, arch, trigpkg, trigver))
+                    self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver))
-                        self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver))
+
-
+        # update the results cache
-            # update the results cache
+        with open(self.results_cache_file + '.new', 'w') as f:
-            with open(self.results_cache_file + '.new', 'w') as f:
+            json.dump(self.test_results, f, indent=2)
-                json.dump(self.test_results, f, indent=2)
+        os.rename(self.results_cache_file + '.new', self.results_cache_file)
-            os.rename(self.results_cache_file + '.new', self.results_cache_file)
+        self.log_verbose('Updated results cache')
-            self.log_verbose('Updated results cache')
+
-
+        # new results remove pending requests, update the on-disk cache
-            # new results remove pending requests, update the on-disk cache
+        self.update_pending_tests()
-            self.update_pending_tests()
-
-        # deprecated results for old Jenkins/lp:auto-package-testing, will go
-        # away
-        self._ensure_rc_file()
-        result_path = self._result_path
-        self._adt_britney("collect", "-O", result_path)
-        self.read()
-        if self.britney.options.verbose:
-            for src in sorted(self.pkglist):
-                for ver in sorted(self.pkglist[src],
-                                  cmp=apt_pkg.version_compare):
-                    for trigsrc in sorted(self.pkglist[src][ver]['causes']):
-                        for trigver, status \
-                                in self.pkglist[src][ver]['causes'][trigsrc]:
-                            self.log_verbose("Collected autopkgtest status "
-                                             "for %s_%s/%s_%s: " "%s" %
-                                             (src, ver, trigsrc, trigver, status))
 
     def results(self, trigsrc, trigver):
         '''Return test results for triggering package
 
-        Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests
+        Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING)
-        that got triggered by trigsrc/trigver.
+        iterator for all package tests that got triggered by trigsrc/trigver.
         '''
-        # deprecated results for old Jenkins/lp:auto-package-testing, will go
+        for testsrc, testver in self.tests_for_source(trigsrc, trigver):
-        # away
+            passed = True
-        for status, src, ver in self.pkgcauses[trigsrc][trigver]:
+            arch_status = {}
-            # Check for regression
+            for arch in self.britney.options.adt_arches.split():
-            if status == 'FAIL':
+                try:
-                passed_once = False
+                    (_, ver_map, ever_passed) = self.test_results[testsrc][arch]
-                for lver in self.pkglist[src]:
+                    (status, triggers) = ver_map[testver]
-                    for trigsrc in self.pkglist[src][lver]['causes']:
+                    # triggers might contain tuples or lists
-                        for trigver, status \
+                    if (trigsrc, trigver) not in triggers and [trigsrc, trigver] not in triggers:
-                                in self.pkglist[src][lver]['causes'][trigsrc]:
+                        raise KeyError('No result for trigger %s/%s yet' % (trigsrc, trigver))
-                            if status == 'PASS':
+                    if status:
-                                passed_once = True
+                        arch_status[arch] = 'PASS'
-                if not passed_once:
+                    else:
-                    status = 'ALWAYSFAIL'
+                        # test failed, check ever_passed flag for that src/arch
-                else:
+                        if ever_passed:
-                    status = 'REGRESSION'
+                            arch_status[arch] = 'REGRESSION'
-            yield status, src, ver
+                            passed = False
+                        else:
+                            arch_status[arch] = 'ALWAYSFAIL'
+                except KeyError:
+                    # no result for testsrc/testver/arch; still running?
+                    try:
+                        self.pending_tests[testsrc][testver][arch]
+                        arch_status[arch] = 'RUNNING'
+                        passed = False
+                    except KeyError:
+                        # neither done nor pending -> exclusion, or disabled
+                        continue
+
+            # disabled or ignored?
+            if not arch_status:
+                continue
+
+            yield (passed, testsrc, testver, arch_status)

britney.py

@@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes,
 from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
                    SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
                    PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL)
-from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash
+from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash
 from boottest import BootTest
 
 
@@ -1849,30 +1849,21 @@ class Britney(object):
             if not self.options.dry_run:
                 autopkgtest.submit()
                 autopkgtest.collect(autopkgtest_packages)
-            jenkins_public = "https://jenkins.qa.ubuntu.com/job"
-            jenkins_private = (
-                "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %
-                self.options.series.title())
             cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s"
             for e in autopkgtest_excuses:
                 adtpass = True
-                adt_jenkins_sources = set()  # temporary: drop when switching to cloud based tests
+                for passed, adtsrc, adtver, arch_status in autopkgtest.results(
-                for status, adtsrc, adtver in autopkgtest.results(
                         e.name, e.ver[1]):
-                    adt_jenkins_sources.add(adtsrc)
+                    archmsg = []
-                    public_url = "%s/%s-adt-%s/lastBuild" % (
+                    for arch in sorted(arch_status):
-                        jenkins_public, self.options.series,
+                        url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc,
-                        adtsrc.replace("+", "-"))
+                                           'r': self.options.series, 'a': arch}
-                    private_url = "%s/%s-adt-%s/lastBuild" % (
+                        archmsg.append('<a href="%s">%s: %s</a>' %
-                        jenkins_private, self.options.series,
+                                       (url, arch, ADT_EXCUSES_LABELS[arch_status[arch]]))
-                        adtsrc.replace("+", "-"))
+                    e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg)))
-                    adt_label = ADT_EXCUSES_LABELS.get(status, status)
+
-                    e.addhtml(
+                    # hints can override failures
-                        "autopkgtest for %s %s: %s (Jenkins: "
+                    if not passed:
-                        "<a href=\"%s\">public</a>, "
-                        "<a href=\"%s\">private</a>)" %
-                        (adtsrc, adtver, adt_label, public_url, private_url))
-                    if status not in ADT_PASS:
                         hints = self.hints.search(
                             'force-badtest', package=adtsrc)
                         hints.extend(
@@ -1884,54 +1875,10 @@ class Britney(object):
                             e.addhtml(
                                 "Should wait for %s %s test, but forced by "
                                 "%s" % (adtsrc, adtver, forces[0].user))
-                        else:
+                            passed = True
-                            adtpass = False
-
-                # temporary: also show results from cloud based tests,
-                # until that becomes the primary mechanism
-                adt_cloud_sources = set()
-                for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]):
-                    adt_cloud_sources.add(testsrc)
-                    msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver)
-                    archmsg = []
-                    for arch in self.options.adt_arches.split():
-                        url = cloud_url % {'h': srchash(testsrc), 's': testsrc,
-                                           'r': self.options.series, 'a': arch}
-                        try:
-                            (_, ver_map, ever_passed) = autopkgtest.test_results[testsrc][arch]
-                            (passed, triggers) = ver_map[testver]
-                            # triggers might contain tuples or lists
-                            if (e.name, e.ver[1]) not in triggers and [e.name, e.ver[1]] not in triggers:
-                                raise KeyError('No result for trigger %s/%s yet' % (e.name, e.ver[1]))
-
-                            if passed:
-                                status = 'PASS'
-                            else:
-                                if ever_passed:
-                                    status = 'REGRESSION'
-                                else:
-                                    status = 'ALWAYSFAIL'
-                        except KeyError:
-                            try:
-                                autopkgtest.pending_tests[testsrc][testver][arch]
-                                status = 'RUNNING'
-                            except KeyError:
-                                # neither done nor pending -> exclusion, or disabled
-                                continue
 
-                        archmsg.append('<a href="%s">%s: %s</a>' %
+                    if not passed:
-                                       (url, arch, ADT_EXCUSES_LABELS[status]))
+                        adtpass = False
-
-                    if archmsg:
-                        e.addhtml(msg + ', '.join(archmsg))
-
-                extra_sources = adt_cloud_sources - adt_jenkins_sources
-                missing_sources = adt_jenkins_sources - adt_cloud_sources
-                if missing_sources:
-                    e.addhtml('(informational) Missing sources in cloud tests: %s' % ' '.join(missing_sources))
-                if extra_sources:
-                    e.addhtml('(informational) Extra sources in cloud tests: %s' % ' '.join(extra_sources))
-                # end of temporary code
 
                 if not adtpass and e.is_valid:
                     hints = self.hints.search('force-skiptest', package=e.name)

tests/__init__.py

@@ -172,3 +172,11 @@ class TestBase(unittest.TestCase):
             excuses = f.read()
 
         return (excuses, out)
+
+    def create_hint(self, username, content):
+        '''Create a hint file for the given username and content'''
+
+        hints_path = os.path.join(
+            self.data.path, 'data', self.data.series + '-proposed', 'Hints', username)
+        with open(hints_path, 'w') as fd:
+            fd.write(content)

tests/test_autopkgtest.py

@@ -7,10 +7,8 @@
 # (at your option) any later version.
 
 import apt_pkg
-import operator
 import os
 import sys
-import subprocess
 import fileinput
 import unittest
 import json
@@ -18,7 +16,6 @@ import json
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, PROJECT_DIR)
 
-from autopkgtest import ADT_EXCUSES_LABELS
 from tests import TestBase, mock_swift
 
 NOT_CONSIDERED = False
@@ -46,16 +43,6 @@ class TestAutoPkgTest(TestBase):
             else:
                 sys.stdout.write(line)
 
-        # fake adt-britney script; necessary until we drop that code
-        self.adt_britney = os.path.join(
-            self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney')
-        os.makedirs(os.path.dirname(self.adt_britney))
-        with open(self.adt_britney, 'w') as f:
-            f.write('''#!/bin/sh -e
-touch $HOME/proposed-migration/autopkgtest/work/adt.request.series
-echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
-        os.chmod(self.adt_britney, 0o755)
-
         # add a bunch of packages to testing to avoid repetition
         self.data.add('libc6', False)
         self.data.add('libgreen1', False, {'Source': 'green',
@@ -89,7 +76,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
         (excuses, out) = self.run_britney()
         self.swift.stop()
 
-        #print('-------\nexcuses: %s\n-----' % excuses)
+        if 'SHOW_EXCUSES' in os.environ:
+            print('-------\nexcuses: %s\n-----' % excuses)
         if 'SHOW_OUTPUT' in os.environ:
             print('-------\nout: %s\n-----' % out)
         if considered:
@@ -120,14 +108,27 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
 
         return out
 
+    def test_no_request_for_uninstallable(self):
+        '''Does not request a test for an uninstallable package'''
+
+        self.do_test(
+            # uninstallable unstable version
+            [('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\blightgreen\b.*>1</a> to .*>1.1~beta<',
+             'lightgreen/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'],
+            # autopkgtest should not be triggered for uninstallable pkg
+            ['autopkgtest'])
+
+        self.assertEqual(self.pending_requests, '')
+        self.assertEqual(self.amqp_requests, set())
+
     def test_multi_rdepends_with_tests_all_running(self):
         '''Multiple reverse dependencies with tests (all running)'''
 
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -153,7 +154,7 @@ lightgreen 1 i386 green 2
         self.assertEqual(self.pending_requests, expected_pending)
 
         # if we run britney again this should *not* trigger any new tests
-        self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1</a> to .*>2<'])
+        self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1</a> to .*>2<'])
         self.assertEqual(self.amqp_requests, set())
         # but the set of pending tests doesn't change
         self.assertEqual(self.pending_requests, expected_pending)
@@ -164,9 +165,7 @@ lightgreen 1 i386 green 2
         # first run requests tests and marks them as pending
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -233,9 +232,7 @@ lightgreen 1 i386 green 2
         # first run requests tests and marks them as pending
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -252,9 +249,7 @@ lightgreen 1 i386 green 2
 
         out = self.do_test(
             [],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
              r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
@@ -267,15 +262,174 @@ lightgreen 1 i386 green 2
         self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
         self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
 
+    def test_multi_rdepends_with_tests_regression(self):
+        '''Multiple reverse dependencies with tests (regression)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+    def test_multi_rdepends_with_tests_regression_last_pass(self):
+        '''Multiple reverse dependencies with tests (regression), last one passes
+
+        This ensures that we don't just evaluate the test result of the last
+        test, but all of them.
+        '''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+    def test_multi_rdepends_with_tests_always_failed(self):
+        '''Multiple reverse dependencies with tests (always failed)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (4, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
+        }})
+
+        out = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(self.pending_requests, '')
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+    def test_hint_force_badtest(self):
+        '''force-badtest hint'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
+        }})
+
+        self.create_hint('pitti', 'force-badtest lightgreen/1')
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'Should wait for lightgreen 1 test, but forced by pitti'])
+
+    def test_hint_force_badtest_different_version(self):
+        '''force-badtest hint with non-matching version'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
+        }})
+
+        self.create_hint('pitti', 'force-badtest lightgreen/0.1')
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'],
+            ['Should wait'])
+
+    def test_hint_force_skiptest(self):
+        '''force-skiptest hint'''
+
+        self.create_hint('pitti', 'force-skiptest green/2')
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'Should wait for.*tests.*green 2.*forced by pitti'])
+
+    def test_hint_force_skiptest_different_version(self):
+        '''force-skiptest hint with non-matching version'''
+
+        self.create_hint('pitti', 'force-skiptest green/1')
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'],
+            ['Should wait'])
+
     def test_package_pair_running(self):
         '''Two packages in unstable that need to go in together (running)'''
 
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
              ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'\blightgreen\b.*>1</a> to .*>2<'])
 
@@ -300,6 +454,58 @@ lightgreen 2 i386 lightgreen 2
 '''
         self.assertEqual(self.pending_requests, expected_pending)
 
+    def test_binary_from_new_source_package_running(self):
+        '''building an existing binary for a new source package (running)'''
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bnewgreen\b.*\(- to .*>2<',
+             r'autopkgtest for newgreen 2: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
+             r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
+
+        self.assertEqual(
+            self.amqp_requests,
+            set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen',
+                 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
+                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
+        expected_pending = '''darkgreen 1 amd64 newgreen 2
+darkgreen 1 i386 newgreen 2
+lightgreen 1 amd64 newgreen 2
+lightgreen 1 i386 newgreen 2
+newgreen 2 amd64 newgreen 2
+newgreen 2 i386 newgreen 2
+'''
+        self.assertEqual(self.pending_requests, expected_pending)
+
+    def test_binary_from_new_source_package_pass(self):
+        '''building an existing binary for a new source package (pass)'''
+
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
+            'series/i386/n/newgreen/20150101_100200@': (0, 'newgreen 2'),
+            'series/amd64/n/newgreen/20150101_100201@': (0, 'newgreen 2'),
+        }})
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')],
+            VALID_CANDIDATE,
+            [r'\bnewgreen\b.*\(- to .*>2<',
+             r'autopkgtest for newgreen 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
+
+        self.assertEqual(
+            self.amqp_requests,
+            set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen',
+                 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
+                 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
+        self.assertEqual(self.pending_requests, '')
+
     def test_result_from_older_version(self):
         '''test result from older version than the uploaded one'''
 
@@ -310,9 +516,7 @@ lightgreen 2 i386 lightgreen 2
 
         self.do_test(
             [('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bdarkgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for darkgreen 2: .*amd64.*in progress.*i386.*in progress'])
 
@@ -342,9 +546,7 @@ lightgreen 2 i386 lightgreen 2
         self.data.remove_all(True)
         self.do_test(
             [('darkgreen', {'Version': '3', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bdarkgreen\b.*>1</a> to .*>3<',
              r'autopkgtest for darkgreen 3: .*amd64.*in progress.*i386.*in progress'])
         self.assertEqual(
@@ -386,9 +588,7 @@ lightgreen 2 i386 lightgreen 2
         # second run: new version re-triggers all tests
         self.do_test(
             [('libgreen1', {'Version': '3', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\green\b.*>1</a> to .*>3<',
              r'autopkgtest for green 3: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@@ -421,10 +621,7 @@ lightgreen 1 i386 green 3
             'series/amd64/l/lightgreen/20150101_100010@': (0, 'lightgreen 1'),
         }})
         self.do_test(
-            [],
+            [], NOT_CONSIDERED,
-            # FIXME: while we only submit requests through AMQP, but don't consider
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\green\b.*>1</a> to .*>3<',
              r'autopkgtest for green 3: .*amd64.*Pass.*i386.*Pass',
              r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
@@ -460,9 +657,7 @@ lightgreen 1 i386 green 3
 
         self.do_test(
             [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\blightgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
             ['in progress'])
@@ -488,9 +683,7 @@ lightgreen 1 i386 green 3
 
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
              r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
@@ -539,9 +732,7 @@ lightgreen 1 i386 green 3
         self.do_test(
             [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
              ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bgreen\b.*>1</a> to .*>2<',
              r'\blightgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
@@ -603,9 +794,7 @@ lightgreen 1 i386 green 3
 
         self.do_test(
             [('lightgreen', {'Version': '2'}, 'autopkgtest')],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\blightgreen\b.*>1</a> to .*>2<',
              r'autopkgtest for lightgreen 2: .*amd64.*in progress.*i386.*in progress',
              r'autopkgtest for rainbow 1: .*amd64.*in progress.*i386.*in progress'])
@@ -618,9 +807,7 @@ lightgreen 1 i386 green 3
 
         self.do_test(
             [('dkms', {'Version': '2'}, None)],
-            # FIXME: while we only submit requests through AMQP, but don't consider
+            NOT_CONSIDERED,
-            # their results, we don't expect this to hold back stuff.
-            VALID_CANDIDATE,
             [r'\bdkms\b.*>1</a> to .*>2<',
              r'autopkgtest for fancy 1: .*amd64.*in progress.*i386.*in progress'])
 
@@ -641,448 +828,5 @@ lightgreen 1 i386 green 3
         self.assertEqual(self.pending_requests, None)
 
 
-class TestAdtBritney(TestBase):
-    '''Legacy adt-britney/lp:auto-package-testing interface'''
-
-    def setUp(self):
-        super(TestAdtBritney, self).setUp()
-
-        # Mofify configuration according to the test context.
-        with open(self.britney_conf, 'r') as fp:
-            original_config = fp.read()
-        # Disable boottests.
-        new_config = original_config.replace(
-            'BOOTTEST_ENABLE   = yes', 'BOOTTEST_ENABLE   = no')
-        with open(self.britney_conf, 'w') as fp:
-            fp.write(new_config)
-
-        # fake adt-britney script
-        self.adt_britney = os.path.join(
-            self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney')
-        os.makedirs(os.path.dirname(self.adt_britney))
-
-        with open(self.adt_britney, 'w') as f:
-            f.write('''#!/bin/sh -e
-echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
-        os.chmod(self.adt_britney, 0o755)
-
-        # add a bunch of packages to testing to avoid repetition
-        self.data.add('libc6', False)
-        self.data.add('libgreen1', False, {'Source': 'green',
-                                           'Depends': 'libc6 (>= 0.9)'})
-        self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1',
-                                       'Conflicts': 'blue'})
-        self.data.add('lightgreen', False, {'Depends': 'libgreen1'})
-        self.data.add('darkgreen', False, {'Depends': 'libgreen1'})
-        self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)',
-                                      'Conflicts': 'green'})
-        self.data.add('justdata', False, {'Architecture': 'all'})
-
-    def __merge_records(self, results, history=""):
-        '''Merges a list of results with records in history.
-
-        This function merges results from a collect with records already in
-        history and sort records by version/name of causes and version/name of
-        source packages with tests. This should be done in the fake
-        adt-britney but it is more convenient to just pass a static list of
-        records and make adt-britney just return this list.
-        '''
-
-        if history is None:
-            history = ""
-        records = [x.split() for x in (results.strip() + '\n' +
-                                       history.strip()).split('\n') if x]
-
-        records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(4))
-        records.sort(key=operator.itemgetter(3))
-        records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(1))
-        records.sort()
-
-        return "\n".join([' '.join(x) for x in records])
-
-    def make_adt_britney(self, request, history=""):
-        with open(self.adt_britney, 'w') as f:
-            f.write('''#!%(py)s
-import argparse, shutil,sys
-
-def request():
-    if args.req:
-        shutil.copy(args.req, '%(path)s/adt-britney.requestarg')
-    with open(args.output, 'w') as f:
-        f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW').replace('RUNNING', 'NEW'))
-
-def submit():
-    with open(args.req, 'w') as f:
-        f.write("""%(rq)s""".replace('PASS', 'RUNNING').
-                    replace('FAIL', 'RUNNING'))
-
-def collect():
-    with open(args.output, 'w') as f:
-        f.write("""%(res)s""")
-
-p = argparse.ArgumentParser()
-p.add_argument('-c', '--config')
-p.add_argument('-a', '--arch')
-p.add_argument('-r', '--release')
-p.add_argument('-P', '--use-proposed', action='store_true')
-p.add_argument('-d', '--debug', action='store_true')
-p.add_argument('-U', '--no-update', action='store_true')
-sp = p.add_subparsers()
-
-prequest = sp.add_parser('request')
-prequest.add_argument('-O', '--output')
-prequest.add_argument('req', nargs='?')
-prequest.set_defaults(func=request)
-
-psubmit = sp.add_parser('submit')
-psubmit.add_argument('req')
-psubmit.set_defaults(func=submit)
-
-pcollect = sp.add_parser('collect')
-pcollect.add_argument('-O', '--output')
-pcollect.add_argument('-n', '--new-only', action='store_true', default=False)
-pcollect.set_defaults(func=collect)
-
-args = p.parse_args()
-args.func()
-                    ''' % {'py': sys.executable, 'path': self.data.path,
-                           'rq': request,
-                           'res': self.__merge_records(request, history)})
-
-    def do_test(self, unstable_add, adt_request, considered, expect=None,
-                no_expect=None, history=""):
-        for (pkg, fields) in unstable_add:
-            self.data.add(pkg, True, fields)
-
-        self.make_adt_britney(adt_request, history)
-
-        (excuses, out) = self.run_britney()
-        #print('-------\nexcuses: %s\n-----' % excuses)
-        #print('-------\nout: %s\n-----' % out)
-        #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf))
-        #subprocess.call(['bash', '-i'], cwd=self.data.path)
-        if considered:
-            self.assertIn('Valid candidate', excuses)
-        else:
-            self.assertIn('Not considered', excuses)
-
-        if expect:
-            for re in expect:
-                self.assertRegexpMatches(excuses, re)
-        if no_expect:
-            for re in no_expect:
-                self.assertNotRegexpMatches(excuses, re)
-
-    def test_no_request_for_uninstallable(self):
-        '''Does not request a test for an uninstallable package'''
-
-        self.do_test(
-            # uninstallable unstable version
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'})],
-            'green 1.1~beta RUNNING green 1.1~beta\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             'green/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'],
-            # autopkgtest should not be triggered for uninstallable pkg
-            ['autopkgtest'])
-
-    def test_request_for_installable_running(self):
-        '''Requests a test for an installable package, test still running'''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'green 1.1~beta RUNNING green 1.1~beta\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']])
-
-    def test_request_for_installable_first_fail(self):
-        '''Requests a test for an installable package. No history and first result is a failure'''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'green 1.1~beta FAIL green 1.1~beta\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
-
-    def test_request_for_installable_fail_regression(self):
-        '''Requests a test for an installable package, test fail'''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'green 1.1~beta FAIL green 1.1~beta\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
-            history='green 1.0~beta PASS green 1.0~beta\n')
-
-    def test_request_for_installable_pass(self):
-        '''Requests a test for an installable package, test pass'''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'green 1.1~beta PASS green 1.1~beta\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['PASS']])
-
-    def test_multi_rdepends_with_tests_running(self):
-        '''Multiple reverse dependencies with tests (still running)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 RUNNING green 2\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']])
-
-    def test_multi_rdepends_with_tests_fail_always(self):
-        '''Multiple reverse dependencies with tests (fail)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 FAIL green 2\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
-
-    def test_multi_rdepends_with_tests_fail_regression(self):
-        '''Multiple reverse dependencies with tests (fail)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 FAIL green 2\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
-            history='darkgreen 1 PASS green 1\n')
-
-    def test_multi_rdepends_with_tests_pass(self):
-        '''Multiple reverse dependencies with tests (pass)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 PASS green 2\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']])
-
-    def test_multi_rdepends_with_some_tests_running(self):
-        '''Multiple reverse dependencies with some tests (running)'''
-
-        # add a third reverse dependency to libgreen1 which does not have a test
-        self.data.add('mint', False, {'Depends': 'libgreen1'})
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 RUNNING green 2\n'
-            'darkgreen 1 RUNNING green 2\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']])
-
-    def test_multi_rdepends_with_some_tests_fail_always(self):
-        '''Multiple reverse dependencies with some tests (fail)'''
-
-        # add a third reverse dependency to libgreen1 which does not have a test
-        self.data.add('mint', False, {'Depends': 'libgreen1'})
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 FAIL green 2\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
-
-    def test_multi_rdepends_with_some_tests_fail_regression(self):
-        '''Multiple reverse dependencies with some tests (fail)'''
-
-        # add a third reverse dependency to libgreen1 which does not have a test
-        self.data.add('mint', False, {'Depends': 'libgreen1'})
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 FAIL green 2\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
-            history='darkgreen 1 PASS green 1\n')
-
-    def test_multi_rdepends_with_some_tests_pass(self):
-        '''Multiple reverse dependencies with some tests (pass)'''
-
-        # add a third reverse dependency to libgreen1 which does not have a test
-        self.data.add('mint', False, {'Depends': 'libgreen1'})
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS green 2\n'
-            'darkgreen 1 PASS green 2\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']])
-
-    def test_binary_from_new_source_package_running(self):
-        '''building an existing binary for a new source package (running)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS newgreen 2\n'
-            'darkgreen 1 RUNNING newgreen 2\n',
-            NOT_CONSIDERED,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']])
-
-    def test_binary_from_new_source_package_fail_always(self):
-        '''building an existing binary for a new source package (fail)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS newgreen 2\n'
-            'darkgreen 1 FAIL newgreen 2\n',
-            VALID_CANDIDATE,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
-
-    def test_binary_from_new_source_package_fail_regression(self):
-        '''building an existing binary for a new source package (fail)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS newgreen 2\n'
-            'darkgreen 1 FAIL newgreen 2\n',
-            NOT_CONSIDERED,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
-            history='darkgreen 1 PASS green 1\n')
-
-    def test_binary_from_new_source_package_pass(self):
-        '''building an existing binary for a new source package (pass)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
-            'lightgreen 1 PASS newgreen 2\n'
-            'darkgreen 1 PASS newgreen 2\n',
-            VALID_CANDIDATE,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
-             '<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']])
-
-    def test_binary_from_new_source_package_uninst(self):
-        '''building an existing binary for a new source package (uninstallable)'''
-
-        self.do_test(
-            [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6, nosuchpkg'})],
-            'darkgreen 1 FAIL newgreen 2\n',
-            NOT_CONSIDERED,
-            [r'\bnewgreen\b.*\(- to .*>2<',
-             'libgreen1/amd64 unsatisfiable Depends: nosuchpkg'],
-            # autopkgtest should not be triggered for uninstallable pkg
-            ['autopkgtest'])
-
-    @unittest.expectedFailure
-    def test_result_from_older_version(self):
-        '''test result from older version than the uploaded one'''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'green 1.1~alpha PASS green 1.1~beta\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             # it's not entirely clear what precisely it should say here
-             '<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']])
-
-    def test_request_for_installable_fail_regression_promoted(self):
-        '''Requests a test for an installable package, test fail, is a regression.
-
-        This test verifies a bug in britney where a package was promoted if latest test
-        appeared before previous result in history, only the last result in
-        alphabetic order was taken into account. For example:
-            A 1 FAIL B 1
-            A 1 PASS A 1
-        In this case results for A 1 didn't appear in the list of results
-        triggered by the upload of B 1 and B 1 was promoted
-        '''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'lightgreen 1 FAIL green 1.1~beta\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
-            history="lightgreen 1 PASS lightgreen 1"
-        )
-
-    def test_history_always_passed(self):
-        '''All the results in history are PASS, and test passed
-
-        '''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'lightgreen 1 PASS green 1.1~beta\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']],
-            history="lightgreen 1 PASS lightgreen 1"
-        )
-
-    def test_history_always_failed(self):
-        '''All the results in history are FAIL, test fails. not a regression.
-
-        '''
-
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'lightgreen 1 FAIL green 1.1~beta\n',
-            VALID_CANDIDATE,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']],
-            history="lightgreen 1 FAIL lightgreen 1"
-        )
-
-    def test_history_regression(self):
-        '''All the results in history are PASS, test fails. Blocked.
-
-        '''
-        self.do_test(
-            [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
-            'lightgreen 1 FAIL green 1.1~beta\n',
-            NOT_CONSIDERED,
-            [r'\bgreen\b.*>1</a> to .*>1.1~beta<',
-             '<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
-            history="lightgreen 1 PASS lightgreen 1"
-        )
-
-    def shell(self):
-        # uninstallable unstable version
-        self.data.add('yellow', True, {'Version': '1.1~beta',
-                                       'Depends': 'libc6 (>= 0.9), nosuchpkg'})
-
-        self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n',
-                              'purple 2 FAIL pink 3.0.~britney\n')
-
-        print('run:\n%s -c %s\n' % (self.britney, self.britney_conf))
-        subprocess.call(['bash', '-i'], cwd=self.data.path)
-
-
 if __name__ == '__main__':
     unittest.main()

tests/test_boottest.py

@@ -11,6 +11,7 @@ import os
 import shutil
 import sys
 import tempfile
+import fileinput
 import unittest
 
 
@@ -236,10 +237,11 @@ args.func()
     def do_test(self, context, expect=None, no_expect=None):
         """Process the given package context and assert britney results."""
         for (pkg, fields) in context:
-            self.data.add(pkg, True, fields)
+            self.data.add(pkg, True, fields, testsuite='autopkgtest')
         self.make_boottest()
         (excuses, out) = self.run_britney()
         # print('-------\nexcuses: %s\n-----' % excuses)
+        # print('-------\nout: %s\n-----' % out)
         if expect:
             for re in expect:
                 self.assertRegexpMatches(excuses, re)
@@ -329,14 +331,6 @@ args.func()
              r'<li>Boottest result: UNKNOWN STATUS \(Jenkins: .*\)',
              '<li>Not considered'])
 
-    def create_hint(self, username, content):
-        """Populates a hint file for the given 'username' with 'content'."""
-        hints_path = os.path.join(
-            self.data.path,
-            'data/{}-proposed/Hints/{}'.format(self.data.series, username))
-        with open(hints_path, 'w') as fd:
-            fd.write(content)
-
     def test_skipped_by_hints(self):
         # `Britney` allows boottests to be skipped by hinting the
         # corresponding source with 'force-skiptest'. The boottest
@@ -420,37 +414,27 @@ args.func()
     def test_with_adt(self):
         # Boottest can run simultaneously with autopkgtest (adt).
 
-        # Enable ADT in britney configuration.
+        fake_amqp = os.path.join(self.data.path, 'amqp')
-        with open(self.britney_conf, 'r') as fp:
-            original_config = fp.read()
-        new_config = original_config.replace(
-            'ADT_ENABLE        = no', 'ADT_ENABLE        = yes')
-        with open(self.britney_conf, 'w') as fp:
-            fp.write(new_config)
 
-        # Create a fake 'adt-britney' that reports a RUNNING job for
+        # Enable ADT in britney configuration.
-        # the testing source ('purple_1.1').
+        for line in fileinput.input(self.britney_conf, inplace=True):
-        script_path = os.path.expanduser(
+            if 'ADT_ENABLE' in line:
-            "~/auto-package-testing/jenkins/adt-britney")
+                print('ADT_ENABLE   = yes')
-        os.makedirs(os.path.dirname(script_path))
+            elif 'ADT_AMQP' in line:
-        with open(script_path, 'w') as f:
+                print('ADT_AMQP = file://%s' % fake_amqp)
-            f.write('''#!/bin/sh -e
+            else:
-mkdir -p ~/proposed-migration/autopkgtest/work
+                sys.stdout.write(line)
-touch ~/proposed-migration/autopkgtest/work/adt.request.series
-echo "purple 1.1 RUNNING purple 1.1" >> ~/proposed-migration/autopkgtest/work/adt.result.series''')
-        os.chmod(script_path, 0o755)
 
         # Britney blocks testing source promotion while ADT and boottest
         # are running.
-        self.data.add('purple', False, {'Version': '1.0'})
+        self.data.add('purple', False, {'Version': '1.0'}, testsuite='autopkgtest')
         context = [
             ('purple', {'Version': '1.1'}),
         ]
         self.do_test(
             context,
             [r'\bpurple\b.*>1.0<.* to .*>1.1<',
-             '<li>autopkgtest for purple 1.1: {}'.format(
+             '<li>autopkgtest for purple 1.1: .*amd64.*in progress.*i386.*in progress',
-                 boottest.BootTest.EXCUSE_LABELS['RUNNING']),
              '<li>Boottest result: {}'.format(
                  boottest.BootTest.EXCUSE_LABELS['RUNNING']),
              '<li>Not considered'])