autopkgtest: Check for test results from newer package version than the requested one

There are scenarios when britney requests a package test for a particular
version but we actually get a result for a later version:

 * When britney runs the later version is not built yet and thus it is in
   excludes; but at the time when the test actually runs the package is built.

 * We don't support running tests for a given older (source) version yet, tests
   always get run from the latest unstable source even if that isn't built yet.

Thus we need to consider results >= the requested version. However, we prefer a
succesful result for the originally requested version so that we can continue
to remove a broken version from unstable. This is already covered by
TestAutoPkgTest.test_remove_from_unstable.

autopkgtest.py

@@ -63,6 +63,26 @@ def merge_triggers(trigs1, trigs2):
     return list(pkgvers.items())
 
 
+def latest_item(ver_map, min_version=None):
+    '''Return (ver, value) from version -> value map with latest version number
+
+    If min_version is given, version has to be >= that, otherwise a KeyError is
+    raised.
+    '''
+    latest = None
+    for ver in ver_map:
+        if latest is None or apt_pkg.version_compare(ver, latest) > 0:
+            latest = ver
+    if min_version is not None and latest is not None and \
+       apt_pkg.version_compare(latest, min_version) < 0:
+        latest = None
+
+    if latest is not None:
+        return (latest, ver_map[latest])
+    else:
+        raise KeyError('no version >= %s' % min_version)
+
+
 class AutoPackageTest(object):
     """autopkgtest integration
 
@@ -499,7 +519,18 @@ class AutoPackageTest(object):
             for arch in self.britney.options.adt_arches.split():
                 try:
                     (_, ver_map, ever_passed) = self.test_results[testsrc][arch]
+
+                    # we prefer passing tests for the specified testver; but if
+                    # they fail, we also check for a result > testver, as test
+                    # runs might see built versions which we didn't see in
+                    # britney yet
+                    try:
                         (status, triggers) = ver_map[testver]
+                        if not status:
+                            raise KeyError
+                    except KeyError:
+                        (testver, (status, triggers)) = latest_item(ver_map, testver)
+
                     # triggers might contain tuples or lists
                     if (trigsrc, trigver) not in triggers and [trigsrc, trigver] not in triggers:
                         raise KeyError('No result for trigger %s/%s yet' % (trigsrc, trigver))

tests/test_autopkgtest.py

@@ -446,6 +446,59 @@ lightgreen 1 i386 green 2
             ['autopkgtest for brokengreen'])
         self.assertEqual(self.amqp_requests, set())
 
+    def test_rdepends_unbuilt_new_version_result(self):
+        '''Unbuilt reverse dependency gets test result for newer version
+
+        This might happen if the autopkgtest infrastructure runs the unstable
+        source tests against the testing binaries. Even if that gets done
+        properly it might still happen that at the time of the britney run the
+        package isn't built yet, but it is once the test gets run.
+        '''
+        # old lightgreen fails, thus new green should be held back
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'),
+            'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
+            'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
+            'series/i386/g/green/20150101_020000@': (0, 'green 1'),
+            'series/amd64/g/green/20150101_020000@': (0, 'green 1'),
+            'series/i386/g/green/20150101_100200@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (0, 'green 2'),
+        }})
+
+        # add unbuilt lightgreen; should run tests against the old version
+        self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'})
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            NOT_CONSIDERED,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'\blightgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1 \(2 is unbuilt/uninstallable\): .*amd64.*Regression.*i386.*Regression',
+             r'lightgreen has no up-to-date binaries on any arch'],
+            ['Valid candidate'])
+        self.assertEqual(self.pending_requests, '')
+
+        # lightgreen 2 stays unbuilt in britney, but we get a test result for it
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/l/lightgreen/20150101_100200@': (0, 'lightgreen 2'),
+            'series/amd64/l/lightgreen/20150101_102000@': (0, 'lightgreen 2'),
+        }})
+        self.do_test(
+            [],
+            VALID_CANDIDATE,
+            [r'\bgreen\b.*>1</a> to .*>2<',
+             r'\blightgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 2.*: .*amd64.*Pass.*i386.*Pass',
+             r'lightgreen has no up-to-date binaries on any arch'])
+
+        self.assertEqual(self.pending_requests, '')
+
     def test_hint_force_badtest(self):
         '''force-badtest hint'''