Consider manually re-ran failed tests for reverse dependencies

Commit 446 only considered a package's own tests. But we also need to check for
newer results of failed reverse dependency tests.  Introduce a new
failed_tests_for_trigger() helper which computes the failed (src, arch) failed
tests for a given package, and fetch new results for all of them.

autopkgtest.py

@@ -245,7 +245,7 @@ class AutoPackageTest(object):
         self.requested_tests.setdefault(src, {}).setdefault(
             ver, {}).setdefault(arch, set()).add((trigsrc, trigver))
 
-    def fetch_swift_results(self, swift_url, src, arch):
+    def fetch_swift_results(self, swift_url, src, arch, trigger=None):
         '''Download new results for source package/arch from swift'''
 
         # prepare query: get all runs with a timestamp later than latest_stamp
@@ -282,13 +282,15 @@ class AutoPackageTest(object):
             return
 
         for p in result_paths:
-            self.fetch_one_result(os.path.join(
+            self.fetch_one_result(
-                swift_url, 'autopkgtest-' + self.series, p, 'result.tar'), src, arch)
+                os.path.join(swift_url, 'autopkgtest-' + self.series, p, 'result.tar'),
+                src, arch, trigger)
 
-    def fetch_one_result(self, url, src, arch):
+    def fetch_one_result(self, url, src, arch, trigger=None):
         '''Download one result URL for source/arch
 
-        Remove matching pending_tests entries.
+        Remove matching pending_tests entries. If trigger is given (src, ver)
+        it is added to the triggers of that result.
         '''
         try:
             f = urlopen(url)
@@ -343,6 +345,8 @@ class AutoPackageTest(object):
                 except KeyError:
                     self.log_error('-> does not match any pending request!')
                     pass
+        if trigger:
+            satisfied_triggers.add(trigger)
 
         # add this result
         src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}])
@@ -352,6 +356,21 @@ class AutoPackageTest(object):
         if stamp > src_arch_results[0]:
             src_arch_results[0] = stamp
 
+    def failed_tests_for_trigger(self, trigsrc, trigver):
+        '''Return (src, arch) set for failed tests for given trigger pkg'''
+
+        result = set()
+        for src, srcinfo in self.test_results.iteritems():
+            for arch, (stamp, vermap) in srcinfo.iteritems():
+                for ver, (passed, triggers) in vermap.iteritems():
+                    if not passed:
+                        # triggers might contain tuples or lists (after loading
+                        # from json), so iterate/check manually
+                        for s, v in triggers:
+                            if trigsrc == s and trigver == v:
+                                result.add((src, arch))
+        return result
+
     #
     # obsolete adt-britney helpers
     #
@@ -586,15 +605,12 @@ class AutoPackageTest(object):
                         self.fetch_swift_results(swift_url, pkg, arch)
             # also update results for excuses whose tests failed, in case a
             # manual retry worked
-            for (pkg, ver) in packages:
+            for (trigpkg, trigver) in packages:
-                if pkg not in self.pending_tests:
+                if trigpkg not in self.pending_tests:
-                    for arch in self.test_results.get(pkg, {}):
+                    for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver):
-                        try:
+                        self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' %
-                            if not self.test_results[pkg][arch][1][ver][0]:
+                                         (pkg, arch, trigpkg, trigver))
-                                self.log_verbose('Checking for new results for failed %s/%s on %s' % (pkg, ver, arch))
+                        self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver))
-                                self.fetch_swift_results(swift_url, pkg, arch)
-                        except KeyError:
-                            pass
 
             # update the results cache
             with open(self.results_cache_file + '.new', 'w') as f:

tests/test_autopkgtest.py

@@ -318,34 +318,46 @@ lightgreen 2 i386 lightgreen 2
         self.assertEqual(self.pending_requests, '')
 
     def test_rerun_failure(self):
-        '''manually re-running a failed test gets picked up'''
+        '''manually re-running failed tests gets picked up'''
 
         # first run fails
         self.swift.set_results({'autopkgtest-series': {
-            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/i386/g/green/20150101_100101@': (4, 'green 2'),
-            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/amd64/g/green/20150101_100101@': (4, 'green 2'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'),
         }})
 
         self.do_test(
-            [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
             # FIXME: while we only submit requests through AMQP, but don't consider
             # their results, we don't expect this to hold back stuff.
             VALID_CANDIDATE,
-            [r'\blightgreen\b.*>1</a> to .*>2<',
+            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'])
+             r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
+             r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
         self.assertEqual(self.pending_requests, '')
 
-        # re-running test manually succeeded
+        # re-running test manually succeeded (note: darkgreen result should be
+        # cached already)
         self.swift.set_results({'autopkgtest-series': {
-            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/i386/g/green/20150101_100101@': (4, 'green 2'),
-            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/amd64/g/green/20150101_100101@': (4, 'green 2'),
-            'series/i386/l/lightgreen/20150101_100201@': (0, 'lightgreen 2'),
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
-            'series/amd64/l/lightgreen/20150101_100201@': (0, 'lightgreen 2'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+
+            'series/i386/g/green/20150101_100201@': (0, 'green 2'),
+            'series/amd64/g/green/20150101_100201@': (0, 'green 2'),
+            'series/i386/l/lightgreen/20150101_100201@': (0, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100201@': (0, 'lightgreen 1'),
         }})
         self.do_test(
             [], VALID_CANDIDATE,
-            [r'\blightgreen\b.*>1</a> to .*>2<',
+            [r'\bgreen\b.*>1</a> to .*>2<',
-             r'autopkgtest for lightgreen 2: .*amd64.*Pass.*i386.*Pass'])
+             r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
+             r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass'])
         self.assertEqual(self.pending_requests, '')
 
     def test_no_amqp_config(self):