Consider manually re-ran failed tests

When collecting results, not only check pending tests, but also new results for
failed tests. This picks up new test results from manual retries which might
now have succeeded.

autopkgtest.py

@@ -563,7 +563,7 @@ class AutoPackageTest(object):
         if os.path.exists(request_path):
             self._adt_britney("submit", request_path)
 
-    def collect(self):
+    def collect(self, packages):
         # fetch results from swift
         try:
             swift_url = self.britney.options.adt_swift_url
@@ -584,6 +584,17 @@ class AutoPackageTest(object):
                 for archinfo in verinfo.values():
                     for arch in archinfo:
                         self.fetch_swift_results(swift_url, pkg, arch)
+            # also update results for excuses whose tests failed, in case a
+            # manual retry worked
+            for (pkg, ver) in packages:
+                if pkg not in self.pending_tests:
+                    for arch in self.test_results.get(pkg, {}):
+                        try:
+                            if not self.test_results[pkg][arch][1][ver][0]:
+                                self.log_verbose('Checking for new results for failed %s/%s on %s' % (pkg, ver, arch))
+                                self.fetch_swift_results(swift_url, pkg, arch)
+                        except KeyError:
+                            pass
 
             # update the results cache
             with open(self.results_cache_file + '.new', 'w') as f:

britney.py

@@ -1848,7 +1848,7 @@ class Britney(object):
             autopkgtest.request(autopkgtest_packages, autopkgtest_excludes)
             if not self.options.dry_run:
                 autopkgtest.submit()
-                autopkgtest.collect()
+                autopkgtest.collect(autopkgtest_packages)
             jenkins_public = "https://jenkins.qa.ubuntu.com/job"
             jenkins_private = (
                 "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %

tests/test_autopkgtest.py

@@ -317,6 +317,37 @@ lightgreen 2 i386 lightgreen 2
 
         self.assertEqual(self.pending_requests, '')
 
+    def test_rerun_failure(self):
+        '''manually re-running a failed test gets picked up'''
+
+        # first run fails
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+        }})
+
+        self.do_test(
+            [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
+            # FIXME: while we only submit requests through AMQP, but don't consider
+            # their results, we don't expect this to hold back stuff.
+            VALID_CANDIDATE,
+            [r'\blightgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'])
+        self.assertEqual(self.pending_requests, '')
+
+        # re-running test manually succeeded
+        self.swift.set_results({'autopkgtest-series': {
+            'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'),
+            'series/i386/l/lightgreen/20150101_100201@': (0, 'lightgreen 2'),
+            'series/amd64/l/lightgreen/20150101_100201@': (0, 'lightgreen 2'),
+        }})
+        self.do_test(
+            [], VALID_CANDIDATE,
+            [r'\blightgreen\b.*>1</a> to .*>2<',
+             r'autopkgtest for lightgreen 2: .*amd64.*Pass.*i386.*Pass'])
+        self.assertEqual(self.pending_requests, '')
+
     def test_no_amqp_config(self):
         '''Run without autopkgtest requests'''