Track "ever passed" in results cache

Add bool whether there is any successful test of src/arch of any version. This
will be used for detecting "regression" vs. "always failed".

WARNING: This changes the results.cache format, so results.cache has to be
removed and recreated before deploying this.

autopkgtest.py

@@ -99,7 +99,7 @@ class AutoPackageTest(object):
             os.mkdir(self.test_state_dir)
         self.read_pending_tests()
 
-        # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers)]
+        # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers), ever_passed]
         # - "passed" is a bool
         # - It's tempting to just use a global "latest" time stamp, but due to
         #   swift's "eventual consistency" we might miss results with older time
@@ -109,6 +109,9 @@ class AutoPackageTest(object):
         # - triggers is a list of (source, version) pairs which unstable
         #   packages triggered this test run. We need to track this to avoid
         #   unnecessarily re-running tests.
+        # - ever_passed is a bool whether there is any successful test of
+        #   src/arch of any version. This is used for detecting "regression"
+        #   vs. "always failed"
         self.test_results = {}
         self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache')
 
@@ -349,7 +352,10 @@ class AutoPackageTest(object):
             satisfied_triggers.add(trigger)
 
         # add this result
-        src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}])
+        src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}, False])
+        if passed:
+            # update ever_passed field
+            src_arch_results[2] = True
         src_arch_results[1][ver] = (passed, merge_triggers(
             src_arch_results[1].get(ver, (None, []))[1], satisfied_triggers))
         # update latest_stamp
@@ -361,7 +367,7 @@ class AutoPackageTest(object):
 
         result = set()
         for src, srcinfo in self.test_results.iteritems():
-            for arch, (stamp, vermap) in srcinfo.iteritems():
+            for arch, (stamp, vermap, ever_passed) in srcinfo.iteritems():
                 for ver, (passed, triggers) in vermap.iteritems():
                     if not passed:
                         # triggers might contain tuples or lists (after loading

tests/test_autopkgtest.py

@@ -204,10 +204,13 @@ lightgreen 1 i386 green 2
         with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f:
             res = json.load(f)
         self.assertEqual(res['green']['i386'],
-                         ['20150101_100200@', {'1': [False, []],
+                         ['20150101_100200@',
-                                               '2': [True, [['green', '2']]]}])
+                          {'1': [False, []], '2': [True, [['green', '2']]]},
+                          True])
         self.assertEqual(res['lightgreen']['amd64'],
-                         ['20150101_100101@', {'1': [True, [['green', '2']]]}])
+                         ['20150101_100101@',
+                          {'1': [True, [['green', '2']]]},
+                          True])
 
         # third run should not trigger any new tests, should all be in the
         # cache
@@ -241,6 +244,7 @@ lightgreen 1 i386 green 2
         # second run collects the results
         self.swift.set_results({'autopkgtest-series': {
             'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
             'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
             'series/i386/g/green/20150101_100200@': (0, 'green 2'),
             'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
@@ -301,7 +305,9 @@ lightgreen 2 i386 lightgreen 2
 
         # one tmpfail result without testpkg-version
         self.swift.set_results({'autopkgtest-series': {
+            'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/i386/l/lightgreen/20150101_100101@': (16, None),
+            'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/amd64/l/lightgreen/20150101_100101@': (16, 'lightgreen 2'),
         }})
 
@@ -321,9 +327,13 @@ lightgreen 2 i386 lightgreen 2
 
         # first run fails
         self.swift.set_results({'autopkgtest-series': {
+            'series/i386/g/green/20150101_100000@': (0, 'green 2'),
             'series/i386/g/green/20150101_100101@': (4, 'green 2'),
+            'series/amd64/g/green/20150101_100000@': (0, 'green 2'),
             'series/amd64/g/green/20150101_100101@': (4, 'green 2'),
+            'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
             'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
             'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'),
@@ -342,9 +352,13 @@ lightgreen 2 i386 lightgreen 2
         # re-running test manually succeeded (note: darkgreen result should be
         # cached already)
         self.swift.set_results({'autopkgtest-series': {
+            'series/i386/g/green/20150101_100000@': (0, 'green 2'),
             'series/i386/g/green/20150101_100101@': (4, 'green 2'),
+            'series/amd64/g/green/20150101_100000@': (0, 'green 2'),
             'series/amd64/g/green/20150101_100101@': (4, 'green 2'),
+            'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
+            'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
 
             'series/i386/g/green/20150101_100201@': (0, 'green 2'),