From 76287b50ca9c85266b6646f794b0c0b44e719376 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 28 Jul 2015 10:46:30 +0200 Subject: [PATCH] Track "ever passed" in results cache Add bool whether there is any successful test of src/arch of any version. This will be used for detecting "regression" vs. "always failed". WARNING: This changes the results.cache format, so results.cache has to be removed and recreated before deploying this. --- autopkgtest.py | 12 +++++++++--- tests/test_autopkgtest.py | 20 +++++++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index 8b390a9..91618e8 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -99,7 +99,7 @@ class AutoPackageTest(object): os.mkdir(self.test_state_dir) self.read_pending_tests() - # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers)] + # results map: src -> arch -> [latest_stamp, ver -> (passed, triggers), ever_passed] # - "passed" is a bool # - It's tempting to just use a global "latest" time stamp, but due to # swift's "eventual consistency" we might miss results with older time @@ -109,6 +109,9 @@ class AutoPackageTest(object): # - triggers is a list of (source, version) pairs which unstable # packages triggered this test run. We need to track this to avoid # unnecessarily re-running tests. + # - ever_passed is a bool whether there is any successful test of + # src/arch of any version. This is used for detecting "regression" + # vs. "always failed" self.test_results = {} self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') @@ -349,7 +352,10 @@ class AutoPackageTest(object): satisfied_triggers.add(trigger) # add this result - src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}]) + src_arch_results = self.test_results.setdefault(src, {}).setdefault(arch, [stamp, {}, False]) + if passed: + # update ever_passed field + src_arch_results[2] = True src_arch_results[1][ver] = (passed, merge_triggers( src_arch_results[1].get(ver, (None, []))[1], satisfied_triggers)) # update latest_stamp @@ -361,7 +367,7 @@ class AutoPackageTest(object): result = set() for src, srcinfo in self.test_results.iteritems(): - for arch, (stamp, vermap) in srcinfo.iteritems(): + for arch, (stamp, vermap, ever_passed) in srcinfo.iteritems(): for ver, (passed, triggers) in vermap.iteritems(): if not passed: # triggers might contain tuples or lists (after loading diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 537e2c2..8d04ede 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -204,10 +204,13 @@ lightgreen 1 i386 green 2 with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: res = json.load(f) self.assertEqual(res['green']['i386'], - ['20150101_100200@', {'1': [False, []], - '2': [True, [['green', '2']]]}]) + ['20150101_100200@', + {'1': [False, []], '2': [True, [['green', '2']]]}, + True]) self.assertEqual(res['lightgreen']['amd64'], - ['20150101_100101@', {'1': [True, [['green', '2']]]}]) + ['20150101_100101@', + {'1': [True, [['green', '2']]]}, + True]) # third run should not trigger any new tests, should all be in the # cache @@ -241,6 +244,7 @@ lightgreen 1 i386 green 2 # second run collects the results self.swift.set_results({'autopkgtest-series': { 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), 'series/i386/g/green/20150101_100200@': (0, 'green 2'), 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), @@ -301,7 +305,9 @@ lightgreen 2 i386 lightgreen 2 # one tmpfail result without testpkg-version self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), 'series/i386/l/lightgreen/20150101_100101@': (16, None), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), 'series/amd64/l/lightgreen/20150101_100101@': (16, 'lightgreen 2'), }}) @@ -321,9 +327,13 @@ lightgreen 2 i386 lightgreen 2 # first run fails self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 2'), 'series/i386/g/green/20150101_100101@': (4, 'green 2'), + 'series/amd64/g/green/20150101_100000@': (0, 'green 2'), 'series/amd64/g/green/20150101_100101@': (4, 'green 2'), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'), @@ -342,9 +352,13 @@ lightgreen 2 i386 lightgreen 2 # re-running test manually succeeded (note: darkgreen result should be # cached already) self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 2'), 'series/i386/g/green/20150101_100101@': (4, 'green 2'), + 'series/amd64/g/green/20150101_100000@': (0, 'green 2'), 'series/amd64/g/green/20150101_100101@': (4, 'green 2'), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), 'series/i386/g/green/20150101_100201@': (0, 'green 2'),