From 2e393a0c312bff99a818180aac3ef6df51c2bbce Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 27 Jan 2016 12:01:00 +0100 Subject: [PATCH] Add option for using a shared r/o results.cache This is needed by the CI train, where we (1) don't want to cache intermediate results for PPA runs, as they might "accidentally" pass in between and fail again for the final silo, (2) want to seed britney with the Ubuntu results.cache, to detect regressions relative to Ubuntu. Introduce ADT_SHARED_RESULTS_CACHE option which can point to a path to results.cache. This will then not be updated by britney. --- autopkgtest.py | 16 ++++++++------ britney.conf | 3 +++ britney_nobreakall.conf | 6 ++++++ tests/test_autopkgtest.py | 45 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index f19ebce..da47107 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -76,7 +76,10 @@ class AutoPackageTest(object): # This is also used for tracking the latest seen time stamp for # requesting only newer results. self.test_results = {} - self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') + if self.britney.options.adt_shared_results_cache: + self.results_cache_file = self.britney.options.adt_shared_results_cache + else: + self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') self.swift_container = 'autopkgtest-' + self.series if self.britney.options.adt_ppas: @@ -528,11 +531,12 @@ class AutoPackageTest(object): for (testsrc, _) in self.tests_for_source(src, ver, arch): self.pkg_test_request(testsrc, arch, src + '/' + ver) - # update the results on-disk cache - self.log_verbose('Updating results cache') - with open(self.results_cache_file + '.new', 'w') as f: - json.dump(self.test_results, f, indent=2) - os.rename(self.results_cache_file + '.new', self.results_cache_file) + # update the results on-disk cache, unless we are using a r/o shared one + if not self.britney.options.adt_shared_results_cache: + self.log_verbose('Updating results cache') + with open(self.results_cache_file + '.new', 'w') as f: + json.dump(self.test_results, f, indent=2) + os.rename(self.results_cache_file + '.new', self.results_cache_file) # update the pending tests on-disk cache self.log_verbose('Updated pending requested tests in %s' % self.pending_tests_file) diff --git a/britney.conf b/britney.conf index bf278c2..4c81c43 100644 --- a/britney.conf +++ b/britney.conf @@ -72,3 +72,6 @@ ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e # space separate list of PPAs to add for test requests and for polling results; # the *last* one determines the swift container name ADT_PPAS = +# set this to the path of a (r/o) results.cache for running many parallel +# britney instances for PPAs without updating the cache +ADT_SHARED_RESULTS_CACHE = diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index ad22af8..b2fb55a 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -69,3 +69,9 @@ ADT_ARCHES = amd64 i386 armhf ppc64el ADT_AMQP = amqp://test_request:password@162.213.33.228 # Swift base URL with the results (must be publicly readable and browsable) ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac +# space separate list of PPAs to add for test requests and for polling results; +# the *last* one determines the swift container name +ADT_PPAS = +# set this to the path of a (r/o) results.cache for running many parallel +# britney instances for PPAs without updating the cache +ADT_SHARED_RESULTS_CACHE = diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index f857f4c..90e0ee0 100755 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -1663,6 +1663,51 @@ class T(TestBase): self.assertFalse(os.path.exists(os.path.join(self.data.path, 'output', 'series', 'output.txt'))) + def test_shared_results_cache(self): + '''Run with shared r/o results.cache''' + + # first run to create results.cache + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen 2': {'i386': 'PASS', 'amd64': 'PASS'}})}, + ) + + # move and remember original contents + local_path = os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache') + shared_path = os.path.join(self.data.path, 'shared_results.cache') + os.rename(local_path, shared_path) + with open(shared_path) as f: + orig_contents = f.read() + + # enable shared cache + for line in fileinput.input(self.britney_conf, inplace=True): + if 'ADT_SHARED_RESULTS_CACHE' in line: + print('ADT_SHARED_RESULTS_CACHE = %s' % shared_path) + else: + sys.stdout.write(line) + + # second run, should now not update cache + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 3', tr('lightgreen/3')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.data.remove_all(True) + self.do_test( + [('lightgreen', {'Version': '3', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen 3': {'i386': 'PASS', 'amd64': 'PASS'}})}, + ) + + # leaves results.cache untouched + self.assertFalse(os.path.exists(local_path)) + with open(shared_path) as f: + self.assertEqual(orig_contents, f.read()) + if __name__ == '__main__': unittest.main()