diff --git a/britney.conf b/britney.conf index 358132f..84c76fb 100644 --- a/britney.conf +++ b/britney.conf @@ -89,3 +89,6 @@ ADT_PPAS = # set this to the path of a (r/o) results.cache for running many parallel # britney instances for PPAs without updating the cache ADT_SHARED_RESULTS_CACHE = +# Swift base URL with the results (must be publicly readable and browsable) +# or file location if results are pre-fetched +ADT_SWIFT_URL = file://cache/debci.json diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py index d01bb7e..7a96f2f 100644 --- a/britney2/policies/autopkgtest.py +++ b/britney2/policies/autopkgtest.py @@ -118,6 +118,21 @@ class AutopkgtestPolicy(BasePolicy): self.log('%s does not exist, re-downloading all results from swift' % self.results_cache_file) + # read in the new results + if self.options.adt_swift_url.startswith('file://'): + debci_file = self.options.adt_swift_url[7:] + if os.path.exists(debci_file): + with open(debci_file) as f: + test_results = json.load(f) + self.log('Read new results from %s' % debci_file) + for result in test_results: + (trigger, src, arch, ver, passed, stamp) = result + self.remove_from_pending(trigger, src, arch) + self.add_trigger_to_results(trigger, src, ver, arch, stamp, passed) + else: + self.log('%s does not exist, no new data will be processed' % + debci_file) + # we need sources, binaries, and installability tester, so for now # remember the whole britney object self.britney = britney @@ -557,35 +572,41 @@ class AutopkgtestPolicy(BasePolicy): # remove matching test requests for trigger in result_triggers: - try: - arch_list = self.pending_tests[trigger][src] - arch_list.remove(arch) - if not arch_list: - del self.pending_tests[trigger][src] - if not self.pending_tests[trigger]: - del self.pending_tests[trigger] - self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger)) - except (KeyError, ValueError): - self.log('-> does not match any pending request for %s/%s' % (src, arch)) + self.remove_from_pending(trigger, src, arch) # add this result for trigger in result_triggers: - # If a test runs because of its own package (newer version), ensure - # that we got a new enough version; FIXME: this should be done more - # generically by matching against testpkg-versions - (trigsrc, trigver) = trigger.split('/', 1) - if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0: - self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E') - continue + self.add_trigger_to_results(trigger, src, ver, arch, stamp, passed) - result = self.test_results.setdefault(trigger, {}).setdefault( - src, {}).setdefault(arch, [False, None, '']) + def remove_from_pending(self, trigger, src, arch): + try: + arch_list = self.pending_tests[trigger][src] + arch_list.remove(arch) + if not arch_list: + del self.pending_tests[trigger][src] + if not self.pending_tests[trigger]: + del self.pending_tests[trigger] + self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger)) + except (KeyError, ValueError): + self.log('-> does not match any pending request for %s/%s' % (src, arch)) - # don't clobber existing passed results with failures from re-runs - if passed or not result[0]: - result[0] = passed - result[1] = ver - result[2] = stamp + def add_trigger_to_results(self, trigger, src, ver, arch, stamp, passed): + # If a test runs because of its own package (newer version), ensure + # that we got a new enough version; FIXME: this should be done more + # generically by matching against testpkg-versions + (trigsrc, trigver) = trigger.split('/', 1) + if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0: + self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E') + return + + result = self.test_results.setdefault(trigger, {}).setdefault( + src, {}).setdefault(arch, [False, None, '']) + + # don't clobber existing passed results with failures from re-runs + if passed or not result[0]: + result[0] = passed + result[1] = ver + result[2] = stamp def send_test_request(self, src, arch, trigger, huge=False): '''Send out AMQP request for testing src/arch for trigger @@ -628,6 +649,8 @@ class AutopkgtestPolicy(BasePolicy): # Don't re-request if we already have a result try: passed = self.test_results[trigger][src][arch][0] + if self.options.adt_swift_url.startswith('file://'): + return if passed: self.log('%s/%s triggered by %s already passed' % (src, arch, trigger)) return @@ -635,6 +658,10 @@ class AutopkgtestPolicy(BasePolicy): (src, arch, trigger)) raise KeyError # fall through except KeyError: + # Without swift we don't expect new results + if self.options.adt_swift_url.startswith('file://'): + pass + self.fetch_swift_results(self.options.adt_swift_url, src, arch) # do we have one now? try: diff --git a/tests/__init__.py b/tests/__init__.py index 21e90af..03e41ff 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -388,8 +388,7 @@ ADT_AMQP = file://output/debci.input ADT_PPAS = ADT_SHARED_RESULTS_CACHE = -# TODO: remove next line -ADT_SWIFT_URL = overwritten_by_the_test_anyways +ADT_SWIFT_URL = http://localhost:18085 ''') assert os.path.exists(self.britney) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 5a5b0bd..33e744c 100755 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -48,10 +48,6 @@ class T(TestBase): for line in fileinput.input(self.britney_conf, inplace=True): if 'ADT_AMQP' in line: print('ADT_AMQP = file://%s' % self.fake_amqp) - elif 'ADT_SWIFT_URL' in line: - print('ADT_SWIFT_URL = http://localhost:18085') - elif 'ADT_ARCHES' in line: - print('ADT_ARCHES = amd64 i386') else: sys.stdout.write(line) @@ -2394,5 +2390,74 @@ class T(TestBase): ### self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': ppa, 'green': ppa}) + def test_swift_url_is_file(self): + '''Run without swift but with debci file (as Debian does)''' + '''Based on test_multi_rdepends_with_tests_regression''' + '''Multiple reverse dependencies with tests (regression)''' + + debci_file = os.path.join(self.data.path, 'debci.output') + + # Don't use swift but debci output file + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('ADT_SWIFT_URL'): + print('ADT_SWIFT_URL = file://%s' % debci_file) + else: + sys.stdout.write(line) + + with open(debci_file, 'w') as f: + f.write(''' +[ + ["green/2", "darkgreen", "i386", "1", true, "20170917_100000"], + ["green/2", "darkgreen", "amd64", "1", true, "20170917_100000"], + ["green/1", "lightgreen", "i386", "1", true, "20170917_101000"], + ["green/2", "lightgreen", "i386", "1", false, "20170917_101001"], + ["green/1", "lightgreen", "amd64", "1", true, "20170917_101000"], + ["green/2", "lightgreen", "amd64", "1", false, "20170917_101001"], + ["green/2", "green", "i386", "2", true, "20170917_102000"], + ["green/1", "green", "amd64", "2", true, "20170917_102000"], + ["green/2", "green", "amd64", "2", false, "20170917_102001"] +] +''') + + self.data.add_default_packages(green=False) + + out, exc, _ = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + ) + + # should have links to log and history, but no artifacts (as this is + # not a PPA) + self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][0], + 'REGRESSION') + link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][1]) + self.assertEqual(link.path, os.path.join(debci_file, 'autopkgtest-testing/testing/amd64/l/lightgreen/20170917_101001/log.gz')) + self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][2:4], + ['http://autopkgtest.ubuntu.com/packages/l/lightgreen/testing/amd64', + None]) + + # should have retry link for the regressions (not a stable URL, test + # separately) + link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][4]) + self.assertEqual(link.netloc, 'autopkgtest.ubuntu.com') + self.assertEqual(link.path, '/request.cgi') + self.assertEqual(urllib.parse.parse_qs(link.query), + {'release': ['testing'], 'arch': ['amd64'], + 'package': ['lightgreen'], 'trigger': ['green/2']}) + + # we already had all results before the run, so this should not trigger + # any new requests + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + if __name__ == '__main__': unittest.main()