Implement swift free usage of autopkgtest

britney.conf

@@ -89,3 +89,6 @@ ADT_PPAS          =
 # set this to the path of a (r/o) results.cache for running many parallel
 # britney instances for PPAs without updating the cache
 ADT_SHARED_RESULTS_CACHE =
+# Swift base URL with the results (must be publicly readable and browsable)
+# or file location if results are pre-fetched
+ADT_SWIFT_URL     = file://cache/debci.json

britney2/policies/autopkgtest.py

@@ -118,6 +118,21 @@ class AutopkgtestPolicy(BasePolicy):
             self.log('%s does not exist, re-downloading all results from swift' %
                      self.results_cache_file)
 
+        # read in the new results
+        if self.options.adt_swift_url.startswith('file://'):
+            debci_file = self.options.adt_swift_url[7:]
+            if os.path.exists(debci_file):
+                with open(debci_file) as f:
+                    test_results = json.load(f)
+                self.log('Read new results from %s' % debci_file)
+                for result in test_results:
+                    (trigger, src, arch, ver, passed, stamp) = result
+                    self.remove_from_pending(trigger, src, arch)
+                    self.add_trigger_to_results(trigger, src, ver, arch, stamp, passed)
+            else:
+                self.log('%s does not exist, no new data will be processed' %
+                             debci_file)
+
         # we need sources, binaries, and installability tester, so for now
         # remember the whole britney object
         self.britney = britney
@@ -557,35 +572,41 @@ class AutopkgtestPolicy(BasePolicy):
 
         # remove matching test requests
         for trigger in result_triggers:
-            try:
+            self.remove_from_pending(trigger, src, arch)
-                arch_list = self.pending_tests[trigger][src]
-                arch_list.remove(arch)
-                if not arch_list:
-                    del self.pending_tests[trigger][src]
-                if not self.pending_tests[trigger]:
-                    del self.pending_tests[trigger]
-                self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger))
-            except (KeyError, ValueError):
-                self.log('-> does not match any pending request for %s/%s' % (src, arch))
 
         # add this result
         for trigger in result_triggers:
-            # If a test runs because of its own package (newer version), ensure
+            self.add_trigger_to_results(trigger, src, ver, arch, stamp, passed)
-            # that we got a new enough version; FIXME: this should be done more
-            # generically by matching against testpkg-versions
-            (trigsrc, trigver) = trigger.split('/', 1)
-            if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0:
-                self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E')
-                continue
 
-            result = self.test_results.setdefault(trigger, {}).setdefault(
+    def remove_from_pending(self, trigger, src, arch):
-                src, {}).setdefault(arch, [False, None, ''])
+        try:
+            arch_list = self.pending_tests[trigger][src]
+            arch_list.remove(arch)
+            if not arch_list:
+                del self.pending_tests[trigger][src]
+            if not self.pending_tests[trigger]:
+                del self.pending_tests[trigger]
+            self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger))
+        except (KeyError, ValueError):
+            self.log('-> does not match any pending request for %s/%s' % (src, arch))
 
-            # don't clobber existing passed results with failures from re-runs
+    def add_trigger_to_results(self, trigger, src, ver, arch, stamp, passed):
-            if passed or not result[0]:
+        # If a test runs because of its own package (newer version), ensure
-                result[0] = passed
+        # that we got a new enough version; FIXME: this should be done more
-                result[1] = ver
+        # generically by matching against testpkg-versions
-                result[2] = stamp
+        (trigsrc, trigver) = trigger.split('/', 1)
+        if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0:
+            self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E')
+            return
+    
+        result = self.test_results.setdefault(trigger, {}).setdefault(
+            src, {}).setdefault(arch, [False, None, ''])
+    
+        # don't clobber existing passed results with failures from re-runs
+        if passed or not result[0]:
+            result[0] = passed
+            result[1] = ver
+            result[2] = stamp
 
     def send_test_request(self, src, arch, trigger, huge=False):
         '''Send out AMQP request for testing src/arch for trigger
@@ -628,6 +649,8 @@ class AutopkgtestPolicy(BasePolicy):
         # Don't re-request if we already have a result
         try:
             passed = self.test_results[trigger][src][arch][0]
+            if self.options.adt_swift_url.startswith('file://'):
+                return
             if passed:
                 self.log('%s/%s triggered by %s already passed' % (src, arch, trigger))
                 return
@@ -635,6 +658,10 @@ class AutopkgtestPolicy(BasePolicy):
                              (src, arch, trigger))
             raise KeyError  # fall through
         except KeyError:
+            # Without swift we don't expect new results
+            if self.options.adt_swift_url.startswith('file://'):
+                pass
+
             self.fetch_swift_results(self.options.adt_swift_url, src, arch)
             # do we have one now?
             try:

tests/__init__.py

@@ -388,8 +388,7 @@ ADT_AMQP          = file://output/debci.input
 ADT_PPAS          =
 ADT_SHARED_RESULTS_CACHE =
 
-# TODO: remove next line
+ADT_SWIFT_URL     = http://localhost:18085
-ADT_SWIFT_URL     = overwritten_by_the_test_anyways
 ''')
         assert os.path.exists(self.britney)
 

tests/test_autopkgtest.py

@@ -48,10 +48,6 @@ class T(TestBase):
         for line in fileinput.input(self.britney_conf, inplace=True):
             if 'ADT_AMQP' in line:
                 print('ADT_AMQP = file://%s' % self.fake_amqp)
-            elif 'ADT_SWIFT_URL' in line:
-                print('ADT_SWIFT_URL = http://localhost:18085')
-            elif 'ADT_ARCHES' in line:
-                print('ADT_ARCHES = amd64 i386')
             else:
                 sys.stdout.write(line)
 
@@ -2394,5 +2390,74 @@ class T(TestBase):
 ###        self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': ppa, 'green': ppa})
 
 
+    def test_swift_url_is_file(self):
+        '''Run without swift but with debci file (as Debian does)'''
+        '''Based on test_multi_rdepends_with_tests_regression'''
+        '''Multiple reverse dependencies with tests (regression)'''
+
+        debci_file = os.path.join(self.data.path, 'debci.output')
+
+        # Don't use swift but debci output file
+        for line in fileinput.input(self.britney_conf, inplace=True):
+            if line.startswith('ADT_SWIFT_URL'):
+                print('ADT_SWIFT_URL     = file://%s' % debci_file)
+            else:
+                sys.stdout.write(line)
+
+        with open(debci_file, 'w') as f:
+            f.write('''
+[
+  ["green/2", "darkgreen",  "i386",  "1", true,  "20170917_100000"],
+  ["green/2", "darkgreen",  "amd64", "1", true,  "20170917_100000"],
+  ["green/1", "lightgreen", "i386",  "1", true,  "20170917_101000"],
+  ["green/2", "lightgreen", "i386",  "1", false, "20170917_101001"],
+  ["green/1", "lightgreen", "amd64", "1", true,  "20170917_101000"],
+  ["green/2", "lightgreen", "amd64", "1", false, "20170917_101001"],
+  ["green/2", "green",      "i386",  "2", true,  "20170917_102000"],
+  ["green/1", "green",      "amd64", "2", true,  "20170917_102000"],
+  ["green/2", "green",      "amd64", "2", false, "20170917_102001"]
+]
+''')
+
+        self.data.add_default_packages(green=False)
+
+        out, exc, _ = self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
+            {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'},
+                               'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'},
+                               'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'},
+                              })
+            },
+            {'green': [('old-version', '1'), ('new-version', '2')]}
+        )
+
+        # should have links to log and history, but no artifacts (as this is
+        # not a PPA)
+        self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][0],
+                         'REGRESSION')
+        link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][1])
+        self.assertEqual(link.path, os.path.join(debci_file, 'autopkgtest-testing/testing/amd64/l/lightgreen/20170917_101001/log.gz'))
+        self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][2:4],
+                         ['http://autopkgtest.ubuntu.com/packages/l/lightgreen/testing/amd64',
+                          None])
+
+        # should have retry link for the regressions (not a stable URL, test
+        # separately)
+        link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][4])
+        self.assertEqual(link.netloc, 'autopkgtest.ubuntu.com')
+        self.assertEqual(link.path, '/request.cgi')
+        self.assertEqual(urllib.parse.parse_qs(link.query),
+                         {'release': ['testing'], 'arch': ['amd64'],
+                          'package': ['lightgreen'], 'trigger': ['green/2']})
+
+        # we already had all results before the run, so this should not trigger
+        # any new requests
+        self.assertEqual(self.amqp_requests, set())
+        self.assertEqual(self.pending_requests, {})
+
+        # not expecting any failures to retrieve from swift
+        self.assertNotIn('Failure', out, out)
+
+
 if __name__ == '__main__':
     unittest.main()