Initial attempt on fetching private results from swift.

britney.conf

@@ -102,6 +102,12 @@ ADT_SHARED_RESULTS_CACHE =
 # Swift base URL with the results (must be publicly readable and browsable)
 # or file location if results are pre-fetched
 ADT_SWIFT_URL     = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac
+# Swift username that should have exclusive read rights to the test results
+# (this is required whenever a private PPA is used for testing)
+ADT_SWIFT_USER    = 
+ADT_SWIFT_PASS    = 
+ADT_SWIFT_AUTH_URL = 
+ADT_SWIFT_TENANT  = 
 # Base URL for autopkgtest site, used for links in the excuses
 ADT_CI_URL        = https://autopkgtest.ubuntu.com/
 ADT_HUGE          = 20

britney.conf.template

@@ -124,6 +124,12 @@ ADT_SHARED_RESULTS_CACHE =
 # or file location if results are pre-fetched
 #ADT_SWIFT_URL     = https://example.com/some/url
 ADT_SWIFT_URL     = file:///path/to/britney/state/debci.json
+# Swift username that should have exclusive read rights to the test results
+# (this is required whenever a private PPA is used for testing)
+ADT_SWIFT_USER    = 
+ADT_SWIFT_PASS    = 
+ADT_SWIFT_AUTH_URL = 
+ADT_SWIFT_TENANT  = 
 # Base URL for autopkgtest site, used for links in the excuses
 ADT_CI_URL        = https://example.com/
 # Enable the huge queue for packages that trigger vast amounts of tests to not

britney2/policies/autopkgtest.py

@@ -197,6 +197,33 @@ class AutopkgtestPolicy(BasePolicy):
         else:
             self.logger.info('%s does not exist, re-downloading all results from swift', self.results_cache_file)
 
+        # log into swift in case we need to fetch some private results
+        # this is optional - if there are no credentials present, results will
+        # be fetched the traditional way
+        if self.options.adt_swift_user:
+            if (not self.options.adt_swift_pass or
+                    not self.options.adt_swift_auth_url or
+                    not self.options.adt_swift_tenant):
+                raise RuntimeError('Incomplete swift credentials given')
+
+            import swiftclient
+
+            if '/v2.0' in self.options.adt_swift_auth_url:
+                auth_version = '2.0'
+            else:
+                auth_version = '1'
+
+            self.logger.info('Creating an authenticated swift connection for user %s', self.adt_swift_user)
+            self.swift_conn = swiftclient.Connection(
+                authurl=self.options.adt_swift_auth_url,
+                user=self.options.adt_swift_user,
+                key=self.options.adt_swift_pass,
+                tenant_name=self.options.adt_swift_tenant,
+                auth_version=auth_version
+                )
+        else:
+            self.swift_conn = None
+
         # read in the new results
         if self.options.adt_swift_url.startswith('file://'):
             debci_file = self.options.adt_swift_url[7:]
@@ -850,55 +877,94 @@ class AutopkgtestPolicy(BasePolicy):
                 query['marker'] = query['prefix'] + latest_run_id
 
         # request new results from swift
-        url = os.path.join(swift_url, self.swift_container)
+        if self.swift_conn:
-        url += '?' + urllib.parse.urlencode(query)
+            # when we have an authenticated swift connection, use that to
-        f = None
+            # fetch the result_path list as we might be fetching from an
-        try:
+            # otherwise unaccessible container
-            f = self.download_retry(url)
+            from swiftclient.exceptions import ClientException
-            if f.getcode() == 200:
+
-                result_paths = f.read().decode().strip().splitlines()
+            try:
-            elif f.getcode() == 204:  # No content
+                _, result_paths = self.swift_conn.get_container(
-                result_paths = []
+                    self.swift_conn.url,
-            else:
+                    token=self.swift_conn.token,
-                # we should not ever end up here as we expect a HTTPError in
+                    container=self.swift_container,
-                # other cases; e. g. 3XX is something that tells us to adjust
+                    query_string=urllib.parse.urlencode(query))
-                # our URLS, so fail hard on those
+            except ClientException as e:
-                raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' %
+                # 401 "Unauthorized" is swift's way of saying "container does not exist"
-                                          (url, f.getcode()))
+                if e.http_code == 401 or e.http_code == 404:
-        except IOError as e:
+                    self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
-            # 401 "Unauthorized" is swift's way of saying "container does not exist"
+                    return
-            if hasattr(e, 'code') and e.code == 401:
+                # Other status codes are usually a transient
-                self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
+                # network/infrastructure failure. Ignoring this can lead to
-                return
+                # re-requesting tests which we already have results for, so
-            # Other status codes are usually a transient
+                # fail hard on this and let the next run retry.
-            # network/infrastructure failure. Ignoring this can lead to
+                self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
-            # re-requesting tests which we already have results for, so
+                sys.exit(1)
-            # fail hard on this and let the next run retry.
+        else:
-            self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
+            url = os.path.join(swift_url, self.swift_container)
-            sys.exit(1)
+            url += '?' + urllib.parse.urlencode(query)
-        finally:
+            f = None
-            if f is not None:
+            try:
-                f.close()
+                f = self.download_retry(url)
+                if f.getcode() == 200:
+                    result_paths = f.read().decode().strip().splitlines()
+                elif f.getcode() == 204:  # No content
+                    result_paths = []
+                else:
+                    # we should not ever end up here as we expect a HTTPError in
+                    # other cases; e. g. 3XX is something that tells us to adjust
+                    # our URLS, so fail hard on those
+                    raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' %
+                                              (url, f.getcode()))
+            except IOError as e:
+                # 401 "Unauthorized" is swift's way of saying "container does not exist"
+                if hasattr(e, 'code') and e.code == 401:
+                    self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
+                    return
+                # same as above in the swift authenticated case
+                self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
+                sys.exit(1)
+            finally:
+                if f is not None:
+                    f.close()
 
         for p in result_paths:
             self.fetch_one_result(
-                os.path.join(swift_url, self.swift_container, p, 'result.tar'), src, arch)
+                swift_url, self.swift_container, p, 'result.tar', src, arch)
 
     fetch_swift_results._done = set()
 
-    def fetch_one_result(self, url, src, arch):
+    def fetch_one_result(self, swift_url, container, path, name, src, arch):
         '''Download one result URL for source/arch
 
         Remove matching pending_tests entries.
         '''
+
         f = None
         try:
-            f = self.download_retry(url)
+            if self.swift_conn:
-            if f.getcode() == 200:
+                from swiftclient.exceptions import ClientException
-                tar_bytes = io.BytesIO(f.read())
+
+                # We don't need any additional retry logic as swiftclient
+                # already performs retries (5 by default).
+                full_path = os.path.join(path, name)
+                try:
+                    _, contents = swift_conn.get_object(container, full_path)
+                except ClientException as e:
+                    self.logger.error('Failure to fetch %s from container %s: %s',
+                                      full_path, container, str(e))
+                    if e.http_code == 404:
+                        return
+                    sys.exit(1)
+                tar_bytes = io.BytesIO(contents)
             else:
-                raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' %
+                url = os.path.join(swift_url, container, path, name)
-                                          (url, f.getcode()))
+                f = self.download_retry(url)
+                if f.getcode() == 200:
+                    tar_bytes = io.BytesIO(f.read())
+                else:
+                    raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' %
+                                              (url, f.getcode()))
         except IOError as e:
             self.logger.error('Failure to fetch %s: %s', url, str(e))
             # we tolerate "not found" (something went wrong on uploading the
@@ -1016,6 +1082,8 @@ class AutopkgtestPolicy(BasePolicy):
 
         params = {'triggers': triggers}
         if self.options.adt_ppas:
+            # Note: the PPA might be a private PPA, and then the PPA parameter
+            # includes the authorization token.
             params['ppas'] = self.options.adt_ppas
             qname = 'debci-ppa-%s-%s' % (self.options.series, arch)
         elif huge:
@@ -1024,6 +1092,9 @@ class AutopkgtestPolicy(BasePolicy):
             qname = 'debci-%s-%s' % (self.options.series, arch)
         params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z')
 
+        if self.swift_conn:
+            params['readable-by'] = self.adt_swift_user
+
         if self.amqp_channel:
             import amqplib.client_0_8 as amqp
             params = json.dumps(params)

tests/__init__.py

@@ -395,6 +395,10 @@ ADT_PPAS          =
 ADT_SHARED_RESULTS_CACHE =
 
 ADT_SWIFT_URL     = http://localhost:18085
+ADT_SWIFT_USER    = 
+ADT_SWIFT_PASS    = 
+ADT_SWIFT_AUTH_URL = 
+ADT_SWIFT_TENANT  = 
 ADT_CI_URL        = https://autopkgtest.ubuntu.com/
 ADT_HUGE          = 20
 

tests/test_policy.py

@@ -43,6 +43,10 @@ def initialize_policy(test_name, policy_class, *args, **kwargs):
         architectures=ARCH,
         adt_swift_url='file://' + debci_data,
         adt_ci_url='',
+        adt_swift_user='',
+        adt_swift_pass='',
+        adt_swift_tenant='',
+        adt_swift_auth_url='',
         adt_success_bounty=3,
         adt_regression_penalty=False,
         adt_retry_url_mech='run_id',