diff --git a/britney.conf b/britney.conf index 74ecd4b..8f1f35c 100644 --- a/britney.conf +++ b/britney.conf @@ -102,6 +102,12 @@ ADT_SHARED_RESULTS_CACHE = # Swift base URL with the results (must be publicly readable and browsable) # or file location if results are pre-fetched ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac +# Swift username that should have exclusive read rights to the test results +# (this is required whenever a private PPA is used for testing) +ADT_SWIFT_USER = +ADT_SWIFT_PASS = +ADT_SWIFT_AUTH_URL = +ADT_SWIFT_TENANT = # Base URL for autopkgtest site, used for links in the excuses ADT_CI_URL = https://autopkgtest.ubuntu.com/ ADT_HUGE = 20 diff --git a/britney.conf.template b/britney.conf.template index b1df068..3c56d16 100644 --- a/britney.conf.template +++ b/britney.conf.template @@ -124,6 +124,12 @@ ADT_SHARED_RESULTS_CACHE = # or file location if results are pre-fetched #ADT_SWIFT_URL = https://example.com/some/url ADT_SWIFT_URL = file:///path/to/britney/state/debci.json +# Swift username that should have exclusive read rights to the test results +# (this is required whenever a private PPA is used for testing) +ADT_SWIFT_USER = +ADT_SWIFT_PASS = +ADT_SWIFT_AUTH_URL = +ADT_SWIFT_TENANT = # Base URL for autopkgtest site, used for links in the excuses ADT_CI_URL = https://example.com/ # Enable the huge queue for packages that trigger vast amounts of tests to not diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py index 40ab279..e59d931 100644 --- a/britney2/policies/autopkgtest.py +++ b/britney2/policies/autopkgtest.py @@ -197,6 +197,33 @@ class AutopkgtestPolicy(BasePolicy): else: self.logger.info('%s does not exist, re-downloading all results from swift', self.results_cache_file) + # log into swift in case we need to fetch some private results + # this is optional - if there are no credentials present, results will + # be fetched the traditional way + if self.options.adt_swift_user: + if (not self.options.adt_swift_pass or + not self.options.adt_swift_auth_url or + not self.options.adt_swift_tenant): + raise RuntimeError('Incomplete swift credentials given') + + import swiftclient + + if '/v2.0' in self.options.adt_swift_auth_url: + auth_version = '2.0' + else: + auth_version = '1' + + self.logger.info('Creating an authenticated swift connection for user %s', self.adt_swift_user) + self.swift_conn = swiftclient.Connection( + authurl=self.options.adt_swift_auth_url, + user=self.options.adt_swift_user, + key=self.options.adt_swift_pass, + tenant_name=self.options.adt_swift_tenant, + auth_version=auth_version + ) + else: + self.swift_conn = None + # read in the new results if self.options.adt_swift_url.startswith('file://'): debci_file = self.options.adt_swift_url[7:] @@ -850,55 +877,94 @@ class AutopkgtestPolicy(BasePolicy): query['marker'] = query['prefix'] + latest_run_id # request new results from swift - url = os.path.join(swift_url, self.swift_container) - url += '?' + urllib.parse.urlencode(query) - f = None - try: - f = self.download_retry(url) - if f.getcode() == 200: - result_paths = f.read().decode().strip().splitlines() - elif f.getcode() == 204: # No content - result_paths = [] - else: - # we should not ever end up here as we expect a HTTPError in - # other cases; e. g. 3XX is something that tells us to adjust - # our URLS, so fail hard on those - raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' % - (url, f.getcode())) - except IOError as e: - # 401 "Unauthorized" is swift's way of saying "container does not exist" - if hasattr(e, 'code') and e.code == 401: - self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url) - return - # Other status codes are usually a transient - # network/infrastructure failure. Ignoring this can lead to - # re-requesting tests which we already have results for, so - # fail hard on this and let the next run retry. - self.logger.error('Failure to fetch swift results from %s: %s', url, str(e)) - sys.exit(1) - finally: - if f is not None: - f.close() + if self.swift_conn: + # when we have an authenticated swift connection, use that to + # fetch the result_path list as we might be fetching from an + # otherwise unaccessible container + from swiftclient.exceptions import ClientException + + try: + _, result_paths = self.swift_conn.get_container( + self.swift_conn.url, + token=self.swift_conn.token, + container=self.swift_container, + query_string=urllib.parse.urlencode(query)) + except ClientException as e: + # 401 "Unauthorized" is swift's way of saying "container does not exist" + if e.http_code == 401 or e.http_code == 404: + self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url) + return + # Other status codes are usually a transient + # network/infrastructure failure. Ignoring this can lead to + # re-requesting tests which we already have results for, so + # fail hard on this and let the next run retry. + self.logger.error('Failure to fetch swift results from %s: %s', url, str(e)) + sys.exit(1) + else: + url = os.path.join(swift_url, self.swift_container) + url += '?' + urllib.parse.urlencode(query) + f = None + try: + f = self.download_retry(url) + if f.getcode() == 200: + result_paths = f.read().decode().strip().splitlines() + elif f.getcode() == 204: # No content + result_paths = [] + else: + # we should not ever end up here as we expect a HTTPError in + # other cases; e. g. 3XX is something that tells us to adjust + # our URLS, so fail hard on those + raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' % + (url, f.getcode())) + except IOError as e: + # 401 "Unauthorized" is swift's way of saying "container does not exist" + if hasattr(e, 'code') and e.code == 401: + self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url) + return + # same as above in the swift authenticated case + self.logger.error('Failure to fetch swift results from %s: %s', url, str(e)) + sys.exit(1) + finally: + if f is not None: + f.close() for p in result_paths: self.fetch_one_result( - os.path.join(swift_url, self.swift_container, p, 'result.tar'), src, arch) + swift_url, self.swift_container, p, 'result.tar', src, arch) fetch_swift_results._done = set() - def fetch_one_result(self, url, src, arch): + def fetch_one_result(self, swift_url, container, path, name, src, arch): '''Download one result URL for source/arch Remove matching pending_tests entries. ''' + f = None try: - f = self.download_retry(url) - if f.getcode() == 200: - tar_bytes = io.BytesIO(f.read()) + if self.swift_conn: + from swiftclient.exceptions import ClientException + + # We don't need any additional retry logic as swiftclient + # already performs retries (5 by default). + full_path = os.path.join(path, name) + try: + _, contents = swift_conn.get_object(container, full_path) + except ClientException as e: + self.logger.error('Failure to fetch %s from container %s: %s', + full_path, container, str(e)) + if e.http_code == 404: + return + sys.exit(1) + tar_bytes = io.BytesIO(contents) else: - raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' % - (url, f.getcode())) + url = os.path.join(swift_url, container, path, name) + f = self.download_retry(url) + if f.getcode() == 200: + tar_bytes = io.BytesIO(f.read()) + else: + raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' % + (url, f.getcode())) except IOError as e: self.logger.error('Failure to fetch %s: %s', url, str(e)) # we tolerate "not found" (something went wrong on uploading the @@ -1016,6 +1082,8 @@ class AutopkgtestPolicy(BasePolicy): params = {'triggers': triggers} if self.options.adt_ppas: + # Note: the PPA might be a private PPA, and then the PPA parameter + # includes the authorization token. params['ppas'] = self.options.adt_ppas qname = 'debci-ppa-%s-%s' % (self.options.series, arch) elif huge: @@ -1024,6 +1092,9 @@ class AutopkgtestPolicy(BasePolicy): qname = 'debci-%s-%s' % (self.options.series, arch) params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z') + if self.swift_conn: + params['readable-by'] = self.adt_swift_user + if self.amqp_channel: import amqplib.client_0_8 as amqp params = json.dumps(params) diff --git a/tests/__init__.py b/tests/__init__.py index c6cd09d..a036b94 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -395,6 +395,10 @@ ADT_PPAS = ADT_SHARED_RESULTS_CACHE = ADT_SWIFT_URL = http://localhost:18085 +ADT_SWIFT_USER = +ADT_SWIFT_PASS = +ADT_SWIFT_AUTH_URL = +ADT_SWIFT_TENANT = ADT_CI_URL = https://autopkgtest.ubuntu.com/ ADT_HUGE = 20 diff --git a/tests/test_policy.py b/tests/test_policy.py index 1952831..553e1b0 100644 --- a/tests/test_policy.py +++ b/tests/test_policy.py @@ -43,6 +43,10 @@ def initialize_policy(test_name, policy_class, *args, **kwargs): architectures=ARCH, adt_swift_url='file://' + debci_data, adt_ci_url='', + adt_swift_user='', + adt_swift_pass='', + adt_swift_tenant='', + adt_swift_auth_url='', adt_success_bounty=3, adt_regression_penalty=False, adt_retry_url_mech='run_id',