Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-02-23 11:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial attempt on fetching private results from swift.

Browse Source
This commit is contained in:
Łukasz 'sil2100' Zemczak 2021-03-03 14:32:08 +01:00
parent fd48bfe54b
commit 7b20866238
5 changed files with 127 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
britney.conf
View File

@ -102,6 +102,12 @@ ADT_SHARED_RESULTS_CACHE =
# Swift base URL with the results (must be publicly readable and browsable)
# or file location if results are pre-fetched
ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac
# Swift username that should have exclusive read rights to the test results
# (this is required whenever a private PPA is used for testing)
ADT_SWIFT_USER =
ADT_SWIFT_PASS =
ADT_SWIFT_AUTH_URL =
ADT_SWIFT_TENANT =
# Base URL for autopkgtest site, used for links in the excuses
ADT_CI_URL = https://autopkgtest.ubuntu.com/
ADT_HUGE = 20

6
britney.conf.template
View File

@ -124,6 +124,12 @@ ADT_SHARED_RESULTS_CACHE =
# or file location if results are pre-fetched
#ADT_SWIFT_URL = https://example.com/some/url
ADT_SWIFT_URL = file:///path/to/britney/state/debci.json
# Swift username that should have exclusive read rights to the test results
# (this is required whenever a private PPA is used for testing)
ADT_SWIFT_USER =
ADT_SWIFT_PASS =
ADT_SWIFT_AUTH_URL =
ADT_SWIFT_TENANT =
# Base URL for autopkgtest site, used for links in the excuses
ADT_CI_URL = https://example.com/
# Enable the huge queue for packages that trigger vast amounts of tests to not

143
britney2/policies/autopkgtest.py
View File

@ -197,6 +197,33 @@ class AutopkgtestPolicy(BasePolicy):
else:
self.logger.info('%s does not exist, re-downloading all results from swift', self.results_cache_file)
# log into swift in case we need to fetch some private results
# this is optional - if there are no credentials present, results will
# be fetched the traditional way
if self.options.adt_swift_user:
if (not self.options.adt_swift_pass or
not self.options.adt_swift_auth_url or
not self.options.adt_swift_tenant):
raise RuntimeError('Incomplete swift credentials given')
import swiftclient
if '/v2.0' in self.options.adt_swift_auth_url:
auth_version = '2.0'
else:
auth_version = '1'
self.logger.info('Creating an authenticated swift connection for user %s', self.adt_swift_user)
self.swift_conn = swiftclient.Connection(
authurl=self.options.adt_swift_auth_url,
user=self.options.adt_swift_user,
key=self.options.adt_swift_pass,
tenant_name=self.options.adt_swift_tenant,
auth_version=auth_version
)
else:
self.swift_conn = None
# read in the new results
if self.options.adt_swift_url.startswith('file://'):
debci_file = self.options.adt_swift_url[7:]
@ -850,55 +877,94 @@ class AutopkgtestPolicy(BasePolicy):
query['marker'] = query['prefix'] + latest_run_id
# request new results from swift
url = os.path.join(swift_url, self.swift_container)
url += '?' + urllib.parse.urlencode(query)
f = None
try:
f = self.download_retry(url)
if f.getcode() == 200:
result_paths = f.read().decode().strip().splitlines()
elif f.getcode() == 204: # No content
result_paths = []
else:
# we should not ever end up here as we expect a HTTPError in
# other cases; e. g. 3XX is something that tells us to adjust
# our URLS, so fail hard on those
raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' %
(url, f.getcode()))
except IOError as e:
# 401 "Unauthorized" is swift's way of saying "container does not exist"
if hasattr(e, 'code') and e.code == 401:
self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
return
# Other status codes are usually a transient
# network/infrastructure failure. Ignoring this can lead to
# re-requesting tests which we already have results for, so
# fail hard on this and let the next run retry.
self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
sys.exit(1)
finally:
if f is not None:
f.close()
if self.swift_conn:
# when we have an authenticated swift connection, use that to
# fetch the result_path list as we might be fetching from an
# otherwise unaccessible container
from swiftclient.exceptions import ClientException
try:
_, result_paths = self.swift_conn.get_container(
self.swift_conn.url,
token=self.swift_conn.token,
container=self.swift_container,
query_string=urllib.parse.urlencode(query))
except ClientException as e:
# 401 "Unauthorized" is swift's way of saying "container does not exist"
if e.http_code == 401 or e.http_code == 404:
self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
return
# Other status codes are usually a transient
# network/infrastructure failure. Ignoring this can lead to
# re-requesting tests which we already have results for, so
# fail hard on this and let the next run retry.
self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
sys.exit(1)
else:
url = os.path.join(swift_url, self.swift_container)
url += '?' + urllib.parse.urlencode(query)
f = None
try:
f = self.download_retry(url)
if f.getcode() == 200:
result_paths = f.read().decode().strip().splitlines()
elif f.getcode() == 204: # No content
result_paths = []
else:
# we should not ever end up here as we expect a HTTPError in
# other cases; e. g. 3XX is something that tells us to adjust
# our URLS, so fail hard on those
raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' %
(url, f.getcode()))
except IOError as e:
# 401 "Unauthorized" is swift's way of saying "container does not exist"
if hasattr(e, 'code') and e.code == 401:
self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
return
# same as above in the swift authenticated case
self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
sys.exit(1)
finally:
if f is not None:
f.close()
for p in result_paths:
self.fetch_one_result(
os.path.join(swift_url, self.swift_container, p, 'result.tar'), src, arch)
swift_url, self.swift_container, p, 'result.tar', src, arch)
fetch_swift_results._done = set()
def fetch_one_result(self, url, src, arch):
def fetch_one_result(self, swift_url, container, path, name, src, arch):
'''Download one result URL for source/arch
Remove matching pending_tests entries.
'''
f = None
try:
f = self.download_retry(url)
if f.getcode() == 200:
tar_bytes = io.BytesIO(f.read())
if self.swift_conn:
from swiftclient.exceptions import ClientException
# We don't need any additional retry logic as swiftclient
# already performs retries (5 by default).
full_path = os.path.join(path, name)
try:
_, contents = swift_conn.get_object(container, full_path)
except ClientException as e:
self.logger.error('Failure to fetch %s from container %s: %s',
full_path, container, str(e))
if e.http_code == 404:
return
sys.exit(1)
tar_bytes = io.BytesIO(contents)
else:
raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' %
(url, f.getcode()))
url = os.path.join(swift_url, container, path, name)
f = self.download_retry(url)
if f.getcode() == 200:
tar_bytes = io.BytesIO(f.read())
else:
raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' %
(url, f.getcode()))
except IOError as e:
self.logger.error('Failure to fetch %s: %s', url, str(e))
# we tolerate "not found" (something went wrong on uploading the
@ -1016,6 +1082,8 @@ class AutopkgtestPolicy(BasePolicy):
params = {'triggers': triggers}
if self.options.adt_ppas:
# Note: the PPA might be a private PPA, and then the PPA parameter
# includes the authorization token.
params['ppas'] = self.options.adt_ppas
qname = 'debci-ppa-%s-%s' % (self.options.series, arch)
elif huge:
@ -1024,6 +1092,9 @@ class AutopkgtestPolicy(BasePolicy):
qname = 'debci-%s-%s' % (self.options.series, arch)
params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z')
if self.swift_conn:
params['readable-by'] = self.adt_swift_user
if self.amqp_channel:
import amqplib.client_0_8 as amqp
params = json.dumps(params)

4
tests/__init__.py
View File

@ -395,6 +395,10 @@ ADT_PPAS =
ADT_SHARED_RESULTS_CACHE =
ADT_SWIFT_URL = http://localhost:18085
ADT_SWIFT_USER =
ADT_SWIFT_PASS =
ADT_SWIFT_AUTH_URL =
ADT_SWIFT_TENANT =
ADT_CI_URL = https://autopkgtest.ubuntu.com/
ADT_HUGE = 20

4
tests/test_policy.py
View File

@ -43,6 +43,10 @@ def initialize_policy(test_name, policy_class, *args, **kwargs):
architectures=ARCH,
adt_swift_url='file://' + debci_data,
adt_ci_url='',
adt_swift_user='',
adt_swift_pass='',
adt_swift_tenant='',
adt_swift_auth_url='',
adt_success_bounty=3,
adt_regression_penalty=False,
adt_retry_url_mech='run_id',