Lubuntu
/
britney2-ubuntu
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch from swiftclient for private PPAs to using HTTP with X-Auth-Token instead.

Browse Source 
This way there's less secrets that need to be shared and less new code to introduce. We also modified the test tooling to be able to check for authentication tokens in the queries.
sil2100/private-runs
Łukasz 'sil2100' Zemczak 4 years ago
parent d1549b9ca9
commit 7ff150ced7
8 changed files with 76 additions and 214 deletions
Show Stats Download Patch File Download Diff File

6
britney.conf
Unescape View File

@ -102,12 +102,10 @@ ADT_SHARED_RESULTS_CACHE =
# Swift base URL with the results (must be publicly readable and browsable) # Swift base URL with the results (must be publicly readable and browsable)
# or file location if results are pre-fetched # or file location if results are pre-fetched
ADT_SWIFT_URL = https://autopkgtest.ubuntu.com/results/ ADT_SWIFT_URL = https://autopkgtest.ubuntu.com/results/
# Swift identity with read access to the private result container # Swift authentication user and token for access to private containers
# (this is required whenever a private PPA is used for testing) # (this is required whenever a private PPA is used for testing)
ADT_SWIFT_USER = ADT_SWIFT_USER =
ADT_SWIFT_PASS = ADT_SWIFT_TOKEN =
ADT_SWIFT_AUTH_URL =
ADT_SWIFT_TENANT =
# List of launchpad users/teams that should have read access to any private # List of launchpad users/teams that should have read access to any private
# result logs # result logs
ADT_PRIVATE_SHARED = ADT_PRIVATE_SHARED =

6
britney.conf.template
Unescape View File

@ -124,12 +124,10 @@ ADT_SHARED_RESULTS_CACHE =
# or file location if results are pre-fetched # or file location if results are pre-fetched
#ADT_SWIFT_URL = https://example.com/some/url #ADT_SWIFT_URL = https://example.com/some/url
ADT_SWIFT_URL = file:///path/to/britney/state/debci.json ADT_SWIFT_URL = file:///path/to/britney/state/debci.json
# Swift identity with read access to the private result container # Swift authentication user and token for access to private containers
# (this is required whenever a private PPA is used for testing) # (this is required whenever a private PPA is used for testing)
ADT_SWIFT_USER = ADT_SWIFT_USER =
ADT_SWIFT_PASS = ADT_SWIFT_TOKEN =
ADT_SWIFT_AUTH_URL =
ADT_SWIFT_TENANT =
# List of launchpad users/teams that should have read access to any private # List of launchpad users/teams that should have read access to any private
# result logs # result logs
ADT_PRIVATE_SHARED = ADT_PRIVATE_SHARED =

80
britney2/policies/autopkgtest.py
Unescape View File

@ -32,7 +32,7 @@ import sys
import time import time
import urllib.parse import urllib.parse
from urllib.error import HTTPError from urllib.error import HTTPError
from urllib.request import urlopen from urllib.request import Request, urlopen
import apt_pkg import apt_pkg
@ -208,11 +208,9 @@ class AutopkgtestPolicy(BasePolicy):
# log into swift in case we need to fetch some private results # log into swift in case we need to fetch some private results
# this is optional - if there are no credentials present, results will # this is optional - if there are no credentials present, results will
# be fetched without authentication # be fetched without authentication
if self.options.adt_swift_user: if self.options.adt_swift_token:
if (not self.options.adt_swift_pass or if self.options.adt_swift_url.startswith('file://'):
not self.options.adt_swift_auth_url or raise RuntimeError('Token authentication requires swift remote storage')
not self.options.adt_swift_tenant):
raise RuntimeError('Incomplete swift credentials given')
# once swift credentials are given, the results will be published # once swift credentials are given, the results will be published
# to a private container # to a private container
@ -225,28 +223,10 @@ class AutopkgtestPolicy(BasePolicy):
# TODO: write a test for this # TODO: write a test for this
if '@' in ppa and not re.match(r'^.+:.+@.+:.+$', ppa): if '@' in ppa and not re.match(r'^.+:.+@.+:.+$', ppa):
raise RuntimeError('Private PPA %s not following required format (user:token@team/name:fingerprint)', ppa) raise RuntimeError('Private PPA %s not following required format (user:token@team/name:fingerprint)', ppa)
import swiftclient
if '/v2.0' in self.options.adt_swift_auth_url:
auth_version = '2.0'
else:
auth_version = '1'
self.logger.info('Creating an authenticated swift connection for user %s', self.options.adt_swift_user)
self.swift_conn = swiftclient.Connection(
authurl=self.options.adt_swift_auth_url,
user=self.options.adt_swift_user,
key=self.options.adt_swift_pass,
tenant_name=self.options.adt_swift_tenant,
auth_version=auth_version
)
else: else:
if any('@' in ppa for ppa in self.options.adt_ppas): if any('@' in ppa for ppa in self.options.adt_ppas):
raise RuntimeError('Private PPA configured but no swift credentials given') raise RuntimeError('Private PPA configured but no swift credentials given')
self.swift_conn = None
# read in the new results # read in the new results
if self.options.adt_swift_url.startswith('file://'): if self.options.adt_swift_url.startswith('file://'):
debci_file = self.options.adt_swift_url[7:] debci_file = self.options.adt_swift_url[7:]
@ -855,10 +835,12 @@ class AutopkgtestPolicy(BasePolicy):
latest_run_for_package._cache = collections.defaultdict(dict) latest_run_for_package._cache = collections.defaultdict(dict)
def download_retry(self, url): def download_retry(self, url, token=None):
headers = {'X-Auth-Token': token} if token else {}
request = Request(url, headers=headers)
for retry in range(5): for retry in range(5):
try: try:
req = urlopen(url, timeout=30) req = urlopen(request, timeout=30)
code = req.getcode() code = req.getcode()
if 200 <= code < 300: if 200 <= code < 300:
return req return req
@ -903,33 +885,11 @@ class AutopkgtestPolicy(BasePolicy):
query['marker'] = query['prefix'] + latest_run_id query['marker'] = query['prefix'] + latest_run_id
# request new results from swift # request new results from swift
if self.swift_conn:
# when we have an authenticated swift connection, use that to
# fetch the result_path list as we might be fetching from an
# otherwise unaccessible container
from swiftclient.exceptions import ClientException
try:
_, result_paths = self.swift_conn.get_container(
self.swift_container,
query_string=urllib.parse.urlencode(query))
except ClientException as e:
# 401 "Unauthorized" is swift's way of saying "container does not exist"
if e.http_status == '401' or e.http_status == '404':
self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', self.swift_container)
return
# Other status codes are usually a transient
# network/infrastructure failure. Ignoring this can lead to
# re-requesting tests which we already have results for, so
# fail hard on this and let the next run retry.
self.logger.error('Failure to fetch swift results from %s: %s', self.swift_container, str(e))
sys.exit(1)
else:
url = os.path.join(swift_url, self.swift_container) url = os.path.join(swift_url, self.swift_container)
url += '?' + urllib.parse.urlencode(query) url += '?' + urllib.parse.urlencode(query)
f = None f = None
try: try:
f = self.download_retry(url) f = self.download_retry(url, self.options.adt_swift_token)
if f.getcode() == 200: if f.getcode() == 200:
result_paths = f.read().decode().strip().splitlines() result_paths = f.read().decode().strip().splitlines()
elif f.getcode() == 204: # No content elif f.getcode() == 204: # No content
@ -966,24 +926,8 @@ class AutopkgtestPolicy(BasePolicy):
f = None f = None
try: try:
if self.swift_conn:
from swiftclient.exceptions import ClientException
# We don't need any additional retry logic as swiftclient
# already performs retries (5 by default).
url = os.path.join(path, name)
try:
_, contents = self.swift_conn.get_object(container, url)
except ClientException as e:
self.logger.error('Failure to fetch %s from container %s: %s',
url, container, str(e))
if e.http_status == '404':
return
sys.exit(1)
tar_bytes = io.BytesIO(contents)
else:
url = os.path.join(swift_url, container, path, name) url = os.path.join(swift_url, container, path, name)
f = self.download_retry(url) f = self.download_retry(url, self.options.adt_swift_token)
if f.getcode() == 200: if f.getcode() == 200:
tar_bytes = io.BytesIO(f.read()) tar_bytes = io.BytesIO(f.read())
else: else:
@ -1116,7 +1060,7 @@ class AutopkgtestPolicy(BasePolicy):
qname = 'debci-%s-%s' % (self.options.series, arch) qname = 'debci-%s-%s' % (self.options.series, arch)
params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z') params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z')
if self.swift_conn: if self.options.adt_swift_user:
params['swiftuser'] = self.options.adt_swift_user params['swiftuser'] = self.options.adt_swift_user
if self.options.adt_private_shared: if self.options.adt_private_shared:
params['readable-by'] = self.options.adt_private_shared params['readable-by'] = self.options.adt_private_shared
@ -1329,7 +1273,7 @@ class AutopkgtestPolicy(BasePolicy):
'log.gz') 'log.gz')
else: else:
# Private runs have a different base url # Private runs have a different base url
results_url = self.options.adt_private_url if self.swift_conn else self.options.adt_swift_url results_url = self.options.adt_private_url if self.options.adt_swift_user else self.options.adt_swift_url
url = os.path.join(results_url, url = os.path.join(results_url,
self.swift_container, self.swift_container,
self.options.series, self.options.series,

4
tests/__init__.py
Unescape View File

@ -396,9 +396,7 @@ ADT_SHARED_RESULTS_CACHE =
ADT_SWIFT_URL = http://localhost:18085 ADT_SWIFT_URL = http://localhost:18085
ADT_SWIFT_USER = ADT_SWIFT_USER =
ADT_SWIFT_PASS = ADT_SWIFT_TOKEN =
ADT_SWIFT_AUTH_URL =
ADT_SWIFT_TENANT =
ADT_PRIVATE_SHARED = ADT_PRIVATE_SHARED =
ADT_PRIVATE_URL = ADT_PRIVATE_URL =
ADT_CI_URL = https://autopkgtest.ubuntu.com/ ADT_CI_URL = https://autopkgtest.ubuntu.com/

34
tests/mock_swift.py
Unescape View File

@ -32,6 +32,7 @@ class SwiftHTTPRequestHandler(BaseHTTPRequestHandler):
''' '''
# map container -> result.tar path -> (exitcode, testpkg-version[, testinfo]) # map container -> result.tar path -> (exitcode, testpkg-version[, testinfo])
results = {} results = {}
expected_token = 'SomeSecretToken'
def do_GET(self): def do_GET(self):
p = urlparse(self.path) p = urlparse(self.path)
@ -43,7 +44,22 @@ class SwiftHTTPRequestHandler(BaseHTTPRequestHandler):
else: else:
self.list_container(container, parse_qs(p.query)) self.list_container(container, parse_qs(p.query))
def check_if_authorized(self, container):
status = True
if container.startswith('private-'):
# private container, so we need authentication
token = self.headers.get('X-Auth-Token')
if not token:
self.send_error(401, 'Unauthorized')
status = False
elif self.expected_token != token:
self.send_error(403, 'Forbidden')
status = False
return status
def serve_file(self, container, path): def serve_file(self, container, path):
if not self.check_if_authorized(container):
return
if os.path.basename(path) != 'result.tar': if os.path.basename(path) != 'result.tar':
self.send_error(404, 'File not found (only result.tar supported)') self.send_error(404, 'File not found (only result.tar supported)')
return return
@ -85,6 +101,8 @@ class SwiftHTTPRequestHandler(BaseHTTPRequestHandler):
self.wfile.write(tar.getvalue()) self.wfile.write(tar.getvalue())
def list_container(self, container, query): def list_container(self, container, query):
if not self.check_if_authorized(container):
return
try: try:
objs = set(['%s/result.tar' % r for r in self.results[container]]) objs = set(['%s/result.tar' % r for r in self.results[container]])
except KeyError: except KeyError:
@ -128,15 +146,7 @@ class AutoPkgTestSwiftServer:
''' '''
SwiftHTTPRequestHandler.results = results SwiftHTTPRequestHandler.results = results
def start(self, swiftclient=False): def start(self):
if swiftclient:
# since we're running britney directly, the only way to reliably
# mock out the swiftclient module is to override it in the local
# path with the dummy version we created
src = os.path.join(TESTS_DIR, 'mock_swiftclient.py')
dst = os.path.join(PROJECT_DIR, 'swiftclient.py')
os.symlink(src, dst)
assert self.server_pid is None, 'already started' assert self.server_pid is None, 'already started'
if self.log: if self.log:
self.log.close() self.log.close()
@ -160,12 +170,6 @@ class AutoPkgTestSwiftServer:
sys.exit(0) sys.exit(0)
def stop(self): def stop(self):
# in case we were 'mocking out' swiftclient, remove the symlink we
# created earlier during start()
swiftclient_mod = os.path.join(PROJECT_DIR, 'swiftclient.py')
if os.path.islink(swiftclient_mod):
os.unlink(swiftclient_mod)
assert self.server_pid, 'not running' assert self.server_pid, 'not running'
os.kill(self.server_pid, 15) os.kill(self.server_pid, 15)
os.waitpid(self.server_pid, 0) os.waitpid(self.server_pid, 0)

70
tests/mock_swiftclient.py
Unescape View File

@ -1,70 +0,0 @@
# Mock the swiftclient Python library, the bare minimum for ADT purposes
# Author: Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com>
import os
import sys
from urllib.request import urlopen
# We want to use this single Python module file to mock out the exception
# module as well.
sys.modules["swiftclient.exceptions"] = sys.modules[__name__]
class ClientException(Exception):
def __init__(self, msg, http_status=''):
super(ClientException, self).__init__(msg)
self.msg = msg
self.http_status = http_status
class Connection:
def __init__(self, authurl, user, key, tenant_name, auth_version):
self._mocked_swift = 'http://localhost:18085'
def get_container(self, container, marker=None, limit=None, prefix=None,
delimiter=None, end_marker=None, path=None,
full_listing=False, headers=None, query_string=None):
url = os.path.join(self._mocked_swift, container) + '?' + query_string
req = None
try:
req = urlopen(url, timeout=30)
code = req.getcode()
if code == 200:
result_paths = req.read().decode().strip().splitlines()
elif code == 204: # No content
result_paths = []
else:
raise ClientException('MockedError', http_status=str(code))
except IOError as e:
# 401 "Unauthorized" is swift's way of saying "container does not exist"
# But here we just assume swiftclient handles this via the usual
# ClientException.
raise ClientException('MockedError', http_status=str(e.code) if hasattr(e, 'code') else '')
finally:
if req is not None:
req.close()
return (None, result_paths)
def get_object(self, container, obj):
url = os.path.join(self._mocked_swift, container, obj)
req = None
try:
req = urlopen(url, timeout=30)
code = req.getcode()
if code == 200:
contents = req.read()
else:
raise ClientException('MockedError', http_status=str(code))
except IOError as e:
# 401 "Unauthorized" is swift's way of saying "container does not exist"
# But here we just assume swiftclient handles this via the usual
# ClientException.
raise ClientException('MockedError', http_status=str(e.code) if hasattr(e, 'code') else '')
finally:
if req is not None:
req.close()
return (None, contents)

18
tests/test_autopkgtest.py
Unescape View File

@ -117,7 +117,7 @@ class TestAutopkgtestBase(TestBase):
with open(email_path, 'w', encoding='utf-8') as email: with open(email_path, 'w', encoding='utf-8') as email:
email.write(json.dumps(self.email_cache)) email.write(json.dumps(self.email_cache))
self.swift.start(swiftclient=True) self.swift.start()
(excuses_yaml, excuses_html, out) = self.run_britney() (excuses_yaml, excuses_html, out) = self.run_britney()
self.swift.stop() self.swift.stop()
@ -2556,12 +2556,8 @@ class AT(TestAutopkgtestBase):
print('ADT_PPAS = first/ppa user:password@joe/foo:DEADBEEF') print('ADT_PPAS = first/ppa user:password@joe/foo:DEADBEEF')
elif line.startswith('ADT_SWIFT_USER'): elif line.startswith('ADT_SWIFT_USER'):
print('ADT_SWIFT_USER = user') print('ADT_SWIFT_USER = user')
elif line.startswith('ADT_SWIFT_PASS'): elif line.startswith('ADT_SWIFT_TOKEN'):
print('ADT_SWIFT_PASS = pass') print('ADT_SWIFT_TOKEN = SomeSecretToken')
elif line.startswith('ADT_SWIFT_TENANT'):
print('ADT_SWIFT_TENANT = tenant')
elif line.startswith('ADT_SWIFT_AUTH_URL'):
print('ADT_SWIFT_AUTH_URL = http://127.0.0.1:5000/v2.0/')
elif line.startswith('ADT_PRIVATE_SHARED'): elif line.startswith('ADT_PRIVATE_SHARED'):
print('ADT_PRIVATE_SHARED = user1 team2') print('ADT_PRIVATE_SHARED = user1 team2')
elif line.startswith('ADT_PRIVATE_URL'): elif line.startswith('ADT_PRIVATE_URL'):
@ -2628,12 +2624,8 @@ class AT(TestAutopkgtestBase):
for line in fileinput.input(self.britney_conf, inplace=True): for line in fileinput.input(self.britney_conf, inplace=True):
if line.startswith('ADT_SWIFT_USER'): if line.startswith('ADT_SWIFT_USER'):
print('ADT_SWIFT_USER = user') print('ADT_SWIFT_USER = user')
elif line.startswith('ADT_SWIFT_PASS'): elif line.startswith('ADT_SWIFT_TOKEN'):
print('ADT_SWIFT_PASS = pass') print('ADT_SWIFT_TOKEN = SomeSecretToken')
elif line.startswith('ADT_SWIFT_TENANT'):
print('ADT_SWIFT_TENANT = tenant')
elif line.startswith('ADT_SWIFT_AUTH_URL'):
print('ADT_SWIFT_AUTH_URL = http://127.0.0.1:5000/v2.0/')
elif line.startswith('ADT_PRIVATE_URL'): elif line.startswith('ADT_PRIVATE_URL'):
print('ADT_PRIVATE_URL = http://localhost:18085/private-results/') print('ADT_PRIVATE_URL = http://localhost:18085/private-results/')
else: else:

4
tests/test_policy.py
Unescape View File

@ -44,9 +44,7 @@ def initialize_policy(test_name, policy_class, *args, **kwargs):
adt_swift_url='file://' + debci_data, adt_swift_url='file://' + debci_data,
adt_ci_url='', adt_ci_url='',
adt_swift_user='', adt_swift_user='',
adt_swift_pass='', adt_swift_token='',
adt_swift_tenant='',
adt_swift_auth_url='',
adt_private_shared=[], adt_private_shared=[],
adt_private_url='', adt_private_url='',
adt_success_bounty=3, adt_success_bounty=3,

Write Preview
Loading…
Cancel
Save