Add a mock swiftclient for testing, add unit tests for private PPAs and private non-PPA runs, fix some issues discovered through the testing.

britney2/policies/autopkgtest.py

@@ -906,20 +906,18 @@ class AutopkgtestPolicy(BasePolicy):
 
             try:
                 _, result_paths = self.swift_conn.get_container(
-                    self.swift_conn.url,
+                    self.swift_container,
-                    token=self.swift_conn.token,
-                    container=self.swift_container,
                     query_string=urllib.parse.urlencode(query))
             except ClientException as e:
                 # 401 "Unauthorized" is swift's way of saying "container does not exist"
-                if e.http_code == 401 or e.http_code == 404:
+                if e.http_status == '401' or e.http_status == '404':
-                    self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', url)
+                    self.logger.info('fetch_swift_results: %s does not exist yet or is inaccessible', self.swift_container)
                     return
                 # Other status codes are usually a transient
                 # network/infrastructure failure. Ignoring this can lead to
                 # re-requesting tests which we already have results for, so
                 # fail hard on this and let the next run retry.
-                self.logger.error('Failure to fetch swift results from %s: %s', url, str(e))
+                self.logger.error('Failure to fetch swift results from %s: %s', self.swift_container, str(e))
                 sys.exit(1)
         else:
             url = os.path.join(swift_url, self.swift_container)
@@ -968,13 +966,13 @@ class AutopkgtestPolicy(BasePolicy):
 
                 # We don't need any additional retry logic as swiftclient
                 # already performs retries (5 by default).
-                full_path = os.path.join(path, name)
+                url = os.path.join(path, name)
                 try:
-                    _, contents = swift_conn.get_object(container, full_path)
+                    _, contents = self.swift_conn.get_object(container, url)
                 except ClientException as e:
                     self.logger.error('Failure to fetch %s from container %s: %s',
-                                      full_path, container, str(e))
+                                      url, container, str(e))
-                    if e.http_code == 404:
+                    if e.http_status == '404':
                         return
                     sys.exit(1)
                 tar_bytes = io.BytesIO(contents)
@@ -1114,7 +1112,7 @@ class AutopkgtestPolicy(BasePolicy):
         params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z')
 
         if self.swift_conn:
-            params['readable-by'] = self.adt_swift_user
+            params['readable-by'] = self.options.adt_swift_user
 
         if self.amqp_channel:
             import amqplib.client_0_8 as amqp

tests/mock_swift.py

@@ -19,6 +19,10 @@ except ImportError:
     from urlparse import urlparse, parse_qs
 
 
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_DIR = os.path.dirname(TESTS_DIR)
+
+
 class SwiftHTTPRequestHandler(BaseHTTPRequestHandler):
     '''Mock swift container with autopkgtest results
 
@@ -124,7 +128,15 @@ class AutoPkgTestSwiftServer:
         '''
         SwiftHTTPRequestHandler.results = results
 
-    def start(self):
+    def start(self, swiftclient=False):
+        if swiftclient:
+            # since we're running britney directly, the only way to reliably
+            # mock out the swiftclient module is to override it in the local
+            # path with the dummy version we created
+            src = os.path.join(TESTS_DIR, 'mock_swiftclient.py')
+            dst = os.path.join(PROJECT_DIR, 'swiftclient.py')
+            os.symlink(src, dst)
+            
         assert self.server_pid is None, 'already started'
         if self.log:
             self.log.close()
@@ -148,12 +160,19 @@ class AutoPkgTestSwiftServer:
         sys.exit(0)
 
     def stop(self):
+        # in case we were 'mocking out' swiftclient, remove the symlink we
+        # created earlier during start()
+        swiftclient_mod = os.path.join(PROJECT_DIR, 'swiftclient.py')
+        if os.path.islink(swiftclient_mod):
+            os.unlink(swiftclient_mod)
+
         assert self.server_pid, 'not running'
         os.kill(self.server_pid, 15)
         os.waitpid(self.server_pid, 0)
         self.server_pid = None
         self.log.close()
 
+
 if __name__ == '__main__':
     srv = AutoPkgTestSwiftServer()
     srv.set_results({'autopkgtest-testing': {

tests/mock_swiftclient.py

@@ -0,0 +1,70 @@
+# Mock the swiftclient Python library, the bare minimum for ADT purposes
+# Author: Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com>
+
+import os
+import sys
+
+from urllib.request import urlopen
+
+
+# We want to use this single Python module file to mock out the exception
+# module as well.
+sys.modules["swiftclient.exceptions"] = sys.modules[__name__]
+
+
+class ClientException(Exception):
+    def __init__(self, msg, http_status=''):
+        super(ClientException, self).__init__(msg)
+        self.msg = msg
+        self.http_status = http_status
+
+
+class Connection:
+    def __init__(self, authurl, user, key, tenant_name, auth_version):
+        self._mocked_swift = 'http://localhost:18085'
+
+    def get_container(self, container, marker=None, limit=None, prefix=None,
+                      delimiter=None, end_marker=None, path=None,
+                      full_listing=False, headers=None, query_string=None):
+        url = os.path.join(self._mocked_swift, container) + '?' + query_string
+        req = None
+        try:
+            req = urlopen(url, timeout=30)
+            code = req.getcode()
+            if code == 200:
+                result_paths = req.read().decode().strip().splitlines()
+            elif code == 204:  # No content
+                result_paths = []
+            else:
+                raise ClientException('MockedError', http_status=str(code))
+        except IOError as e:
+            # 401 "Unauthorized" is swift's way of saying "container does not exist"
+            # But here we just assume swiftclient handles this via the usual
+            # ClientException.
+            raise ClientException('MockedError', http_status=str(e.code) if hasattr(e, 'code') else '')
+        finally:
+            if req is not None:
+                req.close()
+
+        return (None, result_paths)
+
+    def get_object(self, container, obj):
+        url = os.path.join(self._mocked_swift, container, obj)
+        req = None
+        try:
+            req = urlopen(url, timeout=30)
+            code = req.getcode()
+            if code == 200:
+                contents = req.read()
+            else:
+                raise ClientException('MockedError', http_status=str(code))
+        except IOError as e:
+            # 401 "Unauthorized" is swift's way of saying "container does not exist"
+            # But here we just assume swiftclient handles this via the usual
+            # ClientException.
+            raise ClientException('MockedError', http_status=str(e.code) if hasattr(e, 'code') else '')
+        finally:
+            if req is not None:
+                req.close()
+
+        return (None, contents)

tests/test_autopkgtest.py

@@ -16,6 +16,7 @@ import urllib.parse
 
 import apt_pkg
 import yaml
+from unittest.mock import patch, Mock
 
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, PROJECT_DIR)
@@ -116,7 +117,7 @@ class TestAutopkgtestBase(TestBase):
         with open(email_path, 'w', encoding='utf-8') as email:
             email.write(json.dumps(self.email_cache))
 
-        self.swift.start()
+        self.swift.start(swiftclient=True)
         (excuses_yaml, excuses_html, out) = self.run_britney()
         self.swift.stop()
 
@@ -2552,7 +2553,15 @@ class AT(TestAutopkgtestBase):
 
         for line in fileinput.input(self.britney_conf, inplace=True):
             if line.startswith('ADT_PPAS'):
-                print('ADT_PPAS = user:password@joe/foo')
+                print('ADT_PPAS = first/ppa user:password@joe/foo:DEADBEEF')
+            elif line.startswith('ADT_SWIFT_USER'):
+                print('ADT_SWIFT_USER = user')
+            elif line.startswith('ADT_SWIFT_PASS'):
+                print('ADT_SWIFT_PASS = pass')
+            elif line.startswith('ADT_SWIFT_TENANT'):
+                print('ADT_SWIFT_TENANT = tenant')
+            elif line.startswith('ADT_SWIFT_AUTH_URL'):
+                print('ADT_SWIFT_AUTH_URL = http://127.0.0.1:5000/v2.0/')
             else:
                 sys.stdout.write(line)
 
@@ -2562,8 +2571,15 @@ class AT(TestAutopkgtestBase):
             {'lightgreen': [('old-version', '1'), ('new-version', '2')]}
         )[1]
 
+        # check if the private PPA info is propagated to the AMQP queue
+        self.assertEqual(len(self.amqp_requests), 2)
+        for request in self.amqp_requests:
+            self.assertIn('"triggers": ["lightgreen/2"]', request)
+            self.assertIn('"ppas": ["first/ppa", "user:password@joe/foo:DEADBEEF"]', request)
+            self.assertIn('"readable-by": "user"', request)
+
         # add results to PPA specific swift container
-        self.swift.set_results({'autopkgtest-testing-joe-foo': {
+        self.swift.set_results({'private-autopkgtest-testing-joe-foo': {
             'testing/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')),
             'testing/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 2', tr('lightgreen/2')),
             'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
@@ -2581,24 +2597,90 @@ class AT(TestAutopkgtestBase):
             {'lightgreen/2': {
                 'amd64': [
                     'PASS',
-                    'http://localhost:18085/autopkgtest-testing-joe-foo/'
+                    'http://localhost:18085/private-autopkgtest-testing-joe-foo/'
                     'testing/amd64/l/lightgreen/20150101_100101@/log.gz',
                     None,
-                    'http://localhost:18085/autopkgtest-testing-joe-foo/'
+                    'http://localhost:18085/private-autopkgtest-testing-joe-foo/'
                     'testing/amd64/l/lightgreen/20150101_100101@/artifacts.tar.gz',
                     None],
                 'i386': [
                     'REGRESSION',
-                    'http://localhost:18085/autopkgtest-testing-joe-foo/'
+                    'http://localhost:18085/private-autopkgtest-testing-joe-foo/'
                     'testing/i386/l/lightgreen/20150101_100100@/log.gz',
                     None,
-                    'http://localhost:18085/autopkgtest-testing-joe-foo/'
+                    'http://localhost:18085/private-autopkgtest-testing-joe-foo/'
                     'testing/i386/l/lightgreen/20150101_100100@/artifacts.tar.gz',
                     None]},
              'verdict': 'REJECTED_PERMANENTLY'})
         self.assertEqual(self.amqp_requests, set())
         self.assertEqual(self.pending_requests, {})
 
+    def test_private_without_ppa(self):
+        '''Run a private test (without using a private PPA)'''
+
+        self.data.add_default_packages(lightgreen=False)
+
+        for line in fileinput.input(self.britney_conf, inplace=True):
+            if line.startswith('ADT_SWIFT_USER'):
+                print('ADT_SWIFT_USER = user')
+            elif line.startswith('ADT_SWIFT_PASS'):
+                print('ADT_SWIFT_PASS = pass')
+            elif line.startswith('ADT_SWIFT_TENANT'):
+                print('ADT_SWIFT_TENANT = tenant')
+            elif line.startswith('ADT_SWIFT_AUTH_URL'):
+                print('ADT_SWIFT_AUTH_URL = http://127.0.0.1:5000/v2.0/')
+            else:
+                sys.stdout.write(line)
+
+        exc = self.run_it(
+            [('lightgreen', {'Version': '2'}, 'autopkgtest')],
+            {'lightgreen': (True, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL'}})},
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]}
+        )[1]
+
+        # check if the user info is propagated to the AMQP queue
+        self.assertEqual(len(self.amqp_requests), 2)
+        for request in self.amqp_requests:
+            self.assertIn('"triggers": ["lightgreen/2"]', request)
+            self.assertIn('"readable-by": "user"', request)
+
+        # add results to PPA specific swift container
+        self.swift.set_results({'private-autopkgtest-testing': {
+            'testing/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')),
+            'testing/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 2', tr('lightgreen/2')),
+            'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
+        }})
+
+        exc = self.run_it(
+            [],
+            {'lightgreen': (False, {'lightgreen/2': {'i386': 'REGRESSION', 'amd64': 'PASS'}})},
+            {'lightgreen': [('old-version', '1'), ('new-version', '2')]}
+        )[1]
+
+        # check if the right container name is used and that we still have the
+        # retry url (it should only be hidden for private PPAs)
+        self.assertEqual(
+            exc['lightgreen']['policy_info']['autopkgtest'],
+            {'lightgreen/2': {
+                'amd64': [
+                    'PASS',
+                    'http://localhost:18085/private-autopkgtest-testing/'
+                    'testing/amd64/l/lightgreen/20150101_100101@/log.gz',
+                    'https://autopkgtest.ubuntu.com/packages/l/lightgreen/testing/amd64',
+                    None,
+                    None],
+                'i386': [
+                    'REGRESSION',
+                    'http://localhost:18085/private-autopkgtest-testing/'
+                    'testing/i386/l/lightgreen/20150101_100100@/log.gz',
+                    'https://autopkgtest.ubuntu.com/packages/l/lightgreen/testing/i386',
+                    None,
+                    'https://autopkgtest.ubuntu.com/request.cgi?release=testing&arch=i386&package=lightgreen&'
+                    'trigger=lightgreen%2F2']},
+             'verdict': 'REJECTED_PERMANENTLY'})
+        self.assertEqual(self.amqp_requests, set())
+        self.assertEqual(self.pending_requests, {})
+
     def test_disable_upgrade_tester(self):
         '''Run without second stage upgrade tester'''