From 96df4080b936234f175dedd304c6224067cfc261 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 8 Jul 2016 11:14:57 +0200 Subject: [PATCH] Add autopkgtest policy Add new autopkgtest policy: it determines the autopkgtests for a source package (its own, direct reverse binary dependencies, and Testsuite-Triggers), requests tests via AMQP, fetches results from swift, and keeps track of pending tests between run. This also caches the downloaded results from swift, as re-dowloading them all is very expensive. This introduces two new hints: * force-badtest pkg/ver[/arch]: Failing results for that package will be ignored. This is useful to deal with broken tests that get imported from Debian or are from under-maintained packages, or broke due to some infrastructure changes. These are long-lived usually. * force-skiptest pkg/ver: Test results *triggered by* that package (i. e. reverse dependencies) will be ignored. This is mostly useful for landing packages that trigger a huge amount of tests (glibc, perl) where some tests are just too flaky to get them all passing, and one just wants to land it after the remaining failures have been checked. This should be used rarely and the hints should be removed immediately again. Add integration tests that call britney in various scenarios on constructed fake archives, with mocked AMQP and Swift results. --- britney.conf | 12 + britney.py | 3 + britney2/policies/autopkgtest.py | 723 +++++++++++ britney_nobreakall.conf | 12 + tests/__init__.py | 188 +++ tests/mock_swift.py | 170 +++ tests/test_autopkgtest.py | 2036 ++++++++++++++++++++++++++++++ 7 files changed, 3144 insertions(+) create mode 100644 britney2/policies/autopkgtest.py create mode 100644 tests/__init__.py create mode 100644 tests/mock_swift.py create mode 100755 tests/test_autopkgtest.py diff --git a/britney.conf b/britney.conf index 974c846..921461d 100644 --- a/britney.conf +++ b/britney.conf @@ -88,3 +88,15 @@ SMOOTH_UPDATES = badgers IGNORE_CRUFT = 0 REMOVE_OBSOLETE = no + +ADT_ENABLE = yes +ADT_ARCHES = amd64 i386 armhf ppc64el +ADT_AMQP = amqp://test_request:password@162.213.33.228 +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac +# space separate list of PPAs to add for test requests and for polling results; +# the *last* one determines the swift container name +ADT_PPAS = +# set this to the path of a (r/o) results.cache for running many parallel +# britney instances for PPAs without updating the cache +ADT_SHARED_RESULTS_CACHE = diff --git a/britney.py b/britney.py index ce8fafd..adf4083 100755 --- a/britney.py +++ b/britney.py @@ -203,6 +203,7 @@ from britney2.installability.builder import build_installability_tester, ma_pars from britney2.migrationitem import MigrationItem from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, PolicyVerdict from britney2.policies.policy import LPBlockBugPolicy +from britney2.policies.autopkgtest import AutopkgtestPolicy from britney2.utils import (old_libraries_format, undo_changes, compute_reverse_tree, possibly_compressed, read_nuninst, write_nuninst, write_heidi, @@ -531,6 +532,8 @@ class Britney(object): self.policies.append(RCBugPolicy(self.options, self.suite_info)) self.policies.append(PiupartsPolicy(self.options, self.suite_info)) self.policies.append(LPBlockBugPolicy(self.options, self.suite_info)) + if getattr(self.options, 'adt_enable') == 'yes': + self.policies.append(AutopkgtestPolicy(self.options, self.suite_info)) for policy in self.policies: policy.register_hints(self._hint_parser) diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py new file mode 100644 index 0000000..2fef0a1 --- /dev/null +++ b/britney2/policies/autopkgtest.py @@ -0,0 +1,723 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 - 2016 Canonical Ltd. +# Authors: +# Colin Watson +# Jean-Baptiste Lallement +# Martin Pitt + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import os +import json +import tarfile +import io +import re +import sys +import urllib.parse +from urllib.request import urlopen + +import apt_pkg +import amqplib.client_0_8 as amqp + +import britney2.hints +from britney2.policies.policy import BasePolicy, PolicyVerdict +from britney2.consts import VERSION + + +EXCUSES_LABELS = { + "PASS": 'Pass', + "FAIL": 'Failed', + "ALWAYSFAIL": 'Always failed', + "REGRESSION": 'Regression', + "IGNORE-FAIL": 'Ignored failure', + "RUNNING": 'Test in progress', + "RUNNING-ALWAYSFAIL": 'Test in progress (always failed)', +} + + +def srchash(src): + '''archive hash prefix for source package''' + + if src.startswith('lib'): + return src[:4] + else: + return src[0] + + +class AutopkgtestPolicy(BasePolicy): + """autopkgtest regression policy for source migrations + + Run autopkgtests for the excuse and all of its reverse dependencies, and + reject the upload if any of those regress. + """ + + def __init__(self, options, suite_info): + super().__init__('autopkgtest', options, suite_info, {'unstable'}) + self.test_state_dir = os.path.join(options.unstable, 'autopkgtest') + # tests requested in this and previous runs + # trigger -> src -> [arch] + self.pending_tests = None + self.pending_tests_file = os.path.join(self.test_state_dir, 'pending.json') + + # results map: trigger -> src -> arch -> [passed, version, run_id] + # - trigger is "source/version" of an unstable package that triggered + # this test run. + # - "passed" is a bool + # - "version" is the package version of "src" of that test + # - "run_id" is an opaque ID that identifies a particular test run for + # a given src/arch. It's usually a time stamp like "20150120_125959". + # This is also used for tracking the latest seen time stamp for + # requesting only newer results. + self.test_results = {} + if self.options.adt_shared_results_cache: + self.results_cache_file = self.options.adt_shared_results_cache + else: + self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') + + try: + self.options.adt_ppas = self.options.adt_ppas.strip().split() + except AttributeError: + self.options.adt_ppas = [] + + self.swift_container = 'autopkgtest-' + options.series + if self.options.adt_ppas: + self.swift_container += '-' + options.adt_ppas[-1].replace('/', '-') + + # restrict adt_arches to architectures we actually run for + self.adt_arches = [] + for arch in self.options.adt_arches.split(): + if arch in self.options.architectures: + self.adt_arches.append(arch) + else: + self.log("Ignoring ADT_ARCHES %s as it is not in architectures list" % arch) + + def register_hints(self, hint_parser): + hint_parser.register_hint_type('force-badtest', britney2.hints.split_into_one_hint_per_package) + hint_parser.register_hint_type('force-skiptest', britney2.hints.split_into_one_hint_per_package) + + def initialise(self, britney): + super().initialise(britney) + os.makedirs(self.test_state_dir, exist_ok=True) + self.read_pending_tests() + + # read the cached results that we collected so far + if os.path.exists(self.results_cache_file): + with open(self.results_cache_file) as f: + self.test_results = json.load(f) + self.log('Read previous results from %s' % self.results_cache_file) + else: + self.log('%s does not exist, re-downloading all results from swift' % + self.results_cache_file) + + # we need sources, binaries, and installability tester, so for now + # remember the whole britney object + self.britney = britney + + # Initialize AMQP connection + self.amqp_channel = None + self.amqp_file = None + if self.options.dry_run: + return + + amqp_url = self.options.adt_amqp + + if amqp_url.startswith('amqp://'): + # in production mode, connect to AMQP server + creds = urllib.parse.urlsplit(amqp_url, allow_fragments=False) + self.amqp_con = amqp.Connection(creds.hostname, userid=creds.username, + password=creds.password) + self.amqp_channel = self.amqp_con.channel() + self.log('Connected to AMQP server') + elif amqp_url.startswith('file://'): + # in testing mode, adt_amqp will be a file:// URL + self.amqp_file = amqp_url[7:] + else: + raise RuntimeError('Unknown ADT_AMQP schema %s' % amqp_url.split(':', 1)[0]) + + def save_state(self, britney): + super().save_state(britney) + + # update the results on-disk cache, unless we are using a r/o shared one + if not self.options.adt_shared_results_cache: + self.log('Updating results cache') + with open(self.results_cache_file + '.new', 'w') as f: + json.dump(self.test_results, f, indent=2) + os.rename(self.results_cache_file + '.new', self.results_cache_file) + + # update the pending tests on-disk cache + self.log('Updating pending requested tests in %s' % self.pending_tests_file) + with open(self.pending_tests_file + '.new', 'w') as f: + json.dump(self.pending_tests, f, indent=2) + os.rename(self.pending_tests_file + '.new', self.pending_tests_file) + + def apply_policy_impl(self, tests_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse): + # skip/delay autopkgtests until package is built + if excuse.missing_builds or 'depends' in excuse.reason: + self.log('%s has missing builds or is uninstallable, skipping autopkgtest policy' % excuse.name) + return PolicyVerdict.REJECTED_TEMPORARILY + + self.log('Checking autopkgtests for %s' % source_name) + trigger = source_name + '/' + source_data_srcdist.version + + # build a (testsrc, testver) → arch → (status, log_url) map; we trigger/check test + # results per archtitecture for technical/efficiency reasons, but we + # want to evaluate and present the results by tested source package + # first + pkg_arch_result = {} + for arch in self.adt_arches: + # request tests (unless they were already requested earlier or have a result) + tests = self.tests_for_source(source_name, source_data_srcdist.version, arch) + is_huge = len(tests) > 20 + for (testsrc, testver) in tests: + self.pkg_test_request(testsrc, arch, trigger, huge=is_huge) + (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger) + pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url) + + # add test result details to Excuse + verdict = PolicyVerdict.PASS + cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" + for ((testsrc, testver), arch_results) in pkg_arch_result.items(): + r = set([v[0] for v in arch_results.values()]) + if 'REGRESSION' in r: + verdict = PolicyVerdict.REJECTED_PERMANENTLY + elif 'RUNNING' in r and verdict == PolicyVerdict.PASS: + verdict = PolicyVerdict.REJECTED_TEMPORARILY + # skip version if still running on all arches + if not r - {'RUNNING', 'RUNNING-ALWAYSFAIL'}: + testver = None + + html_archmsg = [] + for arch, (status, log_url) in arch_results.items(): + artifact_url = None + retry_url = None + history_url = None + if self.options.adt_ppas: + if log_url.endswith('log.gz'): + artifact_url = log_url.replace('log.gz', 'artifacts.tar.gz') + else: + history_url = cloud_url % { + 'h': srchash(testsrc), 's': testsrc, + 'r': self.options.series, 'a': arch} + if status == 'REGRESSION': + retry_url = 'https://autopkgtest.ubuntu.com/request.cgi?' + \ + urllib.parse.urlencode([('release', self.options.series), + ('arch', arch), + ('package', testsrc), + ('trigger', trigger)] + + [('ppa', p) for p in self.options.adt_ppas]) + if testver: + testname = '%s/%s' % (testsrc, testver) + else: + testname = testsrc + + tests_info.setdefault(testname, {})[arch] = \ + [status, log_url, history_url, artifact_url, retry_url] + + # render HTML snippet for testsrc entry for current arch + if history_url: + message = '%s' % (history_url, arch) + else: + message = arch + message += ': %s' % (log_url, EXCUSES_LABELS[status]) + if retry_url: + message += ' ♻ ' % retry_url + if artifact_url: + message += ' [artifacts]' % artifact_url + html_archmsg.append(message) + + # render HTML line for testsrc entry + excuse.addhtml("autopkgtest for %s: %s" % (testname, ', '.join(html_archmsg))) + + + if verdict != PolicyVerdict.PASS: + # check for force-skiptest hint + hints = self.britney.hints.search('force-skiptest', package=source_name, version=source_data_srcdist.version) + if hints: + excuse.addreason('skiptest') + excuse.addhtml("Should wait for tests relating to %s %s, but forced by %s" % + (source_name, source_data_srcdist.version, hints[0].user)) + excuse.force() + verdict = PolicyVerdict.PASS_HINTED + else: + excuse.addreason('autopkgtest') + return verdict + + # + # helper functions + # + + @classmethod + def has_autodep8(kls, srcinfo, binaries): + '''Check if package is covered by autodep8 + + srcinfo is an item from self.britney.sources + binaries is self.britney.binaries['unstable'][arch][0] + ''' + # autodep8? + for t in srcinfo.testsuite: + if t.startswith('autopkgtest-pkg'): + return True + + # DKMS: some binary depends on "dkms" + for pkg_id in srcinfo.binaries: + try: + bininfo = binaries[pkg_id.package_name] + except KeyError: + continue + if 'dkms' in (bininfo.depends or ''): + return True + return False + + def tests_for_source(self, src, ver, arch): + '''Iterate over all tests that should be run for given source and arch''' + + sources_info = self.britney.sources['testing'] + binaries_info = self.britney.binaries['testing'][arch][0] + + reported_pkgs = set() + + tests = [] + + # hack for vivid's gccgo-5 and xenial's gccgo-6; these build libgcc1 + # too, so test some Go and some libgcc1 consumers + if src in ['gccgo-5', 'gccgo-6']: + for test in ['juju-mongodb', 'mongodb', 'libreoffice']: + try: + tests.append((test, self.britney.sources['testing'][test][VERSION])) + except KeyError: + # no package in that series? *shrug*, then not (mostly for testing) + pass + return tests + + # gcc-N triggers tons of tests via libgcc1, but this is mostly in vain: + # gcc already tests itself during build, and it is being used from + # -proposed, so holding it back on a dozen unrelated test failures + # serves no purpose. Just check some key packages which actually use + # gcc during the test, and libreoffice as an example for a libgcc user. + if src.startswith('gcc-'): + if re.match('gcc-\d$', src): + for test in ['binutils', 'fglrx-installer', 'libreoffice', 'linux']: + try: + tests.append((test, self.britney.sources['testing'][test][VERSION])) + except KeyError: + # no package in that series? *shrug*, then not (mostly for testing) + pass + return tests + else: + # for other compilers such as gcc-snapshot etc. we don't need + # to trigger anything + return [] + + # for linux themselves we don't want to trigger tests -- these should + # all come from linux-meta*. A new kernel ABI without a corresponding + # -meta won't be installed and thus we can't sensibly run tests against + # it. + if src.startswith('linux') and src.replace('linux', 'linux-meta') in self.britney.sources['testing']: + return [] + + # we want to test the package itself, if it still has a test in unstable + srcinfo = self.britney.sources['unstable'][src] + if 'autopkgtest' in srcinfo.testsuite or self.has_autodep8(srcinfo, binaries_info): + reported_pkgs.add(src) + tests.append((src, ver)) + + extra_bins = [] + # Hack: For new kernels trigger all DKMS packages by pretending that + # linux-meta* builds a "dkms" binary as well. With that we ensure that we + # don't regress DKMS drivers with new kernel versions. + if src.startswith('linux-meta'): + # does this have any image on this arch? + for pkg_id in srcinfo.binaries: + if pkg_id.architecture == arch and '-image' in pkg_id.package_name: + try: + extra_bins.append(binaries_info['dkms'].pkg_id) + except KeyError: + pass + + # plus all direct reverse dependencies and test triggers of its + # binaries which have an autopkgtest + for binary in srcinfo.binaries + extra_bins: + rdeps = self.britney._inst_tester.reverse_dependencies_of(binary) + for rdep in rdeps: + try: + rdep_src = binaries_info[rdep.package_name].source + except KeyError: + self.log('%s on %s has no source (NBS?)' % (rdep.package_name, arch)) + continue + + rdep_src_info = sources_info[rdep_src] + if 'autopkgtest' in rdep_src_info.testsuite or self.has_autodep8(rdep_src_info, binaries_info): + if rdep_src not in reported_pkgs: + tests.append((rdep_src, rdep_src_info[VERSION])) + reported_pkgs.add(rdep_src) + + for tdep_src in self.britney.testsuite_triggers.get(binary.package_name, set()): + if tdep_src not in reported_pkgs: + try: + tdep_src_info = sources_info[tdep_src] + except KeyError: + continue + if 'autopkgtest' in tdep_src_info.testsuite or self.has_autodep8(tdep_src_info, binaries_info): + tests.append((tdep_src, tdep_src_info[VERSION])) + reported_pkgs.add(tdep_src) + + # Hardcode linux-meta → linux, lxc, glibc, systemd triggers until we get a more flexible + # implementation: https://bugs.debian.org/779559 + if src.startswith('linux-meta'): + for pkg in ['lxc', 'lxd', 'glibc', src.replace('linux-meta', 'linux'), 'systemd']: + if pkg not in reported_pkgs: + # does this have any image on this arch? + for pkg_id in srcinfo.binaries: + if pkg_id.architecture == arch and '-image' in pkg_id.package_name: + try: + tests.append((pkg, self.britney.sources['unstable'][pkg][VERSION])) + except KeyError: + try: + tests.append((pkg, self.britney.sources['testing'][pkg][VERSION])) + except KeyError: + # package not in that series? *shrug*, then not + pass + break + + tests.sort(key=lambda s_v: s_v[0]) + return tests + + def read_pending_tests(self): + '''Read pending test requests from previous britney runs + + Initialize self.pending_tests with that data. + ''' + assert self.pending_tests is None, 'already initialized' + if not os.path.exists(self.pending_tests_file): + self.log('No %s, starting with no pending tests' % + self.pending_tests_file) + self.pending_tests = {} + return + with open(self.pending_tests_file) as f: + self.pending_tests = json.load(f) + self.log('Read pending requested tests from %s: %s' % + (self.pending_tests_file, self.pending_tests)) + + def latest_run_for_package(self, src, arch): + '''Return latest run ID for src on arch''' + + # this requires iterating over all triggers and thus is expensive; + # cache the results + try: + return self.latest_run_for_package._cache[src][arch] + except KeyError: + pass + + latest_run_id = '' + for srcmap in self.test_results.values(): + try: + run_id = srcmap[src][arch][2] + except KeyError: + continue + if run_id > latest_run_id: + latest_run_id = run_id + self.latest_run_for_package._cache.setdefault(src, {})[arch] = latest_run_id + return latest_run_id + + latest_run_for_package._cache = {} + + def fetch_swift_results(self, swift_url, src, arch): + '''Download new results for source package/arch from swift''' + + # Download results for one particular src/arch at most once in every + # run, as this is expensive + done_entry = src + '/' + arch + if done_entry in self.fetch_swift_results._done: + return + self.fetch_swift_results._done.add(done_entry) + + # prepare query: get all runs with a timestamp later than the latest + # run_id for this package/arch; '@' is at the end of each run id, to + # mark the end of a test run directory path + # example: wily/amd64/libp/libpng/20150630_054517@/result.tar + query = {'delimiter': '@', + 'prefix': '%s/%s/%s/%s/' % (self.options.series, arch, srchash(src), src)} + + # determine latest run_id from results + if not self.options.adt_shared_results_cache: + latest_run_id = self.latest_run_for_package(src, arch) + if latest_run_id: + query['marker'] = query['prefix'] + latest_run_id + + # request new results from swift + url = os.path.join(swift_url, self.swift_container) + url += '?' + urllib.parse.urlencode(query) + try: + f = urlopen(url, timeout=30) + if f.getcode() == 200: + result_paths = f.read().decode().strip().splitlines() + elif f.getcode() == 204: # No content + result_paths = [] + else: + # we should not ever end up here as we expect a HTTPError in + # other cases; e. g. 3XX is something that tells us to adjust + # our URLS, so fail hard on those + raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' % + (url, f.getcode())) + f.close() + except IOError as e: + # 401 "Unauthorized" is swift's way of saying "container does not exist" + if hasattr(e, 'code') and e.code == 401: + self.log('fetch_swift_results: %s does not exist yet or is inaccessible' % url) + return + # Other status codes are usually a transient + # network/infrastructure failure. Ignoring this can lead to + # re-requesting tests which we already have results for, so + # fail hard on this and let the next run retry. + self.log('FATAL: Failure to fetch swift results from %s: %s' % (url, str(e)), 'E') + sys.exit(1) + + for p in result_paths: + self.fetch_one_result( + os.path.join(swift_url, self.swift_container, p, 'result.tar'), src, arch) + + fetch_swift_results._done = set() + + def fetch_one_result(self, url, src, arch): + '''Download one result URL for source/arch + + Remove matching pending_tests entries. + ''' + try: + f = urlopen(url, timeout=30) + if f.getcode() == 200: + tar_bytes = io.BytesIO(f.read()) + f.close() + else: + raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' % + (url, f.getcode())) + except IOError as e: + self.log('Failure to fetch %s: %s' % (url, str(e)), 'E') + # we tolerate "not found" (something went wrong on uploading the + # result), but other things indicate infrastructure problems + if hasattr(e, 'code') and e.code == 404: + return + sys.exit(1) + + try: + with tarfile.open(None, 'r', tar_bytes) as tar: + exitcode = int(tar.extractfile('exitcode').read().strip()) + srcver = tar.extractfile('testpkg-version').read().decode().strip() + (ressrc, ver) = srcver.split() + testinfo = json.loads(tar.extractfile('testinfo.json').read().decode()) + except (KeyError, ValueError, tarfile.TarError) as e: + self.log('%s is damaged, ignoring: %s' % (url, str(e)), 'E') + # ignore this; this will leave an orphaned request in pending.json + # and thus require manual retries after fixing the tmpfail, but we + # can't just blindly attribute it to some pending test. + return + + if src != ressrc: + self.log('%s is a result for package %s, but expected package %s' % + (url, ressrc, src), 'E') + return + + # parse recorded triggers in test result + for e in testinfo.get('custom_environment', []): + if e.startswith('ADT_TEST_TRIGGERS='): + result_triggers = [i for i in e.split('=', 1)[1].split() if '/' in i] + break + else: + self.log('%s result has no ADT_TEST_TRIGGERS, ignoring', 'E') + return + + stamp = os.path.basename(os.path.dirname(url)) + # allow some skipped tests, but nothing else + passed = exitcode in [0, 2] + + self.log('Fetched test result for %s/%s/%s %s (triggers: %s): %s' % ( + src, ver, arch, stamp, result_triggers, passed and 'pass' or 'fail')) + + # remove matching test requests + for trigger in result_triggers: + try: + arch_list = self.pending_tests[trigger][src] + arch_list.remove(arch) + if not arch_list: + del self.pending_tests[trigger][src] + if not self.pending_tests[trigger]: + del self.pending_tests[trigger] + self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger)) + except (KeyError, ValueError): + self.log('-> does not match any pending request for %s/%s' % (src, arch)) + + # add this result + for trigger in result_triggers: + # If a test runs because of its own package (newer version), ensure + # that we got a new enough version; FIXME: this should be done more + # generically by matching against testpkg-versions + (trigsrc, trigver) = trigger.split('/', 1) + if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0: + self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E') + continue + + result = self.test_results.setdefault(trigger, {}).setdefault( + src, {}).setdefault(arch, [False, None, '']) + + # don't clobber existing passed results with failures from re-runs + if passed or not result[0]: + result[0] = passed + result[1] = ver + result[2] = stamp + + def send_test_request(self, src, arch, trigger, huge=False): + '''Send out AMQP request for testing src/arch for trigger + + If huge is true, then the request will be put into the -huge instead of + normal queue. + ''' + if self.options.dry_run: + return + + params = {'triggers': [trigger]} + if self.options.adt_ppas: + params['ppas'] = self.options.adt_ppas + qname = 'debci-ppa-%s-%s' % (self.options.series, arch) + elif huge: + qname = 'debci-huge-%s-%s' % (self.options.series, arch) + else: + qname = 'debci-%s-%s' % (self.options.series, arch) + params = json.dumps(params) + + if self.amqp_channel: + self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params), routing_key=qname) + else: + assert self.amqp_file + with open(self.amqp_file, 'a') as f: + f.write('%s:%s %s\n' % (qname, src, params)) + + def pkg_test_request(self, src, arch, trigger, huge=False): + '''Request one package test for one particular trigger + + trigger is "pkgname/version" of the package that triggers the testing + of src. If huge is true, then the request will be put into the -huge + instead of normal queue. + + This will only be done if that test wasn't already requested in a + previous run (i. e. not already in self.pending_tests) or there already + is a result for it. This ensures to download current results for this + package before requesting any test. + ''' + # Don't re-request if we already have a result + try: + passed = self.test_results[trigger][src][arch][0] + if passed: + self.log('%s/%s triggered by %s already passed' % (src, arch, trigger)) + return + self.log('Checking for new results for failed %s/%s for trigger %s' % + (src, arch, trigger)) + raise KeyError # fall through + except KeyError: + self.fetch_swift_results(self.options.adt_swift_url, src, arch) + # do we have one now? + try: + self.test_results[trigger][src][arch] + return + except KeyError: + pass + + # Don't re-request if it's already pending + arch_list = self.pending_tests.setdefault(trigger, {}).setdefault(src, []) + if arch in arch_list: + self.log('Test %s/%s for %s is already pending, not queueing' % + (src, arch, trigger)) + else: + self.log('Requesting %s autopkgtest on %s to verify %s' % + (src, arch, trigger)) + arch_list.append(arch) + arch_list.sort() + self.send_test_request(src, arch, trigger, huge=huge) + + def check_ever_passed(self, src, arch): + '''Check if tests for src ever passed on arch''' + + # FIXME: add caching + for srcmap in self.test_results.values(): + try: + if srcmap[src][arch][0]: + return True + except KeyError: + pass + return False + + def pkg_test_result(self, src, ver, arch, trigger): + '''Get current test status of a particular package + + Return (status, real_version, log_url) tuple; status is a key in + EXCUSES_LABELS. log_url is None if the test is still running. + ''' + # determine current test result status + ever_passed = self.check_ever_passed(src, arch) + url = None + try: + r = self.test_results[trigger][src][arch] + ver = r[1] + run_id = r[2] + if r[0]: + result = 'PASS' + else: + # Special-case triggers from linux-meta*: we cannot compare + # results against different kernels, as e. g. a DKMS module + # might work against the default kernel but fail against a + # different flavor; so for those, ignore the "ever + # passed" check; FIXME: check against trigsrc only + if trigger.startswith('linux-meta') or trigger.startswith('linux/'): + ever_passed = False + + if ever_passed: + if self.has_force_badtest(src, ver, arch): + result = 'IGNORE-FAIL' + else: + result = 'REGRESSION' + else: + result = 'ALWAYSFAIL' + + url = os.path.join(self.options.adt_swift_url, + self.swift_container, + self.options.series, + arch, + srchash(src), + src, + run_id, + 'log.gz') + except KeyError: + # no result for src/arch; still running? + if arch in self.pending_tests.get(trigger, {}).get(src, []): + if ever_passed and not self.has_force_badtest(src, ver, arch): + result = 'RUNNING' + else: + result = 'RUNNING-ALWAYSFAIL' + url = 'http://autopkgtest.ubuntu.com/running' + else: + raise RuntimeError('Result for %s/%s/%s (triggered by %s) is neither known nor pending!' % + (src, ver, arch, trigger)) + + return (result, ver, url) + + def has_force_badtest(self, src, ver, arch): + '''Check if src/ver/arch has a force-badtest hint''' + + hints = self.britney.hints.search('force-badtest', package=src) + if hints: + self.log('Checking hints for %s/%s/%s: %s' % (src, ver, arch, [str(h) for h in hints])) + for hint in hints: + if [mi for mi in hint.packages if mi.architecture in ['source', arch] and + (mi.version == 'all' or apt_pkg.version_compare(ver, mi.version) <= 0)]: + return True + + return False + diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index ada24ba..c71f539 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -86,3 +86,15 @@ HINTS_UBUNTU-TOUCH/OGRA = block unblock SMOOTH_UPDATES = badgers REMOVE_OBSOLETE = no + +ADT_ENABLE = yes +ADT_ARCHES = amd64 i386 armhf ppc64el +ADT_AMQP = amqp://test_request:password@162.213.33.228 +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac +# space separate list of PPAs to add for test requests and for polling results; +# the *last* one determines the swift container name +ADT_PPAS = +# set this to the path of a (r/o) results.cache for running many parallel +# britney instances for PPAs without updating the cache +ADT_SHARED_RESULTS_CACHE = diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e08edc3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,188 @@ +# (C) 2015 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import shutil +import subprocess +import tempfile +import unittest + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'] + + +class TestData: + + def __init__(self): + '''Construct local test package indexes. + + The archive is initially empty. You can create new packages with + create_deb(). self.path contains the path of the archive, and + self.apt_source provides an apt source "deb" line. + + It is kept in a temporary directory which gets removed when the Archive + object gets deleted. + ''' + self.path = tempfile.mkdtemp(prefix='testarchive.') + self.apt_source = 'deb file://%s /' % self.path + self.series = 'series' + self.dirs = {False: os.path.join(self.path, 'data', self.series), + True: os.path.join( + self.path, 'data', '%s-proposed' % self.series)} + os.makedirs(self.dirs[False]) + os.mkdir(self.dirs[True]) + self.added_sources = {False: set(), True: set()} + self.added_binaries = {False: set(), True: set()} + + # pre-create all files for all architectures + for arch in architectures: + for dir in self.dirs.values(): + with open(os.path.join(dir, 'Packages_' + arch), 'w'): + pass + for dir in self.dirs.values(): + for fname in ['Dates', 'Blocks']: + with open(os.path.join(dir, fname), 'w'): + pass + for dname in ['Hints']: + os.mkdir(os.path.join(dir, dname)) + + os.mkdir(os.path.join(self.path, 'output')) + + # create temporary home dir for proposed-migration autopktest status + self.home = os.path.join(self.path, 'home') + os.environ['HOME'] = self.home + os.makedirs(os.path.join(self.home, 'proposed-migration', + 'autopkgtest', 'work')) + + def __del__(self): + shutil.rmtree(self.path) + + def add(self, name, unstable, fields={}, add_src=True, testsuite=None, srcfields=None): + '''Add a binary package to the index file. + + You need to specify at least the package name and in which list to put + it (unstable==True for unstable/proposed, or False for + testing/release). fields specifies all additional entries, e. g. + {'Depends': 'foo, bar', 'Conflicts: baz'}. There are defaults for most + fields. + + Unless add_src is set to False, this will also automatically create a + source record, based on fields['Source'] and name. In that case, the + "Testsuite:" field is set to the testsuite argument. + ''' + assert (name not in self.added_binaries[unstable]) + self.added_binaries[unstable].add(name) + + fields.setdefault('Architecture', 'all') + fields.setdefault('Version', '1') + fields.setdefault('Priority', 'optional') + fields.setdefault('Section', 'devel') + fields.setdefault('Description', 'test pkg') + if fields['Architecture'] == 'all': + for a in architectures: + self._append(name, unstable, 'Packages_' + a, fields) + else: + self._append(name, unstable, 'Packages_' + fields['Architecture'], + fields) + + if add_src: + src = fields.get('Source', name) + if src not in self.added_sources[unstable]: + if srcfields is None: + srcfields = {} + srcfields['Version'] = fields['Version'] + srcfields['Section'] = fields['Section'] + if testsuite: + srcfields['Testsuite'] = testsuite + self.add_src(src, unstable, srcfields) + + def add_src(self, name, unstable, fields={}): + '''Add a source package to the index file. + + You need to specify at least the package name and in which list to put + it (unstable==True for unstable/proposed, or False for + testing/release). fields specifies all additional entries, which can be + Version (default: 1), Section (default: devel), Testsuite (default: + none), and Extra-Source-Only. + ''' + assert (name not in self.added_sources[unstable]) + self.added_sources[unstable].add(name) + + fields.setdefault('Version', '1') + fields.setdefault('Section', 'devel') + self._append(name, unstable, 'Sources', fields) + + def _append(self, name, unstable, file_name, fields): + with open(os.path.join(self.dirs[unstable], file_name), 'a') as f: + f.write('''Package: %s +Maintainer: Joe +''' % name) + + for k, v in fields.items(): + f.write('%s: %s\n' % (k, v)) + f.write('\n') + + def remove_all(self, unstable): + '''Remove all added packages''' + + self.added_binaries[unstable] = set() + self.added_sources[unstable] = set() + for a in architectures: + open(os.path.join(self.dirs[unstable], 'Packages_' + a), 'w').close() + open(os.path.join(self.dirs[unstable], 'Sources'), 'w').close() + + +class TestBase(unittest.TestCase): + + def setUp(self): + super(TestBase, self).setUp() + self.maxDiff = None + self.data = TestData() + self.britney = os.path.join(PROJECT_DIR, 'britney.py') + # create temporary config so that tests can hack it + self.britney_conf = os.path.join(self.data.path, 'britney.conf') + shutil.copy(os.path.join(PROJECT_DIR, 'britney.conf'), self.britney_conf) + assert os.path.exists(self.britney) + + def tearDown(self): + del self.data + + def run_britney(self, args=[]): + '''Run britney. + + Assert that it succeeds and does not produce anything on stderr. + Return (excuses.yaml, excuses.html, britney_out). + ''' + britney = subprocess.Popen([self.britney, '-v', '-c', self.britney_conf, + '--distribution=ubuntu', + '--series=%s' % self.data.series], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.data.path, + universal_newlines=True) + (out, err) = britney.communicate() + self.assertEqual(britney.returncode, 0, out + err) + self.assertEqual(err, '') + + with open(os.path.join(self.data.path, 'output', self.data.series, + 'excuses.yaml')) as f: + yaml = f.read() + with open(os.path.join(self.data.path, 'output', self.data.series, + 'excuses.html')) as f: + html = f.read() + + return (yaml, html, out) + + def create_hint(self, username, content): + '''Create a hint file for the given username and content''' + + hints_path = os.path.join( + self.data.path, 'data', self.data.series + '-proposed', 'Hints', username) + with open(hints_path, 'a') as fd: + fd.write(content) + fd.write('\n') diff --git a/tests/mock_swift.py b/tests/mock_swift.py new file mode 100644 index 0000000..ecaa2fc --- /dev/null +++ b/tests/mock_swift.py @@ -0,0 +1,170 @@ +# Mock a Swift server with autopkgtest results +# Author: Martin Pitt + +import os +import tarfile +import io +import sys +import socket +import time +import tempfile +import json + +try: + from http.server import HTTPServer, BaseHTTPRequestHandler + from urllib.parse import urlparse, parse_qs +except ImportError: + # Python 2 + from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler + from urlparse import urlparse, parse_qs + + +class SwiftHTTPRequestHandler(BaseHTTPRequestHandler): + '''Mock swift container with autopkgtest results + + This accepts retrieving a particular result.tar (e. g. + /container/path/result.tar) or listing the container contents + (/container/?prefix=foo&delimiter=@&marker=foo/bar). + ''' + # map container -> result.tar path -> (exitcode, testpkg-version[, testinfo]) + results = {} + + def do_GET(self): + p = urlparse(self.path) + path_comp = p.path.split('/') + container = path_comp[1] + path = '/'.join(path_comp[2:]) + if path: + self.serve_file(container, path) + else: + self.list_container(container, parse_qs(p.query)) + + def serve_file(self, container, path): + if os.path.basename(path) != 'result.tar': + self.send_error(404, 'File not found (only result.tar supported)') + return + try: + fields = self.results[container][os.path.dirname(path)] + try: + (exitcode, pkgver, testinfo) = fields + except ValueError: + (exitcode, pkgver) = fields + testinfo = None + except KeyError: + self.send_error(404, 'File not found') + return + + self.send_response(200) + self.send_header('Content-type', 'application/octet-stream') + self.end_headers() + + tar = io.BytesIO() + with tarfile.open('result.tar', 'w', tar) as results: + # add exitcode + contents = ('%i' % exitcode).encode() + ti = tarfile.TarInfo('exitcode') + ti.size = len(contents) + results.addfile(ti, io.BytesIO(contents)) + # add testpkg-version + if pkgver is not None: + contents = pkgver.encode() + ti = tarfile.TarInfo('testpkg-version') + ti.size = len(contents) + results.addfile(ti, io.BytesIO(contents)) + # add testinfo.json + if testinfo: + contents = json.dumps(testinfo).encode() + ti = tarfile.TarInfo('testinfo.json') + ti.size = len(contents) + results.addfile(ti, io.BytesIO(contents)) + + self.wfile.write(tar.getvalue()) + + def list_container(self, container, query): + try: + objs = set(['%s/result.tar' % r for r in self.results[container]]) + except KeyError: + self.send_error(401, 'Container does not exist') + return + if 'prefix' in query: + p = query['prefix'][-1] + objs = set([o for o in objs if o.startswith(p)]) + if 'delimiter' in query: + d = query['delimiter'][-1] + # if find() returns a value, we want to include the delimiter, thus + # bump its result; for "not found" return None + find_adapter = lambda i: (i >= 0) and (i + 1) or None + objs = set([o[:find_adapter(o.find(d))] for o in objs]) + if 'marker' in query: + m = query['marker'][-1] + objs = set([o for o in objs if o > m]) + + self.send_response(objs and 200 or 204) # 204: "No Content" + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write(('\n'.join(sorted(objs)) + '\n').encode('UTF-8')) + + +class AutoPkgTestSwiftServer: + def __init__(self, port=8080): + self.port = port + self.server_pid = None + self.log = None + + def __del__(self): + if self.server_pid: + self.stop() + + @classmethod + def set_results(klass, results): + '''Set served results. + + results is a map: container -> result.tar path -> + (exitcode, testpkg-version, testinfo) + ''' + SwiftHTTPRequestHandler.results = results + + def start(self): + assert self.server_pid is None, 'already started' + if self.log: + self.log.close() + self.log = tempfile.TemporaryFile() + p = os.fork() + if p: + # parent: wait until server starts + self.server_pid = p + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + while True: + if s.connect_ex(('127.0.0.1', self.port)) == 0: + break + time.sleep(0.1) + s.close() + return + + # child; quiesce logging on stderr + os.dup2(self.log.fileno(), sys.stderr.fileno()) + srv = HTTPServer(('', self.port), SwiftHTTPRequestHandler) + srv.serve_forever() + sys.exit(0) + + def stop(self): + assert self.server_pid, 'not running' + os.kill(self.server_pid, 15) + os.waitpid(self.server_pid, 0) + self.server_pid = None + self.log.close() + +if __name__ == '__main__': + srv = AutoPkgTestSwiftServer() + srv.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/g/green/20150101_100000@': (0, 'green 1', {'custom_environment': ['ADT_TEST_TRIGGERS=green']}), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'), + 'series/i386/l/lightgreen/20150101_100102@': (0, 'lightgreen 3'), + }}) + srv.start() + print('Running on http://localhost:8080/autopkgtest-series') + print('Press Enter to quit.') + sys.stdin.readline() + srv.stop() diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py new file mode 100755 index 0000000..1257f17 --- /dev/null +++ b/tests/test_autopkgtest.py @@ -0,0 +1,2036 @@ +#!/usr/bin/python3 +# (C) 2014 - 2015 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import sys +import fileinput +import unittest +import json +import pprint +import urllib.parse + +import apt_pkg +import yaml + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PROJECT_DIR) + +from tests import TestBase, mock_swift + +apt_pkg.init() + + +# shortcut for test triggers +def tr(s): + return {'custom_environment': ['ADT_TEST_TRIGGERS=%s' % s]} + +ON_ALL_ARCHES = {'on-architectures': ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []} + + +class T(TestBase): + '''AMQP/cloud interface''' + + ################################################################ + # Common test code + ################################################################ + + def setUp(self): + super().setUp() + self.fake_amqp = os.path.join(self.data.path, 'amqp') + + # Set fake AMQP and Swift server + for line in fileinput.input(self.britney_conf, inplace=True): + if 'ADT_AMQP' in line: + print('ADT_AMQP = file://%s' % self.fake_amqp) + elif 'ADT_SWIFT_URL' in line: + print('ADT_SWIFT_URL = http://localhost:18085') + elif 'ADT_ARCHES' in line: + print('ADT_ARCHES = amd64 i386') + else: + sys.stdout.write(line) + + # add a bunch of packages to testing to avoid repetition + self.data.add('libc6', False) + self.data.add('libgreen1', False, {'Source': 'green', + 'Depends': 'libc6 (>= 0.9)'}, + testsuite='autopkgtest') + self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Conflicts': 'blue'}, + testsuite='autopkgtest') + self.data.add('lightgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest') + # autodep8 or similar test + self.data.add('darkgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest-pkg-foo') + self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', + 'Conflicts': 'green'}, + testsuite='specialtest') + + # create mock Swift server (but don't start it yet, as tests first need + # to poke in results) + self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085) + self.swift.set_results({}) + + def tearDown(self): + del self.swift + + def do_test(self, unstable_add, expect_status, expect_excuses={}): + '''Run britney with some unstable packages and verify excuses. + + unstable_add is a list of (binpkgname, field_dict, testsuite_value) + passed to TestData.add for "unstable". + + expect_status is a dict sourcename → (is_candidate, testsrc → arch → status) + that is checked against the excuses YAML. + + expect_excuses is a dict sourcename → [(key, value), ...] + matches that are checked against the excuses YAML. + + Return (output, excuses_dict, excuses_html). + ''' + for (pkg, fields, testsuite) in unstable_add: + self.data.add(pkg, True, fields, True, testsuite) + + self.swift.start() + (excuses_yaml, excuses_html, out) = self.run_britney() + self.swift.stop() + + # convert excuses to source indexed dict + excuses_dict = {} + for s in yaml.load(excuses_yaml)['sources']: + excuses_dict[s['source']] = s + + if 'SHOW_EXCUSES' in os.environ: + print('------- excuses -----') + pprint.pprint(excuses_dict, width=200) + if 'SHOW_HTML' in os.environ: + print('------- excuses.html -----\n%s\n' % excuses_html) + if 'SHOW_OUTPUT' in os.environ: + print('------- output -----\n%s\n' % out) + + for src, (is_candidate, testmap) in expect_status.items(): + self.assertEqual(excuses_dict[src]['is-candidate'], is_candidate, + src + ': ' + pprint.pformat(excuses_dict[src])) + for testsrc, archmap in testmap.items(): + for arch, status in archmap.items(): + self.assertEqual(excuses_dict[src]['policy_info']['autopkgtest'][testsrc][arch][0], + status, + excuses_dict[src]['policy_info']['autopkgtest'][testsrc]) + + for src, matches in expect_excuses.items(): + for k, v in matches: + if isinstance(excuses_dict[src][k], list): + self.assertIn(v, excuses_dict[src][k]) + else: + self.assertEqual(excuses_dict[src][k], v) + + self.amqp_requests = set() + try: + with open(self.fake_amqp) as f: + for line in f: + self.amqp_requests.add(line.strip()) + os.unlink(self.fake_amqp) + except IOError: + pass + + try: + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/pending.json')) as f: + self.pending_requests = json.load(f) + except IOError: + self.pending_requests = None + + self.assertNotIn('FIXME', out) + + return (out, excuses_dict, excuses_html) + + ################################################################ + # Tests for generic packages + ################################################################ + + def test_no_request_for_uninstallable(self): + '''Does not request a test for an uninstallable package''' + + exc = self.do_test( + # uninstallable unstable version + [('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')], + {'lightgreen': (False, {})}, + {'lightgreen': [('old-version', '1'), ('new-version', '1.1~beta'), + ('reason', 'depends'), + ('excuses', 'lightgreen/amd64 unsatisfiable Depends: libgreen1 (>= 2)') + ] + })[1] + # autopkgtest should not be triggered for uninstallable pkg + self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], {}) + + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + with open(os.path.join(self.data.path, 'output', 'series', 'output.txt')) as f: + upgrade_out = f.read() + self.assertNotIn('accepted:', upgrade_out) + self.assertIn('SUCCESS (0/0)', upgrade_out) + + def test_no_wait_for_always_failed_test(self): + '''We do not need to wait for results for tests which have always failed''' + + # The package has failed before, and with a trigger too on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (4, 'green 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (4, 'green 1', tr('failedbefore/1')), + }}) + + exc = self.do_test( + [('darkgreen', {'Version': '2'}, 'autopkgtest')], + {'darkgreen': (True, {'darkgreen': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}})}, + )[1] + + # the test should still be triggered though + self.assertEqual(exc['darkgreen']['policy_info']['autopkgtest'], + {'darkgreen': { + 'amd64': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + 'http://autopkgtest.ubuntu.com/packages/d/darkgreen/series/amd64', + None, + None], + 'i386': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + 'http://autopkgtest.ubuntu.com/packages/d/darkgreen/series/i386', + None, + None]}}) + + self.assertEqual(self.pending_requests, + {'darkgreen/2': {'darkgreen': ['amd64', 'i386']}}) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-amd64:darkgreen {"triggers": ["darkgreen/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["darkgreen/2"]}'])) + + with open(os.path.join(self.data.path, 'output', 'series', 'output.txt')) as f: + upgrade_out = f.read() + self.assertIn('accepted: darkgreen', upgrade_out) + self.assertIn('SUCCESS (1/0)', upgrade_out) + + def test_multi_rdepends_with_tests_all_running(self): + '''Multiple reverse dependencies with tests (all running)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2'), + ('reason', 'autopkgtest')]}) + + # we expect the package's and its reverse dependencies' tests to get + # triggered + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green {"triggers": ["green/2"]}', + 'debci-series-amd64:green {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}'])) + + # ... and that they get recorded as pending + expected_pending = {'green/2': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386']}} + self.assertEqual(self.pending_requests, expected_pending) + + # if we run britney again this should *not* trigger any new tests + self.do_test([], {'green': (False, {})}) + self.assertEqual(self.amqp_requests, set()) + # but the set of pending tests doesn't change + self.assertEqual(self.pending_requests, expected_pending) + + def test_multi_rdepends_with_tests_all_pass(self): + '''Multiple reverse dependencies with tests (all pass)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + # first run requests tests and marks them as pending + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + # a reverse dep that does not exist in testing should not be triggered + ('brittle', {'Depends': 'libgreen1'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]})[1] + self.assertNotIn('brittle', exc['green']['policy_info']['autopkgtest']) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/2')), + # version in testing fails + 'series/i386/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + # version in unstable succeeds + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + # new "brittle" succeeds + 'series/i386/b/brittle/20150101_100200@': (0, 'brittle 1', tr('brittle/1')), + 'series/amd64/b/brittle/20150101_100201@': (0, 'brittle 1', tr('brittle/1')), + }}) + + out = self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'brittle': (True, {'brittle/1': {'amd64': 'PASS', 'i386': 'PASS'}}) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + # all tests ran, there should be no more pending ones + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + # caches the results and triggers + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: + res = json.load(f) + self.assertEqual(res['green/1']['green']['amd64'], + [False, '1', '20150101_020000@']) + self.assertEqual(set(res['green/2']), {'darkgreen', 'green', 'lightgreen'}) + self.assertEqual(res['green/2']['lightgreen']['i386'], + [True, '1', '20150101_100100@']) + + # third run should not trigger any new tests, should all be in the + # cache + self.swift.set_results({}) + out = self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + })[0] + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_mixed(self): + '''Multiple reverse dependencies with tests (mixed results)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]}) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + # unrelated results (wrong trigger), ignore this! + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('blue/1')), + }}) + + out = self.do_test( + [], + {'green': (False, {'green/2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}, + 'darkgreen/1': {'amd64': 'RUNNING', 'i386': 'PASS'}, + }) + })[0] + + self.assertIn('Update Excuses generation completed', out) + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out) + + # there should be some pending ones + self.assertEqual(self.pending_requests, + {'green/2': {'darkgreen': ['amd64'], 'lightgreen': ['i386']}}) + + def test_results_without_triggers(self): + '''Old results without recorded triggers''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100100@': (0, 'green 1', tr('passedbefore/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + # none of the above results should be accepted + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }) + + # there should be some pending ones + self.assertEqual(self.pending_requests, + {'green/2': {'lightgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'darkgreen': ['amd64', 'i386']}}) + + def test_multi_rdepends_with_tests_regression(self): + '''Multiple reverse dependencies with tests (regression)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + }}) + + out, exc, _ = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + ) + + # should have links to log and history, but no artifacts (as this is + # not a PPA) + self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][:4], + ['REGRESSION', + 'http://localhost:18085/autopkgtest-series/series/amd64/l/lightgreen/20150101_100101@/log.gz', + 'http://autopkgtest.ubuntu.com/packages/l/lightgreen/series/amd64', + None]) + + # should have retry link for the regressions (not a stable URL, test + # seaprately) + link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][4]) + self.assertEqual(link.netloc, 'autopkgtest.ubuntu.com') + self.assertEqual(link.path, '/request.cgi') + self.assertEqual(urllib.parse.parse_qs(link.query), + {'release': ['series'], 'arch': ['amd64'], + 'package': ['lightgreen'], 'trigger': ['green/2']}) + + # we already had all results before the run, so this should not trigger + # any new requests + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_regression_last_pass(self): + '''Multiple reverse dependencies with tests (regression), last one passes + + This ensures that we don't just evaluate the test result of the last + test, but all of them. + ''' + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + self.assertEqual(self.pending_requests, {}) + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_always_failed(self): + '''Multiple reverse dependencies with tests (always failed)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (4, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'ALWAYSFAIL', 'i386': 'ALWAYSFAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + self.assertEqual(self.pending_requests, {}) + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_arch_specific(self): + '''Multiple reverse dependencies with arch specific tests''' + + # green has passed before on amd64, doesn't exist on i386 + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/g/green64/20150101_100000@': (0, 'green64 0.1', tr('passedbefore/1')), + }}) + + self.data.add('green64', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Architecture': 'amd64'}, + testsuite='autopkgtest') + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'green64': {'amd64': 'RUNNING'}, + }) + }) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green {"triggers": ["green/2"]}', + 'debci-series-amd64:green {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:green64 {"triggers": ["green/2"]}'])) + + self.assertEqual(self.pending_requests, + {'green/2': {'lightgreen': ['amd64', 'i386'], + 'darkgreen': ['amd64', 'i386'], + 'green64': ['amd64'], + 'green': ['amd64', 'i386']}}) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/2')), + # version in testing fails + 'series/i386/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + # version in unstable succeeds + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + # only amd64 result for green64 + 'series/amd64/g/green64/20150101_100200@': (0, 'green64 1', tr('green/2')), + }}) + + out = self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'green64/1': {'amd64': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + # all tests ran, there should be no more pending ones + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_unbuilt(self): + '''Unbuilt package should not trigger tests or get considered''' + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + exc = self.do_test( + # uninstallable unstable version + [], + {'green': (False, {})}, + {'green': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES), + ] + })[1] + # autopkgtest should not be triggered for unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest'], {}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_partial_unbuilt(self): + '''Unbuilt package on some arches should not trigger tests''' + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.data.add('libgreen1', True, {'Version': '2', 'Source': 'green', 'Architecture': 'i386'}, add_src=False) + exc = self.do_test( + [], + {'green': (False, {})}, + {'green': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', {'on-architectures': ['amd64', 'arm64', 'armhf', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []}) + ] + })[1] + # autopkgtest should not be triggered for unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest'], {}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_partial_unbuilt_block(self): + '''Unbuilt blocked package on some arches should not trigger tests''' + + self.create_hint('freeze', 'block-all source') + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.data.add('libgreen1', True, {'Version': '2', 'Source': 'green', 'Architecture': 'i386'}, add_src=False) + exc = self.do_test( + [], + {'green': (False, {})}, + {'green': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', {'on-architectures': ['amd64', 'arm64', 'armhf', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []}) + ] + })[1] + # autopkgtest should not be triggered for unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest'], {}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_rdepends_unbuilt(self): + '''Unbuilt reverse dependency''' + + # old lightgreen fails, thus new green should be held back + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/i386/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 1.1', tr('green/1.1')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 1.1', tr('green/1.1')), + }}) + + # add unbuilt lightgreen; should run tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)], + } + ) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run should not trigger any new requests + self.do_test([], {'green': (False, {}), 'lightgreen': (False, {})}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # now lightgreen 2 gets built, should trigger a new test run + self.data.remove_all(True) + self.do_test( + [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '2'}, 'autopkgtest')], + {}) + self.assertEqual(self.amqp_requests, + set(['debci-series-amd64:lightgreen {"triggers": ["lightgreen/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["lightgreen/2"]}'])) + + # next run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100200@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_102000@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + self.do_test( + [], + # green hasn't changed, the above re-run was for trigger lightgreen/2 + {'green': (False, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (True, {'lightgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2')], + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_rdepends_unbuilt_unstable_only(self): + '''Unbuilt reverse dependency which is not in testing''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + }}) + # run britney once to pick up previous results + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}})}) + + # add new uninstallable brokengreen; should not run test at all + exc = self.do_test( + [('brokengreen', {'Version': '1', 'Depends': 'libgreen1, nonexisting'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + 'brokengreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'brokengreen': [('old-version', '-'), ('new-version', '1'), + ('reason', 'depends'), + ('excuses', 'brokengreen/amd64 unsatisfiable Depends: nonexisting')], + })[1] + # autopkgtest should not be triggered for uninstallable pkg + self.assertEqual(exc['brokengreen']['policy_info']['autopkgtest'], {}) + + self.assertEqual(self.amqp_requests, set()) + + def test_rdepends_unbuilt_new_version_result(self): + '''Unbuilt reverse dependency gets test result for newer version + + This might happen if the autopkgtest infrastructure runs the unstable + source tests against the testing binaries. Even if that gets done + properly it might still happen that at the time of the britney run the + package isn't built yet, but it is once the test gets run. + ''' + # old lightgreen fails, thus new green should be held back + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/i386/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 1.1', tr('green/1.1')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 1.1', tr('green/1.1')), + }}) + + # add unbuilt lightgreen; should run tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)] + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # lightgreen 2 stays unbuilt in britney, but we get a test result for it + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100200@': (0, 'lightgreen 2', tr('green/1.1')), + 'series/amd64/l/lightgreen/20150101_102000@': (0, 'lightgreen 2', tr('green/1.1')), + }}) + self.do_test( + [], + {'green': (True, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)] + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run should not trigger any new requests + self.do_test([], {'green': (True, {}), 'lightgreen': (False, {})}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_rdepends_unbuilt_new_version_fail(self): + '''Unbuilt reverse dependency gets failure for newer version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')), + }}) + + # add unbuilt lightgreen; should request tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)], + } + ) + self.assertEqual(len(self.amqp_requests), 6) + + # we only get a result for lightgreen 2, not for the requested 1 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 0.5', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 0.5', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100200@': (4, 'lightgreen 2', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100200@': (4, 'lightgreen 2', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + }}) + self.do_test( + [], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)], + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run should not trigger any new requests + self.do_test([], {'green': (False, {}), 'lightgreen': (False, {})}) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + def test_same_version_binary_in_unstable(self): + '''binary from new architecture in unstable with testing version''' + + # i386 is in testing already, but amd64 just recently built and is in unstable + self.data.add_src('brown', False, {'Testsuite': 'autopkgtest'}) + self.data.add('brown', False, {'Architecture': 'i386'}, add_src=False) + self.data.add('brown', True, {'Architecture': 'amd64'}, add_src=False) + + exc = self.do_test( + # we need some other package to create unstable Sources + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'brown': (True, {})} + )[1] + self.assertEqual(exc['brown']['item-name'], 'brown/amd64') + + def test_package_pair_running(self): + '''Two packages in unstable that need to go in together (running)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + 'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '2')], + }) + + # we expect the package's and its reverse dependencies' tests to get + # triggered; lightgreen should be triggered for each trigger + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green {"triggers": ["green/2"]}', + 'debci-series-amd64:green {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["lightgreen/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["lightgreen/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}'])) + + # ... and that they get recorded as pending + self.assertEqual(self.pending_requests, + {'lightgreen/2': {'lightgreen': ['amd64', 'i386']}, + 'green/2': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386']}}) + + def test_binary_from_new_source_package_running(self): + '''building an existing binary for a new source package (running)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'newgreen': (True, {'newgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'newgreen': [('old-version', '-'), ('new-version', '2')]}) + + self.assertEqual(len(self.amqp_requests), 8) + self.assertEqual(self.pending_requests, + {'newgreen/2': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386'], + 'newgreen': ['amd64', 'i386']}}) + + def test_binary_from_new_source_package_pass(self): + '''building an existing binary for a new source package (pass)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('newgreen/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('newgreen/2')), + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('newgreen/2')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 1', tr('newgreen/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('newgreen/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('newgreen/2')), + 'series/i386/n/newgreen/20150101_100200@': (0, 'newgreen 2', tr('newgreen/2')), + 'series/amd64/n/newgreen/20150101_100201@': (0, 'newgreen 2', tr('newgreen/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'newgreen': (True, {'newgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'green/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'newgreen': [('old-version', '-'), ('new-version', '2')]}) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_result_from_older_version(self): + '''test result from older version than the uploaded one''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('darkgreen/1')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('darkgreen/1')), + }}) + + self.do_test( + [('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], + {'darkgreen': (False, {'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})}) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:darkgreen {"triggers": ["darkgreen/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["darkgreen/2"]}'])) + self.assertEqual(self.pending_requests, + {'darkgreen/2': {'darkgreen': ['amd64', 'i386']}}) + + # second run gets the results for darkgreen 2 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100010@': (0, 'darkgreen 2', tr('darkgreen/2')), + 'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 2', tr('darkgreen/2')), + }}) + self.do_test( + [], + {'darkgreen': (True, {'darkgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}})}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run sees a newer darkgreen, should re-run tests + self.data.remove_all(True) + self.do_test( + [('darkgreen', {'Version': '3', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], + {'darkgreen': (False, {'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})}) + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:darkgreen {"triggers": ["darkgreen/3"]}', + 'debci-series-amd64:darkgreen {"triggers": ["darkgreen/3"]}'])) + self.assertEqual(self.pending_requests, + {'darkgreen/3': {'darkgreen': ['amd64', 'i386']}}) + + def test_old_result_from_rdep_version(self): + '''re-runs reverse dependency test on new versions''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100010@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100010@': (0, 'green 2', tr('green/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + self.data.remove_all(True) + + # second run: new version re-triggers all tests + self.do_test( + [('libgreen1', {'Version': '3', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + 'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + }), + }) + + self.assertEqual(len(self.amqp_requests), 6) + self.assertEqual(self.pending_requests, + {'green/3': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386']}}) + + # third run gets the results for green and lightgreen, darkgreen is + # still running + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100020@': (0, 'green 3', tr('green/3')), + 'series/amd64/g/green/20150101_100020@': (0, 'green 3', tr('green/3')), + 'series/i386/l/lightgreen/20150101_100010@': (0, 'lightgreen 1', tr('green/3')), + 'series/amd64/l/lightgreen/20150101_100010@': (0, 'lightgreen 1', tr('green/3')), + }}) + self.do_test( + [], + {'green': (False, {'green/3': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + }), + }) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, + {'green/3': {'darkgreen': ['amd64', 'i386']}}) + + # fourth run finally gets the new darkgreen result + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100010@': (0, 'darkgreen 1', tr('green/3')), + 'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 1', tr('green/3')), + }}) + self.do_test( + [], + {'green': (True, {'green/3': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_different_versions_on_arches(self): + '''different tested package versions on different architectures''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('passedbefore/1')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('passedbefore/1')), + }}) + + # first run: no results yet + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')], + {'green': (False, {'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})}) + + # second run: i386 result has version 1.1 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100010@': (0, 'darkgreen 1.1', tr('green/2')) + }}) + self.do_test( + [], + {'green': (False, {'darkgreen': {'amd64': 'RUNNING'}, + 'darkgreen/1.1': {'i386': 'PASS'}, + })}) + + # third run: amd64 result has version 1.2 + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 1.2', tr('green/2')), + }}) + self.do_test( + [], + {'green': (True, {'darkgreen/1.2': {'amd64': 'PASS'}, + 'darkgreen/1.1': {'i386': 'PASS'}, + })}) + + def test_tmpfail(self): + '''tmpfail results''' + + # one tmpfail result without testpkg-version, should be ignored + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'series/i386/l/lightgreen/20150101_100101@': (16, None, tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (16, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}})}) + self.assertEqual(self.pending_requests, + {'lightgreen/2': {'lightgreen': ['i386']}}) + + # one more tmpfail result, should not confuse britney with None version + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100201@': (16, None, tr('lightgreen/2')), + }}) + self.do_test( + [], + {'lightgreen': (False, {'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}})}) + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: + contents = f.read() + self.assertNotIn('null', contents) + self.assertNotIn('None', contents) + + def test_rerun_failure(self): + '''manually re-running failed tests gets picked up''' + + # first run fails + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 2', tr('green/1')), + 'series/i386/g/green/20150101_100101@': (4, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100101@': (4, 'green 2', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + + # re-running test manually succeeded (note: darkgreen result should be + # cached already) + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100201@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100201@': (0, 'lightgreen 1', tr('green/2')), + }}) + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + + def test_new_runs_dont_clobber_pass(self): + '''passing once is sufficient + + If a test succeeded once for a particular version and trigger, + subsequent failures (which might be triggered by other unstable + uploads) should not invalidate the PASS, as that new failure is the + fault of the new upload, not the original one. + ''' + # new libc6 works fine with green + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('libc6/2')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 1', tr('libc6/2')), + }}) + + self.do_test( + [('libc6', {'Version': '2'}, None)], + {'libc6': (True, {'green/1': {'amd64': 'PASS', 'i386': 'PASS'}})}) + self.assertEqual(self.pending_requests, {}) + + # new green fails; that's not libc6's fault though, so it should stay + # valid + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100100@': (4, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100100@': (4, 'green 2', tr('green/2')), + }}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}}), + 'libc6': (True, {'green/1': {'amd64': 'PASS', 'i386': 'PASS'}}), + }) + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + ])) + + def test_remove_from_unstable(self): + '''broken package gets removed from unstable''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100101@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_100101@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100201@': (4, 'lightgreen 2', tr('green/2 lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100201@': (4, 'lightgreen 2', tr('green/2 lightgreen/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + # remove new lightgreen by resetting archive indexes, and re-adding + # green + self.data.remove_all(True) + + self.swift.set_results({'autopkgtest-series': { + # add new result for lightgreen 1 + 'series/i386/l/lightgreen/20150101_100301@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100301@': (0, 'lightgreen 1', tr('green/2')), + }}) + + # next run should re-trigger lightgreen 1 to test against green/2 + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + })[1] + self.assertNotIn('lightgreen 2', exc['green']['policy_info']['autopkgtest']) + + # should not trigger new requests + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + # but the next run should not trigger anything new + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + def test_multiarch_dep(self): + '''multi-arch dependency''' + + # lightgreen has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + }}) + + self.data.add('rainbow', False, {'Depends': 'lightgreen:any'}, + testsuite='autopkgtest') + + self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'rainbow': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'lightgreen': [('old-version', '1'), ('new-version', '2')]} + ) + + def test_nbs(self): + '''source-less binaries do not cause harm''' + + # NBS in testing + self.data.add('liboldgreen0', False, add_src=False) + # NBS in unstable + self.data.add('liboldgreen1', True, add_src=False) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')], + {'green': (True, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')]}) + + def test_newer_version_in_testing(self): + '''Testing version is newer than in unstable''' + + exc = self.do_test( + [('lightgreen', {'Version': '0.9~beta'}, 'autopkgtest')], + {'lightgreen': (False, {})}, + {'lightgreen': [('old-version', '1'), ('new-version', '0.9~beta'), + ('reason', 'newerintesting'), + ('excuses', 'ALERT: lightgreen is newer in testing (1 0.9~beta)') + ] + })[1] + + # autopkgtest should not be triggered + self.assertNotIn('autopkgtest', exc['lightgreen'].get('policy_info', {})) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + def test_testsuite_triggers(self): + '''Testsuite-Triggers''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/r/rainbow/20150101_100000@': (0, 'rainbow 1', tr('passedbefore/1')), + }}) + + self.data.add('rainbow', False, testsuite='autopkgtest', + srcfields={'Testsuite-Triggers': 'unicorn, lightgreen, sugar'}) + + self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'rainbow': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + }), + } + ) + + def test_huge_number_of_tests(self): + '''package triggers huge number of tests''' + + for i in range(30): + self.data.add('green%i' % i, False, {'Depends': 'libgreen1'}, testsuite='autopkgtest') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')], + {'green': (True, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'green0': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'green29': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + ) + + # requests should all go into the -huge queues + self.assertEqual([x for x in self.amqp_requests if 'huge' not in x], []) + for i in range(30): + for arch in ['i386', 'amd64']: + self.assertIn('debci-huge-series-%s:green%i {"triggers": ["green/2"]}' % + (arch, i), self.amqp_requests) + + ################################################################ + # Tests for hint processing + ################################################################ + + def test_hint_force_badtest(self): + '''force-badtest hint''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_badtest_multi_version(self): + '''force-badtest hint''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'i386': 'IGNORE-FAIL'}, + 'lightgreen/2': {'amd64': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + # hint the version on amd64 too + self.create_hint('pitti', 'force-badtest lightgreen/2') + + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'i386': 'IGNORE-FAIL'}, + 'lightgreen/2': {'amd64': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_badtest_different_version(self): + '''force-badtest hint with non-matching version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + # lower hint version should not apply + self.create_hint('pitti', 'force-badtest lightgreen/0.1') + + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('reason', 'autopkgtest')]} + )[1] + self.assertNotIn('forced-reason', exc['green']) + + # higher hint version should apply + self.create_hint('pitti', 'force-badtest lightgreen/3') + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {} + ) + + def test_hint_force_badtest_arch(self): + '''force-badtest hint for architecture instead of version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/amd64/all') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + # hint i386 too, then it should become valid + self.create_hint('pitti', 'force-badtest lightgreen/i386/all') + + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_badtest_running(self): + '''force-badtest hint on running test''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_skiptest(self): + '''force-skiptest hint''' + + self.create_hint('pitti', 'force-skiptest green/2') + + # regression of green, darkgreen ok, lightgreen running + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + 'series/i386/g/green/20150101_100200@': (4, 'green 2', tr('green/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + }}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'REGRESSION'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2'), + ('forced-reason', 'skiptest'), + ('excuses', 'Should wait for tests relating to green 2, but forced by pitti')] + }) + + def test_hint_force_skiptest_different_version(self): + '''force-skiptest hint with non-matching version''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + self.create_hint('pitti', 'force-skiptest green/1') + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'green': [('reason', 'autopkgtest')]} + )[1] + self.assertNotIn('forced-reason', exc['green']) + + def test_hint_blockall_runs_tests(self): + '''block-all hint still runs tests''' + + self.create_hint('freeze', 'block-all source') + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})} + ) + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [], + {'lightgreen': (False, {'lightgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}})}, + {'lightgreen': [('reason', 'block')]} + ) + + + ################################################################ + # Tests for non-hint policies + ################################################################ + + def test_lp_bug_block(self): + with open(os.path.join(self.data.path, 'data/series-proposed/Blocks'), 'w') as f: + f.write('darkgreen 12345 1471505000\ndarkgreen 98765 1471500000\n') + + exc = self.do_test( + [('darkgreen', {'Version': '2'}, 'autopkgtest')], + {'darkgreen': (False, {'darkgreen': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}})}, + {'darkgreen': [('reason', 'block'), + ('excuses', 'Not touching package as requested in bug 12345 on Thu Aug 18 07:23:20 2016'), + ('is-candidate', False), + ] + } + )[1] + self.assertEqual(exc['darkgreen']['policy_info']['block-bugs'], + {'12345': 1471505000, '98765': 1471500000}) + + + ################################################################ + # Kernel related tests + ################################################################ + + def test_detect_dkms_autodep8(self): + '''DKMS packages are autopkgtested (via autodep8)''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/f/fancy/20150101_100101@': (0, 'fancy 0.1', tr('passedbefore/1')) + }}) + + self.do_test( + [('dkms', {'Version': '2'}, None)], + {'dkms': (False, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}})}, + {'dkms': [('old-version', '1'), ('new-version', '2')]}) + + def test_kernel_triggers_dkms(self): + '''DKMS packages get triggered by kernel uploads''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + self.do_test( + [('linux-image-generic', {'Source': 'linux-meta'}, None), + ('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ], + {'linux-meta': (True, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}}), + 'linux-meta-lts-grumpy': (True, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}}), + 'linux-meta-64only': (True, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL'}}), + }) + + # one separate test should be triggered for each kernel + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:fancy {"triggers": ["linux-meta/1"]}', + 'debci-series-amd64:fancy {"triggers": ["linux-meta/1"]}', + 'debci-series-i386:fancy {"triggers": ["linux-meta-lts-grumpy/1"]}', + 'debci-series-amd64:fancy {"triggers": ["linux-meta-lts-grumpy/1"]}', + 'debci-series-amd64:fancy {"triggers": ["linux-meta-64only/1"]}'])) + + # ... and that they get recorded as pending + self.assertEqual(self.pending_requests, + {'linux-meta-lts-grumpy/1': {'fancy': ['amd64', 'i386']}, + 'linux-meta/1': {'fancy': ['amd64', 'i386']}, + 'linux-meta-64only/1': {'fancy': ['amd64']}}) + + def test_dkms_results_per_kernel(self): + '''DKMS results get mapped to the triggering kernel version''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + # works against linux-meta and -64only, fails against grumpy i386, no + # result yet for grumpy amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/f/fancy/20150101_100301@': (0, 'fancy 0.5', tr('passedbefore/1')), + 'series/i386/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100201@': (0, 'fancy 1', tr('linux-meta-64only/1')), + 'series/i386/f/fancy/20150101_100301@': (4, 'fancy 1', tr('linux-meta-lts-grumpy/1')), + }}) + + self.do_test( + [('linux-image-generic', {'Source': 'linux-meta'}, None), + ('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ], + {'linux-meta': (True, {'fancy/1': {'amd64': 'PASS', 'i386': 'PASS'}}), + 'linux-meta-lts-grumpy': (False, {'fancy/1': {'amd64': 'RUNNING', 'i386': 'ALWAYSFAIL'}}), + 'linux-meta-64only': (True, {'fancy/1': {'amd64': 'PASS'}}), + }) + + self.assertEqual(self.pending_requests, + {'linux-meta-lts-grumpy/1': {'fancy': ['amd64']}}) + + def test_dkms_results_per_kernel_old_results(self): + '''DKMS results get mapped to the triggering kernel version, old results''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + # works against linux-meta and -64only, fails against grumpy i386, no + # result yet for grumpy amd64 + self.swift.set_results({'autopkgtest-series': { + # old results without trigger info + 'series/i386/f/fancy/20140101_100101@': (0, 'fancy 1', {}), + 'series/amd64/f/fancy/20140101_100101@': (8, 'fancy 1', {}), + # current results with triggers + 'series/i386/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100201@': (0, 'fancy 1', tr('linux-meta-64only/1')), + 'series/i386/f/fancy/20150101_100301@': (4, 'fancy 1', tr('linux-meta-lts-grumpy/1')), + }}) + + self.do_test( + [('linux-image-generic', {'Source': 'linux-meta'}, None), + ('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ], + {'linux-meta': (True, {'fancy/1': {'amd64': 'PASS', 'i386': 'PASS'}}), + # we don't have an explicit result for amd64 + 'linux-meta-lts-grumpy': (False, {'fancy/1': {'amd64': 'RUNNING', 'i386': 'ALWAYSFAIL'}}), + 'linux-meta-64only': (True, {'fancy/1': {'amd64': 'PASS'}}), + }) + + self.assertEqual(self.pending_requests, + {'linux-meta-lts-grumpy/1': {'fancy': ['amd64']}}) + + def test_kernel_triggered_tests(self): + '''linux, lxc, glibc tests get triggered by linux-meta* uploads''' + + self.data.remove_all(False) + self.data.add('libc6-dev', False, {'Source': 'glibc', 'Depends': 'linux-libc-dev'}, + testsuite='autopkgtest') + self.data.add('lxc', False, {'Testsuite-Triggers': 'linux-generic'}, + testsuite='autopkgtest') + self.data.add('systemd', False, {'Testsuite-Triggers': 'linux-generic'}, + testsuite='autopkgtest') + self.data.add('linux-image-1', False, {'Source': 'linux'}, testsuite='autopkgtest') + self.data.add('linux-libc-dev', False, {'Source': 'linux'}, testsuite='autopkgtest') + self.data.add('linux-image', False, {'Source': 'linux-meta', 'Depends': 'linux-image-1'}) + + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/l/lxc/20150101_100101@': (0, 'lxc 0.1', tr('passedbefore/1')) + }}) + + exc = self.do_test( + [('linux-image', {'Version': '2', 'Depends': 'linux-image-2', 'Source': 'linux-meta'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ('linux-image-2', {'Version': '2', 'Source': 'linux'}, 'autopkgtest'), + ('linux-libc-dev', {'Version': '2', 'Source': 'linux'}, 'autopkgtest'), + ], + {'linux-meta': (False, {'lxc': {'amd64': 'RUNNING', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'glibc': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'linux': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'systemd': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + 'linux-meta-64only': (False, {'lxc': {'amd64': 'RUNNING'}}), + 'linux': (False, {}), + })[1] + # the kernel itself should not trigger tests; we want to trigger + # everything from -meta + self.assertEqual(exc['linux']['policy_info']['autopkgtest'], {}) + + def test_kernel_waits_on_meta(self): + '''linux waits on linux-meta''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + self.data.add('linux-image-generic', False, {'Version': '0.1', 'Source': 'linux-meta', 'Depends': 'linux-image-1'}) + self.data.add('linux-image-1', False, {'Source': 'linux'}, testsuite='autopkgtest') + self.data.add('linux-firmware', False, {'Source': 'linux-firmware'}, testsuite='autopkgtest') + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/f/fancy/20150101_090000@': (0, 'fancy 0.5', tr('passedbefore/1')), + 'series/i386/l/linux/20150101_100000@': (0, 'linux 2', tr('linux-meta/0.2')), + 'series/amd64/l/linux/20150101_100000@': (0, 'linux 2', tr('linux-meta/0.2')), + 'series/i386/l/linux-firmware/20150101_100000@': (0, 'linux-firmware 2', tr('linux-firmware/2')), + 'series/amd64/l/linux-firmware/20150101_100000@': (0, 'linux-firmware 2', tr('linux-firmware/2')), + }}) + + self.do_test( + [('linux-image-generic', {'Version': '0.2', 'Source': 'linux-meta', 'Depends': 'linux-image-2'}, None), + ('linux-image-2', {'Version': '2', 'Source': 'linux'}, 'autopkgtest'), + ('linux-firmware', {'Version': '2', 'Source': 'linux-firmware'}, 'autopkgtest'), + ], + {'linux-meta': (False, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'linux/2': {'amd64': 'PASS', 'i386': 'PASS'} + }), + # no tests, but should wait on linux-meta + 'linux': (False, {}), + # this one does not have a -meta, so don't wait + 'linux-firmware': (True, {'linux-firmware/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + }, + {'linux': [('reason', 'depends'), + ('excuses', 'Invalidated by dependency'), + ('dependencies', {'blocked-by': ['linux-meta']})] + } + ) + + # now linux-meta is ready to go + self.swift.set_results({'autopkgtest-series': { + 'series/i386/f/fancy/20150101_100000@': (0, 'fancy 1', tr('linux-meta/0.2')), + 'series/amd64/f/fancy/20150101_100000@': (0, 'fancy 1', tr('linux-meta/0.2')), + }}) + self.do_test( + [], + {'linux-meta': (True, {'fancy/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'linux/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + 'linux': (True, {}), + 'linux-firmware': (True, {'linux-firmware/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + }, + {'linux': [('dependencies', {'migrate-after': ['linux-meta']})] + } + ) + + ################################################################ + # Tests for special-cased packages + ################################################################ + + def test_gcc(self): + '''gcc only triggers some key packages''' + + self.data.add('binutils', False, {}, testsuite='autopkgtest') + self.data.add('linux', False, {}, testsuite='autopkgtest') + self.data.add('notme', False, {'Depends': 'libgcc1'}, testsuite='autopkgtest') + + # binutils has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/b/binutils/20150101_100000@': (0, 'binutils 1', tr('passedbefore/1')), + }}) + + exc = self.do_test( + [('libgcc1', {'Source': 'gcc-5', 'Version': '2'}, None)], + {'gcc-5': (False, {'binutils': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'linux': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}})})[1] + self.assertNotIn('notme 1', exc['gcc-5']['policy_info']['autopkgtest']) + + def test_alternative_gcc(self): + '''alternative gcc does not trigger anything''' + + self.data.add('binutils', False, {}, testsuite='autopkgtest') + self.data.add('notme', False, {'Depends': 'libgcc1'}, testsuite='autopkgtest') + + exc = self.do_test( + [('libgcc1', {'Source': 'gcc-snapshot', 'Version': '2'}, None)], + {'gcc-snapshot': (True, {})})[1] + self.assertEqual(exc['gcc-snapshot']['policy_info']['autopkgtest'], {}) + + ################################################################ + # Tests for non-default ADT_* configuration modes + ################################################################ + + def test_disable_adt(self): + '''Run without autopkgtest requests''' + + # Disable AMQP server config, to ensure we don't touch them with ADT + # disabled + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('ADT_ENABLE'): + print('ADT_ENABLE = no') + elif not line.startswith('ADT_AMQP') and not line.startswith('ADT_SWIFT_URL'): + sys.stdout.write(line) + + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {})}, + {'green': [('old-version', '1'), ('new-version', '2')]})[1] + self.assertNotIn('autopkgtest', exc['green']['policy_info']) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, None) + + def test_ppas(self): + '''Run test requests with additional PPAs''' + + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('ADT_PPAS'): + print('ADT_PPAS = joe/foo awesome-developers/staging') + else: + sys.stdout.write(line) + + exc = self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL'}})}, + {'lightgreen': [('old-version', '1'), ('new-version', '2')]} + )[1] + self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], + {'lightgreen': { + 'amd64': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + None, + None, + None], + 'i386': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + None, + None, + None]} + }) + + for arch in ['i386', 'amd64']: + self.assertTrue('debci-ppa-series-%s:lightgreen {"triggers": ["lightgreen/2"], "ppas": ["joe/foo", "awesome-developers/staging"]}' % arch in self.amqp_requests or + 'debci-ppa-series-%s:lightgreen {"ppas": ["joe/foo", "awesome-developers/staging"], "triggers": ["lightgreen/2"]}' % arch in self.amqp_requests, + self.amqp_requests) + self.assertEqual(len(self.amqp_requests), 2) + + # add results to PPA specific swift container + self.swift.set_results({'autopkgtest-series-awesome-developers-staging': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + exc = self.do_test( + [], + {'lightgreen': (False, {'lightgreen/2': {'i386': 'REGRESSION', 'amd64': 'PASS'}})}, + {'lightgreen': [('old-version', '1'), ('new-version', '2')]} + )[1] + self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], + {'lightgreen/2': { + 'amd64': ['PASS', + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/amd64/l/lightgreen/20150101_100101@/log.gz', + None, + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/amd64/l/lightgreen/20150101_100101@/artifacts.tar.gz', + None], + 'i386': ['REGRESSION', + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/i386/l/lightgreen/20150101_100100@/log.gz', + None, + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/i386/l/lightgreen/20150101_100100@/artifacts.tar.gz', + 'https://autopkgtest.ubuntu.com/request.cgi?release=series&arch=i386&package=lightgreen&' + 'trigger=lightgreen%2F2&ppa=joe%2Ffoo&ppa=awesome-developers%2Fstaging']} + }) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_disable_upgrade_tester(self): + '''Run without second stage upgrade tester''' + + for line in fileinput.input(self.britney_conf, inplace=True): + if not line.startswith('UPGRADE_OUTPUT'): + sys.stdout.write(line) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {})[1] + + self.assertFalse(os.path.exists(os.path.join(self.data.path, 'output', 'series', 'output.txt'))) + + def test_shared_results_cache(self): + '''Run with shared r/o results.cache''' + + # first run to create results.cache + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen/2': {'i386': 'PASS', 'amd64': 'PASS'}})}, + ) + + # move and remember original contents + local_path = os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache') + shared_path = os.path.join(self.data.path, 'shared_results.cache') + os.rename(local_path, shared_path) + with open(shared_path) as f: + orig_contents = f.read() + + # enable shared cache + for line in fileinput.input(self.britney_conf, inplace=True): + if 'ADT_SHARED_RESULTS_CACHE' in line: + print('ADT_SHARED_RESULTS_CACHE = %s' % shared_path) + else: + sys.stdout.write(line) + + # second run, should now not update cache + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 3', tr('lightgreen/3')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.data.remove_all(True) + self.do_test( + [('lightgreen', {'Version': '3', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen/3': {'i386': 'PASS', 'amd64': 'PASS'}})}, + ) + + # leaves results.cache untouched + self.assertFalse(os.path.exists(local_path)) + with open(shared_path) as f: + self.assertEqual(orig_contents, f.read()) + + +if __name__ == '__main__': + unittest.main()