diff --git a/britney.conf b/britney.conf index 974c846..921461d 100644 --- a/britney.conf +++ b/britney.conf @@ -88,3 +88,15 @@ SMOOTH_UPDATES = badgers IGNORE_CRUFT = 0 REMOVE_OBSOLETE = no + +ADT_ENABLE = yes +ADT_ARCHES = amd64 i386 armhf ppc64el +ADT_AMQP = amqp://test_request:password@162.213.33.228 +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac +# space separate list of PPAs to add for test requests and for polling results; +# the *last* one determines the swift container name +ADT_PPAS = +# set this to the path of a (r/o) results.cache for running many parallel +# britney instances for PPAs without updating the cache +ADT_SHARED_RESULTS_CACHE = diff --git a/britney.py b/britney.py index ce8fafd..adf4083 100755 --- a/britney.py +++ b/britney.py @@ -203,6 +203,7 @@ from britney2.installability.builder import build_installability_tester, ma_pars from britney2.migrationitem import MigrationItem from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, PolicyVerdict from britney2.policies.policy import LPBlockBugPolicy +from britney2.policies.autopkgtest import AutopkgtestPolicy from britney2.utils import (old_libraries_format, undo_changes, compute_reverse_tree, possibly_compressed, read_nuninst, write_nuninst, write_heidi, @@ -531,6 +532,8 @@ class Britney(object): self.policies.append(RCBugPolicy(self.options, self.suite_info)) self.policies.append(PiupartsPolicy(self.options, self.suite_info)) self.policies.append(LPBlockBugPolicy(self.options, self.suite_info)) + if getattr(self.options, 'adt_enable') == 'yes': + self.policies.append(AutopkgtestPolicy(self.options, self.suite_info)) for policy in self.policies: policy.register_hints(self._hint_parser) diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py new file mode 100644 index 0000000..2fef0a1 --- /dev/null +++ b/britney2/policies/autopkgtest.py @@ -0,0 +1,723 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 - 2016 Canonical Ltd. +# Authors: +# Colin Watson +# Jean-Baptiste Lallement +# Martin Pitt + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import os +import json +import tarfile +import io +import re +import sys +import urllib.parse +from urllib.request import urlopen + +import apt_pkg +import amqplib.client_0_8 as amqp + +import britney2.hints +from britney2.policies.policy import BasePolicy, PolicyVerdict +from britney2.consts import VERSION + + +EXCUSES_LABELS = { + "PASS": 'Pass', + "FAIL": 'Failed', + "ALWAYSFAIL": 'Always failed', + "REGRESSION": 'Regression', + "IGNORE-FAIL": 'Ignored failure', + "RUNNING": 'Test in progress', + "RUNNING-ALWAYSFAIL": 'Test in progress (always failed)', +} + + +def srchash(src): + '''archive hash prefix for source package''' + + if src.startswith('lib'): + return src[:4] + else: + return src[0] + + +class AutopkgtestPolicy(BasePolicy): + """autopkgtest regression policy for source migrations + + Run autopkgtests for the excuse and all of its reverse dependencies, and + reject the upload if any of those regress. + """ + + def __init__(self, options, suite_info): + super().__init__('autopkgtest', options, suite_info, {'unstable'}) + self.test_state_dir = os.path.join(options.unstable, 'autopkgtest') + # tests requested in this and previous runs + # trigger -> src -> [arch] + self.pending_tests = None + self.pending_tests_file = os.path.join(self.test_state_dir, 'pending.json') + + # results map: trigger -> src -> arch -> [passed, version, run_id] + # - trigger is "source/version" of an unstable package that triggered + # this test run. + # - "passed" is a bool + # - "version" is the package version of "src" of that test + # - "run_id" is an opaque ID that identifies a particular test run for + # a given src/arch. It's usually a time stamp like "20150120_125959". + # This is also used for tracking the latest seen time stamp for + # requesting only newer results. + self.test_results = {} + if self.options.adt_shared_results_cache: + self.results_cache_file = self.options.adt_shared_results_cache + else: + self.results_cache_file = os.path.join(self.test_state_dir, 'results.cache') + + try: + self.options.adt_ppas = self.options.adt_ppas.strip().split() + except AttributeError: + self.options.adt_ppas = [] + + self.swift_container = 'autopkgtest-' + options.series + if self.options.adt_ppas: + self.swift_container += '-' + options.adt_ppas[-1].replace('/', '-') + + # restrict adt_arches to architectures we actually run for + self.adt_arches = [] + for arch in self.options.adt_arches.split(): + if arch in self.options.architectures: + self.adt_arches.append(arch) + else: + self.log("Ignoring ADT_ARCHES %s as it is not in architectures list" % arch) + + def register_hints(self, hint_parser): + hint_parser.register_hint_type('force-badtest', britney2.hints.split_into_one_hint_per_package) + hint_parser.register_hint_type('force-skiptest', britney2.hints.split_into_one_hint_per_package) + + def initialise(self, britney): + super().initialise(britney) + os.makedirs(self.test_state_dir, exist_ok=True) + self.read_pending_tests() + + # read the cached results that we collected so far + if os.path.exists(self.results_cache_file): + with open(self.results_cache_file) as f: + self.test_results = json.load(f) + self.log('Read previous results from %s' % self.results_cache_file) + else: + self.log('%s does not exist, re-downloading all results from swift' % + self.results_cache_file) + + # we need sources, binaries, and installability tester, so for now + # remember the whole britney object + self.britney = britney + + # Initialize AMQP connection + self.amqp_channel = None + self.amqp_file = None + if self.options.dry_run: + return + + amqp_url = self.options.adt_amqp + + if amqp_url.startswith('amqp://'): + # in production mode, connect to AMQP server + creds = urllib.parse.urlsplit(amqp_url, allow_fragments=False) + self.amqp_con = amqp.Connection(creds.hostname, userid=creds.username, + password=creds.password) + self.amqp_channel = self.amqp_con.channel() + self.log('Connected to AMQP server') + elif amqp_url.startswith('file://'): + # in testing mode, adt_amqp will be a file:// URL + self.amqp_file = amqp_url[7:] + else: + raise RuntimeError('Unknown ADT_AMQP schema %s' % amqp_url.split(':', 1)[0]) + + def save_state(self, britney): + super().save_state(britney) + + # update the results on-disk cache, unless we are using a r/o shared one + if not self.options.adt_shared_results_cache: + self.log('Updating results cache') + with open(self.results_cache_file + '.new', 'w') as f: + json.dump(self.test_results, f, indent=2) + os.rename(self.results_cache_file + '.new', self.results_cache_file) + + # update the pending tests on-disk cache + self.log('Updating pending requested tests in %s' % self.pending_tests_file) + with open(self.pending_tests_file + '.new', 'w') as f: + json.dump(self.pending_tests, f, indent=2) + os.rename(self.pending_tests_file + '.new', self.pending_tests_file) + + def apply_policy_impl(self, tests_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse): + # skip/delay autopkgtests until package is built + if excuse.missing_builds or 'depends' in excuse.reason: + self.log('%s has missing builds or is uninstallable, skipping autopkgtest policy' % excuse.name) + return PolicyVerdict.REJECTED_TEMPORARILY + + self.log('Checking autopkgtests for %s' % source_name) + trigger = source_name + '/' + source_data_srcdist.version + + # build a (testsrc, testver) → arch → (status, log_url) map; we trigger/check test + # results per archtitecture for technical/efficiency reasons, but we + # want to evaluate and present the results by tested source package + # first + pkg_arch_result = {} + for arch in self.adt_arches: + # request tests (unless they were already requested earlier or have a result) + tests = self.tests_for_source(source_name, source_data_srcdist.version, arch) + is_huge = len(tests) > 20 + for (testsrc, testver) in tests: + self.pkg_test_request(testsrc, arch, trigger, huge=is_huge) + (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger) + pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url) + + # add test result details to Excuse + verdict = PolicyVerdict.PASS + cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" + for ((testsrc, testver), arch_results) in pkg_arch_result.items(): + r = set([v[0] for v in arch_results.values()]) + if 'REGRESSION' in r: + verdict = PolicyVerdict.REJECTED_PERMANENTLY + elif 'RUNNING' in r and verdict == PolicyVerdict.PASS: + verdict = PolicyVerdict.REJECTED_TEMPORARILY + # skip version if still running on all arches + if not r - {'RUNNING', 'RUNNING-ALWAYSFAIL'}: + testver = None + + html_archmsg = [] + for arch, (status, log_url) in arch_results.items(): + artifact_url = None + retry_url = None + history_url = None + if self.options.adt_ppas: + if log_url.endswith('log.gz'): + artifact_url = log_url.replace('log.gz', 'artifacts.tar.gz') + else: + history_url = cloud_url % { + 'h': srchash(testsrc), 's': testsrc, + 'r': self.options.series, 'a': arch} + if status == 'REGRESSION': + retry_url = 'https://autopkgtest.ubuntu.com/request.cgi?' + \ + urllib.parse.urlencode([('release', self.options.series), + ('arch', arch), + ('package', testsrc), + ('trigger', trigger)] + + [('ppa', p) for p in self.options.adt_ppas]) + if testver: + testname = '%s/%s' % (testsrc, testver) + else: + testname = testsrc + + tests_info.setdefault(testname, {})[arch] = \ + [status, log_url, history_url, artifact_url, retry_url] + + # render HTML snippet for testsrc entry for current arch + if history_url: + message = '%s' % (history_url, arch) + else: + message = arch + message += ': %s' % (log_url, EXCUSES_LABELS[status]) + if retry_url: + message += ' ♻ ' % retry_url + if artifact_url: + message += ' [artifacts]' % artifact_url + html_archmsg.append(message) + + # render HTML line for testsrc entry + excuse.addhtml("autopkgtest for %s: %s" % (testname, ', '.join(html_archmsg))) + + + if verdict != PolicyVerdict.PASS: + # check for force-skiptest hint + hints = self.britney.hints.search('force-skiptest', package=source_name, version=source_data_srcdist.version) + if hints: + excuse.addreason('skiptest') + excuse.addhtml("Should wait for tests relating to %s %s, but forced by %s" % + (source_name, source_data_srcdist.version, hints[0].user)) + excuse.force() + verdict = PolicyVerdict.PASS_HINTED + else: + excuse.addreason('autopkgtest') + return verdict + + # + # helper functions + # + + @classmethod + def has_autodep8(kls, srcinfo, binaries): + '''Check if package is covered by autodep8 + + srcinfo is an item from self.britney.sources + binaries is self.britney.binaries['unstable'][arch][0] + ''' + # autodep8? + for t in srcinfo.testsuite: + if t.startswith('autopkgtest-pkg'): + return True + + # DKMS: some binary depends on "dkms" + for pkg_id in srcinfo.binaries: + try: + bininfo = binaries[pkg_id.package_name] + except KeyError: + continue + if 'dkms' in (bininfo.depends or ''): + return True + return False + + def tests_for_source(self, src, ver, arch): + '''Iterate over all tests that should be run for given source and arch''' + + sources_info = self.britney.sources['testing'] + binaries_info = self.britney.binaries['testing'][arch][0] + + reported_pkgs = set() + + tests = [] + + # hack for vivid's gccgo-5 and xenial's gccgo-6; these build libgcc1 + # too, so test some Go and some libgcc1 consumers + if src in ['gccgo-5', 'gccgo-6']: + for test in ['juju-mongodb', 'mongodb', 'libreoffice']: + try: + tests.append((test, self.britney.sources['testing'][test][VERSION])) + except KeyError: + # no package in that series? *shrug*, then not (mostly for testing) + pass + return tests + + # gcc-N triggers tons of tests via libgcc1, but this is mostly in vain: + # gcc already tests itself during build, and it is being used from + # -proposed, so holding it back on a dozen unrelated test failures + # serves no purpose. Just check some key packages which actually use + # gcc during the test, and libreoffice as an example for a libgcc user. + if src.startswith('gcc-'): + if re.match('gcc-\d$', src): + for test in ['binutils', 'fglrx-installer', 'libreoffice', 'linux']: + try: + tests.append((test, self.britney.sources['testing'][test][VERSION])) + except KeyError: + # no package in that series? *shrug*, then not (mostly for testing) + pass + return tests + else: + # for other compilers such as gcc-snapshot etc. we don't need + # to trigger anything + return [] + + # for linux themselves we don't want to trigger tests -- these should + # all come from linux-meta*. A new kernel ABI without a corresponding + # -meta won't be installed and thus we can't sensibly run tests against + # it. + if src.startswith('linux') and src.replace('linux', 'linux-meta') in self.britney.sources['testing']: + return [] + + # we want to test the package itself, if it still has a test in unstable + srcinfo = self.britney.sources['unstable'][src] + if 'autopkgtest' in srcinfo.testsuite or self.has_autodep8(srcinfo, binaries_info): + reported_pkgs.add(src) + tests.append((src, ver)) + + extra_bins = [] + # Hack: For new kernels trigger all DKMS packages by pretending that + # linux-meta* builds a "dkms" binary as well. With that we ensure that we + # don't regress DKMS drivers with new kernel versions. + if src.startswith('linux-meta'): + # does this have any image on this arch? + for pkg_id in srcinfo.binaries: + if pkg_id.architecture == arch and '-image' in pkg_id.package_name: + try: + extra_bins.append(binaries_info['dkms'].pkg_id) + except KeyError: + pass + + # plus all direct reverse dependencies and test triggers of its + # binaries which have an autopkgtest + for binary in srcinfo.binaries + extra_bins: + rdeps = self.britney._inst_tester.reverse_dependencies_of(binary) + for rdep in rdeps: + try: + rdep_src = binaries_info[rdep.package_name].source + except KeyError: + self.log('%s on %s has no source (NBS?)' % (rdep.package_name, arch)) + continue + + rdep_src_info = sources_info[rdep_src] + if 'autopkgtest' in rdep_src_info.testsuite or self.has_autodep8(rdep_src_info, binaries_info): + if rdep_src not in reported_pkgs: + tests.append((rdep_src, rdep_src_info[VERSION])) + reported_pkgs.add(rdep_src) + + for tdep_src in self.britney.testsuite_triggers.get(binary.package_name, set()): + if tdep_src not in reported_pkgs: + try: + tdep_src_info = sources_info[tdep_src] + except KeyError: + continue + if 'autopkgtest' in tdep_src_info.testsuite or self.has_autodep8(tdep_src_info, binaries_info): + tests.append((tdep_src, tdep_src_info[VERSION])) + reported_pkgs.add(tdep_src) + + # Hardcode linux-meta → linux, lxc, glibc, systemd triggers until we get a more flexible + # implementation: https://bugs.debian.org/779559 + if src.startswith('linux-meta'): + for pkg in ['lxc', 'lxd', 'glibc', src.replace('linux-meta', 'linux'), 'systemd']: + if pkg not in reported_pkgs: + # does this have any image on this arch? + for pkg_id in srcinfo.binaries: + if pkg_id.architecture == arch and '-image' in pkg_id.package_name: + try: + tests.append((pkg, self.britney.sources['unstable'][pkg][VERSION])) + except KeyError: + try: + tests.append((pkg, self.britney.sources['testing'][pkg][VERSION])) + except KeyError: + # package not in that series? *shrug*, then not + pass + break + + tests.sort(key=lambda s_v: s_v[0]) + return tests + + def read_pending_tests(self): + '''Read pending test requests from previous britney runs + + Initialize self.pending_tests with that data. + ''' + assert self.pending_tests is None, 'already initialized' + if not os.path.exists(self.pending_tests_file): + self.log('No %s, starting with no pending tests' % + self.pending_tests_file) + self.pending_tests = {} + return + with open(self.pending_tests_file) as f: + self.pending_tests = json.load(f) + self.log('Read pending requested tests from %s: %s' % + (self.pending_tests_file, self.pending_tests)) + + def latest_run_for_package(self, src, arch): + '''Return latest run ID for src on arch''' + + # this requires iterating over all triggers and thus is expensive; + # cache the results + try: + return self.latest_run_for_package._cache[src][arch] + except KeyError: + pass + + latest_run_id = '' + for srcmap in self.test_results.values(): + try: + run_id = srcmap[src][arch][2] + except KeyError: + continue + if run_id > latest_run_id: + latest_run_id = run_id + self.latest_run_for_package._cache.setdefault(src, {})[arch] = latest_run_id + return latest_run_id + + latest_run_for_package._cache = {} + + def fetch_swift_results(self, swift_url, src, arch): + '''Download new results for source package/arch from swift''' + + # Download results for one particular src/arch at most once in every + # run, as this is expensive + done_entry = src + '/' + arch + if done_entry in self.fetch_swift_results._done: + return + self.fetch_swift_results._done.add(done_entry) + + # prepare query: get all runs with a timestamp later than the latest + # run_id for this package/arch; '@' is at the end of each run id, to + # mark the end of a test run directory path + # example: wily/amd64/libp/libpng/20150630_054517@/result.tar + query = {'delimiter': '@', + 'prefix': '%s/%s/%s/%s/' % (self.options.series, arch, srchash(src), src)} + + # determine latest run_id from results + if not self.options.adt_shared_results_cache: + latest_run_id = self.latest_run_for_package(src, arch) + if latest_run_id: + query['marker'] = query['prefix'] + latest_run_id + + # request new results from swift + url = os.path.join(swift_url, self.swift_container) + url += '?' + urllib.parse.urlencode(query) + try: + f = urlopen(url, timeout=30) + if f.getcode() == 200: + result_paths = f.read().decode().strip().splitlines() + elif f.getcode() == 204: # No content + result_paths = [] + else: + # we should not ever end up here as we expect a HTTPError in + # other cases; e. g. 3XX is something that tells us to adjust + # our URLS, so fail hard on those + raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' % + (url, f.getcode())) + f.close() + except IOError as e: + # 401 "Unauthorized" is swift's way of saying "container does not exist" + if hasattr(e, 'code') and e.code == 401: + self.log('fetch_swift_results: %s does not exist yet or is inaccessible' % url) + return + # Other status codes are usually a transient + # network/infrastructure failure. Ignoring this can lead to + # re-requesting tests which we already have results for, so + # fail hard on this and let the next run retry. + self.log('FATAL: Failure to fetch swift results from %s: %s' % (url, str(e)), 'E') + sys.exit(1) + + for p in result_paths: + self.fetch_one_result( + os.path.join(swift_url, self.swift_container, p, 'result.tar'), src, arch) + + fetch_swift_results._done = set() + + def fetch_one_result(self, url, src, arch): + '''Download one result URL for source/arch + + Remove matching pending_tests entries. + ''' + try: + f = urlopen(url, timeout=30) + if f.getcode() == 200: + tar_bytes = io.BytesIO(f.read()) + f.close() + else: + raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' % + (url, f.getcode())) + except IOError as e: + self.log('Failure to fetch %s: %s' % (url, str(e)), 'E') + # we tolerate "not found" (something went wrong on uploading the + # result), but other things indicate infrastructure problems + if hasattr(e, 'code') and e.code == 404: + return + sys.exit(1) + + try: + with tarfile.open(None, 'r', tar_bytes) as tar: + exitcode = int(tar.extractfile('exitcode').read().strip()) + srcver = tar.extractfile('testpkg-version').read().decode().strip() + (ressrc, ver) = srcver.split() + testinfo = json.loads(tar.extractfile('testinfo.json').read().decode()) + except (KeyError, ValueError, tarfile.TarError) as e: + self.log('%s is damaged, ignoring: %s' % (url, str(e)), 'E') + # ignore this; this will leave an orphaned request in pending.json + # and thus require manual retries after fixing the tmpfail, but we + # can't just blindly attribute it to some pending test. + return + + if src != ressrc: + self.log('%s is a result for package %s, but expected package %s' % + (url, ressrc, src), 'E') + return + + # parse recorded triggers in test result + for e in testinfo.get('custom_environment', []): + if e.startswith('ADT_TEST_TRIGGERS='): + result_triggers = [i for i in e.split('=', 1)[1].split() if '/' in i] + break + else: + self.log('%s result has no ADT_TEST_TRIGGERS, ignoring', 'E') + return + + stamp = os.path.basename(os.path.dirname(url)) + # allow some skipped tests, but nothing else + passed = exitcode in [0, 2] + + self.log('Fetched test result for %s/%s/%s %s (triggers: %s): %s' % ( + src, ver, arch, stamp, result_triggers, passed and 'pass' or 'fail')) + + # remove matching test requests + for trigger in result_triggers: + try: + arch_list = self.pending_tests[trigger][src] + arch_list.remove(arch) + if not arch_list: + del self.pending_tests[trigger][src] + if not self.pending_tests[trigger]: + del self.pending_tests[trigger] + self.log('-> matches pending request %s/%s for trigger %s' % (src, arch, trigger)) + except (KeyError, ValueError): + self.log('-> does not match any pending request for %s/%s' % (src, arch)) + + # add this result + for trigger in result_triggers: + # If a test runs because of its own package (newer version), ensure + # that we got a new enough version; FIXME: this should be done more + # generically by matching against testpkg-versions + (trigsrc, trigver) = trigger.split('/', 1) + if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0: + self.log('test trigger %s, but run for older version %s, ignoring' % (trigger, ver), 'E') + continue + + result = self.test_results.setdefault(trigger, {}).setdefault( + src, {}).setdefault(arch, [False, None, '']) + + # don't clobber existing passed results with failures from re-runs + if passed or not result[0]: + result[0] = passed + result[1] = ver + result[2] = stamp + + def send_test_request(self, src, arch, trigger, huge=False): + '''Send out AMQP request for testing src/arch for trigger + + If huge is true, then the request will be put into the -huge instead of + normal queue. + ''' + if self.options.dry_run: + return + + params = {'triggers': [trigger]} + if self.options.adt_ppas: + params['ppas'] = self.options.adt_ppas + qname = 'debci-ppa-%s-%s' % (self.options.series, arch) + elif huge: + qname = 'debci-huge-%s-%s' % (self.options.series, arch) + else: + qname = 'debci-%s-%s' % (self.options.series, arch) + params = json.dumps(params) + + if self.amqp_channel: + self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params), routing_key=qname) + else: + assert self.amqp_file + with open(self.amqp_file, 'a') as f: + f.write('%s:%s %s\n' % (qname, src, params)) + + def pkg_test_request(self, src, arch, trigger, huge=False): + '''Request one package test for one particular trigger + + trigger is "pkgname/version" of the package that triggers the testing + of src. If huge is true, then the request will be put into the -huge + instead of normal queue. + + This will only be done if that test wasn't already requested in a + previous run (i. e. not already in self.pending_tests) or there already + is a result for it. This ensures to download current results for this + package before requesting any test. + ''' + # Don't re-request if we already have a result + try: + passed = self.test_results[trigger][src][arch][0] + if passed: + self.log('%s/%s triggered by %s already passed' % (src, arch, trigger)) + return + self.log('Checking for new results for failed %s/%s for trigger %s' % + (src, arch, trigger)) + raise KeyError # fall through + except KeyError: + self.fetch_swift_results(self.options.adt_swift_url, src, arch) + # do we have one now? + try: + self.test_results[trigger][src][arch] + return + except KeyError: + pass + + # Don't re-request if it's already pending + arch_list = self.pending_tests.setdefault(trigger, {}).setdefault(src, []) + if arch in arch_list: + self.log('Test %s/%s for %s is already pending, not queueing' % + (src, arch, trigger)) + else: + self.log('Requesting %s autopkgtest on %s to verify %s' % + (src, arch, trigger)) + arch_list.append(arch) + arch_list.sort() + self.send_test_request(src, arch, trigger, huge=huge) + + def check_ever_passed(self, src, arch): + '''Check if tests for src ever passed on arch''' + + # FIXME: add caching + for srcmap in self.test_results.values(): + try: + if srcmap[src][arch][0]: + return True + except KeyError: + pass + return False + + def pkg_test_result(self, src, ver, arch, trigger): + '''Get current test status of a particular package + + Return (status, real_version, log_url) tuple; status is a key in + EXCUSES_LABELS. log_url is None if the test is still running. + ''' + # determine current test result status + ever_passed = self.check_ever_passed(src, arch) + url = None + try: + r = self.test_results[trigger][src][arch] + ver = r[1] + run_id = r[2] + if r[0]: + result = 'PASS' + else: + # Special-case triggers from linux-meta*: we cannot compare + # results against different kernels, as e. g. a DKMS module + # might work against the default kernel but fail against a + # different flavor; so for those, ignore the "ever + # passed" check; FIXME: check against trigsrc only + if trigger.startswith('linux-meta') or trigger.startswith('linux/'): + ever_passed = False + + if ever_passed: + if self.has_force_badtest(src, ver, arch): + result = 'IGNORE-FAIL' + else: + result = 'REGRESSION' + else: + result = 'ALWAYSFAIL' + + url = os.path.join(self.options.adt_swift_url, + self.swift_container, + self.options.series, + arch, + srchash(src), + src, + run_id, + 'log.gz') + except KeyError: + # no result for src/arch; still running? + if arch in self.pending_tests.get(trigger, {}).get(src, []): + if ever_passed and not self.has_force_badtest(src, ver, arch): + result = 'RUNNING' + else: + result = 'RUNNING-ALWAYSFAIL' + url = 'http://autopkgtest.ubuntu.com/running' + else: + raise RuntimeError('Result for %s/%s/%s (triggered by %s) is neither known nor pending!' % + (src, ver, arch, trigger)) + + return (result, ver, url) + + def has_force_badtest(self, src, ver, arch): + '''Check if src/ver/arch has a force-badtest hint''' + + hints = self.britney.hints.search('force-badtest', package=src) + if hints: + self.log('Checking hints for %s/%s/%s: %s' % (src, ver, arch, [str(h) for h in hints])) + for hint in hints: + if [mi for mi in hint.packages if mi.architecture in ['source', arch] and + (mi.version == 'all' or apt_pkg.version_compare(ver, mi.version) <= 0)]: + return True + + return False + diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index ada24ba..c71f539 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -86,3 +86,15 @@ HINTS_UBUNTU-TOUCH/OGRA = block unblock SMOOTH_UPDATES = badgers REMOVE_OBSOLETE = no + +ADT_ENABLE = yes +ADT_ARCHES = amd64 i386 armhf ppc64el +ADT_AMQP = amqp://test_request:password@162.213.33.228 +# Swift base URL with the results (must be publicly readable and browsable) +ADT_SWIFT_URL = https://objectstorage.prodstack4-5.canonical.com/v1/AUTH_77e2ada1e7a84929a74ba3b87153c0ac +# space separate list of PPAs to add for test requests and for polling results; +# the *last* one determines the swift container name +ADT_PPAS = +# set this to the path of a (r/o) results.cache for running many parallel +# britney instances for PPAs without updating the cache +ADT_SHARED_RESULTS_CACHE = diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e08edc3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,188 @@ +# (C) 2015 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import shutil +import subprocess +import tempfile +import unittest + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'] + + +class TestData: + + def __init__(self): + '''Construct local test package indexes. + + The archive is initially empty. You can create new packages with + create_deb(). self.path contains the path of the archive, and + self.apt_source provides an apt source "deb" line. + + It is kept in a temporary directory which gets removed when the Archive + object gets deleted. + ''' + self.path = tempfile.mkdtemp(prefix='testarchive.') + self.apt_source = 'deb file://%s /' % self.path + self.series = 'series' + self.dirs = {False: os.path.join(self.path, 'data', self.series), + True: os.path.join( + self.path, 'data', '%s-proposed' % self.series)} + os.makedirs(self.dirs[False]) + os.mkdir(self.dirs[True]) + self.added_sources = {False: set(), True: set()} + self.added_binaries = {False: set(), True: set()} + + # pre-create all files for all architectures + for arch in architectures: + for dir in self.dirs.values(): + with open(os.path.join(dir, 'Packages_' + arch), 'w'): + pass + for dir in self.dirs.values(): + for fname in ['Dates', 'Blocks']: + with open(os.path.join(dir, fname), 'w'): + pass + for dname in ['Hints']: + os.mkdir(os.path.join(dir, dname)) + + os.mkdir(os.path.join(self.path, 'output')) + + # create temporary home dir for proposed-migration autopktest status + self.home = os.path.join(self.path, 'home') + os.environ['HOME'] = self.home + os.makedirs(os.path.join(self.home, 'proposed-migration', + 'autopkgtest', 'work')) + + def __del__(self): + shutil.rmtree(self.path) + + def add(self, name, unstable, fields={}, add_src=True, testsuite=None, srcfields=None): + '''Add a binary package to the index file. + + You need to specify at least the package name and in which list to put + it (unstable==True for unstable/proposed, or False for + testing/release). fields specifies all additional entries, e. g. + {'Depends': 'foo, bar', 'Conflicts: baz'}. There are defaults for most + fields. + + Unless add_src is set to False, this will also automatically create a + source record, based on fields['Source'] and name. In that case, the + "Testsuite:" field is set to the testsuite argument. + ''' + assert (name not in self.added_binaries[unstable]) + self.added_binaries[unstable].add(name) + + fields.setdefault('Architecture', 'all') + fields.setdefault('Version', '1') + fields.setdefault('Priority', 'optional') + fields.setdefault('Section', 'devel') + fields.setdefault('Description', 'test pkg') + if fields['Architecture'] == 'all': + for a in architectures: + self._append(name, unstable, 'Packages_' + a, fields) + else: + self._append(name, unstable, 'Packages_' + fields['Architecture'], + fields) + + if add_src: + src = fields.get('Source', name) + if src not in self.added_sources[unstable]: + if srcfields is None: + srcfields = {} + srcfields['Version'] = fields['Version'] + srcfields['Section'] = fields['Section'] + if testsuite: + srcfields['Testsuite'] = testsuite + self.add_src(src, unstable, srcfields) + + def add_src(self, name, unstable, fields={}): + '''Add a source package to the index file. + + You need to specify at least the package name and in which list to put + it (unstable==True for unstable/proposed, or False for + testing/release). fields specifies all additional entries, which can be + Version (default: 1), Section (default: devel), Testsuite (default: + none), and Extra-Source-Only. + ''' + assert (name not in self.added_sources[unstable]) + self.added_sources[unstable].add(name) + + fields.setdefault('Version', '1') + fields.setdefault('Section', 'devel') + self._append(name, unstable, 'Sources', fields) + + def _append(self, name, unstable, file_name, fields): + with open(os.path.join(self.dirs[unstable], file_name), 'a') as f: + f.write('''Package: %s +Maintainer: Joe +''' % name) + + for k, v in fields.items(): + f.write('%s: %s\n' % (k, v)) + f.write('\n') + + def remove_all(self, unstable): + '''Remove all added packages''' + + self.added_binaries[unstable] = set() + self.added_sources[unstable] = set() + for a in architectures: + open(os.path.join(self.dirs[unstable], 'Packages_' + a), 'w').close() + open(os.path.join(self.dirs[unstable], 'Sources'), 'w').close() + + +class TestBase(unittest.TestCase): + + def setUp(self): + super(TestBase, self).setUp() + self.maxDiff = None + self.data = TestData() + self.britney = os.path.join(PROJECT_DIR, 'britney.py') + # create temporary config so that tests can hack it + self.britney_conf = os.path.join(self.data.path, 'britney.conf') + shutil.copy(os.path.join(PROJECT_DIR, 'britney.conf'), self.britney_conf) + assert os.path.exists(self.britney) + + def tearDown(self): + del self.data + + def run_britney(self, args=[]): + '''Run britney. + + Assert that it succeeds and does not produce anything on stderr. + Return (excuses.yaml, excuses.html, britney_out). + ''' + britney = subprocess.Popen([self.britney, '-v', '-c', self.britney_conf, + '--distribution=ubuntu', + '--series=%s' % self.data.series], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.data.path, + universal_newlines=True) + (out, err) = britney.communicate() + self.assertEqual(britney.returncode, 0, out + err) + self.assertEqual(err, '') + + with open(os.path.join(self.data.path, 'output', self.data.series, + 'excuses.yaml')) as f: + yaml = f.read() + with open(os.path.join(self.data.path, 'output', self.data.series, + 'excuses.html')) as f: + html = f.read() + + return (yaml, html, out) + + def create_hint(self, username, content): + '''Create a hint file for the given username and content''' + + hints_path = os.path.join( + self.data.path, 'data', self.data.series + '-proposed', 'Hints', username) + with open(hints_path, 'a') as fd: + fd.write(content) + fd.write('\n') diff --git a/tests/mock_swift.py b/tests/mock_swift.py new file mode 100644 index 0000000..ecaa2fc --- /dev/null +++ b/tests/mock_swift.py @@ -0,0 +1,170 @@ +# Mock a Swift server with autopkgtest results +# Author: Martin Pitt + +import os +import tarfile +import io +import sys +import socket +import time +import tempfile +import json + +try: + from http.server import HTTPServer, BaseHTTPRequestHandler + from urllib.parse import urlparse, parse_qs +except ImportError: + # Python 2 + from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler + from urlparse import urlparse, parse_qs + + +class SwiftHTTPRequestHandler(BaseHTTPRequestHandler): + '''Mock swift container with autopkgtest results + + This accepts retrieving a particular result.tar (e. g. + /container/path/result.tar) or listing the container contents + (/container/?prefix=foo&delimiter=@&marker=foo/bar). + ''' + # map container -> result.tar path -> (exitcode, testpkg-version[, testinfo]) + results = {} + + def do_GET(self): + p = urlparse(self.path) + path_comp = p.path.split('/') + container = path_comp[1] + path = '/'.join(path_comp[2:]) + if path: + self.serve_file(container, path) + else: + self.list_container(container, parse_qs(p.query)) + + def serve_file(self, container, path): + if os.path.basename(path) != 'result.tar': + self.send_error(404, 'File not found (only result.tar supported)') + return + try: + fields = self.results[container][os.path.dirname(path)] + try: + (exitcode, pkgver, testinfo) = fields + except ValueError: + (exitcode, pkgver) = fields + testinfo = None + except KeyError: + self.send_error(404, 'File not found') + return + + self.send_response(200) + self.send_header('Content-type', 'application/octet-stream') + self.end_headers() + + tar = io.BytesIO() + with tarfile.open('result.tar', 'w', tar) as results: + # add exitcode + contents = ('%i' % exitcode).encode() + ti = tarfile.TarInfo('exitcode') + ti.size = len(contents) + results.addfile(ti, io.BytesIO(contents)) + # add testpkg-version + if pkgver is not None: + contents = pkgver.encode() + ti = tarfile.TarInfo('testpkg-version') + ti.size = len(contents) + results.addfile(ti, io.BytesIO(contents)) + # add testinfo.json + if testinfo: + contents = json.dumps(testinfo).encode() + ti = tarfile.TarInfo('testinfo.json') + ti.size = len(contents) + results.addfile(ti, io.BytesIO(contents)) + + self.wfile.write(tar.getvalue()) + + def list_container(self, container, query): + try: + objs = set(['%s/result.tar' % r for r in self.results[container]]) + except KeyError: + self.send_error(401, 'Container does not exist') + return + if 'prefix' in query: + p = query['prefix'][-1] + objs = set([o for o in objs if o.startswith(p)]) + if 'delimiter' in query: + d = query['delimiter'][-1] + # if find() returns a value, we want to include the delimiter, thus + # bump its result; for "not found" return None + find_adapter = lambda i: (i >= 0) and (i + 1) or None + objs = set([o[:find_adapter(o.find(d))] for o in objs]) + if 'marker' in query: + m = query['marker'][-1] + objs = set([o for o in objs if o > m]) + + self.send_response(objs and 200 or 204) # 204: "No Content" + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write(('\n'.join(sorted(objs)) + '\n').encode('UTF-8')) + + +class AutoPkgTestSwiftServer: + def __init__(self, port=8080): + self.port = port + self.server_pid = None + self.log = None + + def __del__(self): + if self.server_pid: + self.stop() + + @classmethod + def set_results(klass, results): + '''Set served results. + + results is a map: container -> result.tar path -> + (exitcode, testpkg-version, testinfo) + ''' + SwiftHTTPRequestHandler.results = results + + def start(self): + assert self.server_pid is None, 'already started' + if self.log: + self.log.close() + self.log = tempfile.TemporaryFile() + p = os.fork() + if p: + # parent: wait until server starts + self.server_pid = p + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + while True: + if s.connect_ex(('127.0.0.1', self.port)) == 0: + break + time.sleep(0.1) + s.close() + return + + # child; quiesce logging on stderr + os.dup2(self.log.fileno(), sys.stderr.fileno()) + srv = HTTPServer(('', self.port), SwiftHTTPRequestHandler) + srv.serve_forever() + sys.exit(0) + + def stop(self): + assert self.server_pid, 'not running' + os.kill(self.server_pid, 15) + os.waitpid(self.server_pid, 0) + self.server_pid = None + self.log.close() + +if __name__ == '__main__': + srv = AutoPkgTestSwiftServer() + srv.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/g/green/20150101_100000@': (0, 'green 1', {'custom_environment': ['ADT_TEST_TRIGGERS=green']}), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2'), + 'series/i386/l/lightgreen/20150101_100102@': (0, 'lightgreen 3'), + }}) + srv.start() + print('Running on http://localhost:8080/autopkgtest-series') + print('Press Enter to quit.') + sys.stdin.readline() + srv.stop() diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py new file mode 100755 index 0000000..1257f17 --- /dev/null +++ b/tests/test_autopkgtest.py @@ -0,0 +1,2036 @@ +#!/usr/bin/python3 +# (C) 2014 - 2015 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import sys +import fileinput +import unittest +import json +import pprint +import urllib.parse + +import apt_pkg +import yaml + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PROJECT_DIR) + +from tests import TestBase, mock_swift + +apt_pkg.init() + + +# shortcut for test triggers +def tr(s): + return {'custom_environment': ['ADT_TEST_TRIGGERS=%s' % s]} + +ON_ALL_ARCHES = {'on-architectures': ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []} + + +class T(TestBase): + '''AMQP/cloud interface''' + + ################################################################ + # Common test code + ################################################################ + + def setUp(self): + super().setUp() + self.fake_amqp = os.path.join(self.data.path, 'amqp') + + # Set fake AMQP and Swift server + for line in fileinput.input(self.britney_conf, inplace=True): + if 'ADT_AMQP' in line: + print('ADT_AMQP = file://%s' % self.fake_amqp) + elif 'ADT_SWIFT_URL' in line: + print('ADT_SWIFT_URL = http://localhost:18085') + elif 'ADT_ARCHES' in line: + print('ADT_ARCHES = amd64 i386') + else: + sys.stdout.write(line) + + # add a bunch of packages to testing to avoid repetition + self.data.add('libc6', False) + self.data.add('libgreen1', False, {'Source': 'green', + 'Depends': 'libc6 (>= 0.9)'}, + testsuite='autopkgtest') + self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Conflicts': 'blue'}, + testsuite='autopkgtest') + self.data.add('lightgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest') + # autodep8 or similar test + self.data.add('darkgreen', False, {'Depends': 'libgreen1'}, + testsuite='autopkgtest-pkg-foo') + self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', + 'Conflicts': 'green'}, + testsuite='specialtest') + + # create mock Swift server (but don't start it yet, as tests first need + # to poke in results) + self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085) + self.swift.set_results({}) + + def tearDown(self): + del self.swift + + def do_test(self, unstable_add, expect_status, expect_excuses={}): + '''Run britney with some unstable packages and verify excuses. + + unstable_add is a list of (binpkgname, field_dict, testsuite_value) + passed to TestData.add for "unstable". + + expect_status is a dict sourcename → (is_candidate, testsrc → arch → status) + that is checked against the excuses YAML. + + expect_excuses is a dict sourcename → [(key, value), ...] + matches that are checked against the excuses YAML. + + Return (output, excuses_dict, excuses_html). + ''' + for (pkg, fields, testsuite) in unstable_add: + self.data.add(pkg, True, fields, True, testsuite) + + self.swift.start() + (excuses_yaml, excuses_html, out) = self.run_britney() + self.swift.stop() + + # convert excuses to source indexed dict + excuses_dict = {} + for s in yaml.load(excuses_yaml)['sources']: + excuses_dict[s['source']] = s + + if 'SHOW_EXCUSES' in os.environ: + print('------- excuses -----') + pprint.pprint(excuses_dict, width=200) + if 'SHOW_HTML' in os.environ: + print('------- excuses.html -----\n%s\n' % excuses_html) + if 'SHOW_OUTPUT' in os.environ: + print('------- output -----\n%s\n' % out) + + for src, (is_candidate, testmap) in expect_status.items(): + self.assertEqual(excuses_dict[src]['is-candidate'], is_candidate, + src + ': ' + pprint.pformat(excuses_dict[src])) + for testsrc, archmap in testmap.items(): + for arch, status in archmap.items(): + self.assertEqual(excuses_dict[src]['policy_info']['autopkgtest'][testsrc][arch][0], + status, + excuses_dict[src]['policy_info']['autopkgtest'][testsrc]) + + for src, matches in expect_excuses.items(): + for k, v in matches: + if isinstance(excuses_dict[src][k], list): + self.assertIn(v, excuses_dict[src][k]) + else: + self.assertEqual(excuses_dict[src][k], v) + + self.amqp_requests = set() + try: + with open(self.fake_amqp) as f: + for line in f: + self.amqp_requests.add(line.strip()) + os.unlink(self.fake_amqp) + except IOError: + pass + + try: + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/pending.json')) as f: + self.pending_requests = json.load(f) + except IOError: + self.pending_requests = None + + self.assertNotIn('FIXME', out) + + return (out, excuses_dict, excuses_html) + + ################################################################ + # Tests for generic packages + ################################################################ + + def test_no_request_for_uninstallable(self): + '''Does not request a test for an uninstallable package''' + + exc = self.do_test( + # uninstallable unstable version + [('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')], + {'lightgreen': (False, {})}, + {'lightgreen': [('old-version', '1'), ('new-version', '1.1~beta'), + ('reason', 'depends'), + ('excuses', 'lightgreen/amd64 unsatisfiable Depends: libgreen1 (>= 2)') + ] + })[1] + # autopkgtest should not be triggered for uninstallable pkg + self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], {}) + + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + with open(os.path.join(self.data.path, 'output', 'series', 'output.txt')) as f: + upgrade_out = f.read() + self.assertNotIn('accepted:', upgrade_out) + self.assertIn('SUCCESS (0/0)', upgrade_out) + + def test_no_wait_for_always_failed_test(self): + '''We do not need to wait for results for tests which have always failed''' + + # The package has failed before, and with a trigger too on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (4, 'green 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (4, 'green 1', tr('failedbefore/1')), + }}) + + exc = self.do_test( + [('darkgreen', {'Version': '2'}, 'autopkgtest')], + {'darkgreen': (True, {'darkgreen': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}})}, + )[1] + + # the test should still be triggered though + self.assertEqual(exc['darkgreen']['policy_info']['autopkgtest'], + {'darkgreen': { + 'amd64': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + 'http://autopkgtest.ubuntu.com/packages/d/darkgreen/series/amd64', + None, + None], + 'i386': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + 'http://autopkgtest.ubuntu.com/packages/d/darkgreen/series/i386', + None, + None]}}) + + self.assertEqual(self.pending_requests, + {'darkgreen/2': {'darkgreen': ['amd64', 'i386']}}) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-amd64:darkgreen {"triggers": ["darkgreen/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["darkgreen/2"]}'])) + + with open(os.path.join(self.data.path, 'output', 'series', 'output.txt')) as f: + upgrade_out = f.read() + self.assertIn('accepted: darkgreen', upgrade_out) + self.assertIn('SUCCESS (1/0)', upgrade_out) + + def test_multi_rdepends_with_tests_all_running(self): + '''Multiple reverse dependencies with tests (all running)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2'), + ('reason', 'autopkgtest')]}) + + # we expect the package's and its reverse dependencies' tests to get + # triggered + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green {"triggers": ["green/2"]}', + 'debci-series-amd64:green {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}'])) + + # ... and that they get recorded as pending + expected_pending = {'green/2': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386']}} + self.assertEqual(self.pending_requests, expected_pending) + + # if we run britney again this should *not* trigger any new tests + self.do_test([], {'green': (False, {})}) + self.assertEqual(self.amqp_requests, set()) + # but the set of pending tests doesn't change + self.assertEqual(self.pending_requests, expected_pending) + + def test_multi_rdepends_with_tests_all_pass(self): + '''Multiple reverse dependencies with tests (all pass)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + # first run requests tests and marks them as pending + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + # a reverse dep that does not exist in testing should not be triggered + ('brittle', {'Depends': 'libgreen1'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]})[1] + self.assertNotIn('brittle', exc['green']['policy_info']['autopkgtest']) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/2')), + # version in testing fails + 'series/i386/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + # version in unstable succeeds + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + # new "brittle" succeeds + 'series/i386/b/brittle/20150101_100200@': (0, 'brittle 1', tr('brittle/1')), + 'series/amd64/b/brittle/20150101_100201@': (0, 'brittle 1', tr('brittle/1')), + }}) + + out = self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'brittle': (True, {'brittle/1': {'amd64': 'PASS', 'i386': 'PASS'}}) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + # all tests ran, there should be no more pending ones + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + # caches the results and triggers + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: + res = json.load(f) + self.assertEqual(res['green/1']['green']['amd64'], + [False, '1', '20150101_020000@']) + self.assertEqual(set(res['green/2']), {'darkgreen', 'green', 'lightgreen'}) + self.assertEqual(res['green/2']['lightgreen']['i386'], + [True, '1', '20150101_100100@']) + + # third run should not trigger any new tests, should all be in the + # cache + self.swift.set_results({}) + out = self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + })[0] + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_mixed(self): + '''Multiple reverse dependencies with tests (mixed results)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]}) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + # unrelated results (wrong trigger), ignore this! + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('blue/1')), + }}) + + out = self.do_test( + [], + {'green': (False, {'green/2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}, + 'darkgreen/1': {'amd64': 'RUNNING', 'i386': 'PASS'}, + }) + })[0] + + self.assertIn('Update Excuses generation completed', out) + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out) + + # there should be some pending ones + self.assertEqual(self.pending_requests, + {'green/2': {'darkgreen': ['amd64'], 'lightgreen': ['i386']}}) + + def test_results_without_triggers(self): + '''Old results without recorded triggers''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100100@': (0, 'green 1', tr('passedbefore/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + # none of the above results should be accepted + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }) + + # there should be some pending ones + self.assertEqual(self.pending_requests, + {'green/2': {'lightgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'darkgreen': ['amd64', 'i386']}}) + + def test_multi_rdepends_with_tests_regression(self): + '''Multiple reverse dependencies with tests (regression)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + }}) + + out, exc, _ = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + ) + + # should have links to log and history, but no artifacts (as this is + # not a PPA) + self.assertEqual(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][:4], + ['REGRESSION', + 'http://localhost:18085/autopkgtest-series/series/amd64/l/lightgreen/20150101_100101@/log.gz', + 'http://autopkgtest.ubuntu.com/packages/l/lightgreen/series/amd64', + None]) + + # should have retry link for the regressions (not a stable URL, test + # seaprately) + link = urllib.parse.urlparse(exc['green']['policy_info']['autopkgtest']['lightgreen/1']['amd64'][4]) + self.assertEqual(link.netloc, 'autopkgtest.ubuntu.com') + self.assertEqual(link.path, '/request.cgi') + self.assertEqual(urllib.parse.parse_qs(link.query), + {'release': ['series'], 'arch': ['amd64'], + 'package': ['lightgreen'], 'trigger': ['green/2']}) + + # we already had all results before the run, so this should not trigger + # any new requests + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_regression_last_pass(self): + '''Multiple reverse dependencies with tests (regression), last one passes + + This ensures that we don't just evaluate the test result of the last + test, but all of them. + ''' + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + self.assertEqual(self.pending_requests, {}) + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_always_failed(self): + '''Multiple reverse dependencies with tests (always failed)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (4, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2', tr('green/2')), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'ALWAYSFAIL', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'ALWAYSFAIL', 'i386': 'ALWAYSFAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + self.assertEqual(self.pending_requests, {}) + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_arch_specific(self): + '''Multiple reverse dependencies with arch specific tests''' + + # green has passed before on amd64, doesn't exist on i386 + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/g/green64/20150101_100000@': (0, 'green64 0.1', tr('passedbefore/1')), + }}) + + self.data.add('green64', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Architecture': 'amd64'}, + testsuite='autopkgtest') + + # first run requests tests and marks them as pending + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'green64': {'amd64': 'RUNNING'}, + }) + }) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green {"triggers": ["green/2"]}', + 'debci-series-amd64:green {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:green64 {"triggers": ["green/2"]}'])) + + self.assertEqual(self.pending_requests, + {'green/2': {'lightgreen': ['amd64', 'i386'], + 'darkgreen': ['amd64', 'i386'], + 'green64': ['amd64'], + 'green': ['amd64', 'i386']}}) + + # second run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/2')), + # version in testing fails + 'series/i386/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (4, 'green 1', tr('green/1')), + # version in unstable succeeds + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + # only amd64 result for green64 + 'series/amd64/g/green64/20150101_100200@': (0, 'green64 1', tr('green/2')), + }}) + + out = self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'green64/1': {'amd64': 'PASS'}, + }) + }, + {'green': [('old-version', '1'), ('new-version', '2')]} + )[0] + + # all tests ran, there should be no more pending ones + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_unbuilt(self): + '''Unbuilt package should not trigger tests or get considered''' + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + exc = self.do_test( + # uninstallable unstable version + [], + {'green': (False, {})}, + {'green': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES), + ] + })[1] + # autopkgtest should not be triggered for unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest'], {}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_partial_unbuilt(self): + '''Unbuilt package on some arches should not trigger tests''' + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.data.add('libgreen1', True, {'Version': '2', 'Source': 'green', 'Architecture': 'i386'}, add_src=False) + exc = self.do_test( + [], + {'green': (False, {})}, + {'green': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', {'on-architectures': ['amd64', 'arm64', 'armhf', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []}) + ] + })[1] + # autopkgtest should not be triggered for unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest'], {}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_partial_unbuilt_block(self): + '''Unbuilt blocked package on some arches should not trigger tests''' + + self.create_hint('freeze', 'block-all source') + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.data.add('libgreen1', True, {'Version': '2', 'Source': 'green', 'Architecture': 'i386'}, add_src=False) + exc = self.do_test( + [], + {'green': (False, {})}, + {'green': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', {'on-architectures': ['amd64', 'arm64', 'armhf', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []}) + ] + })[1] + # autopkgtest should not be triggered for unbuilt pkg + self.assertEqual(exc['green']['policy_info']['autopkgtest'], {}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_rdepends_unbuilt(self): + '''Unbuilt reverse dependency''' + + # old lightgreen fails, thus new green should be held back + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/i386/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 1.1', tr('green/1.1')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 1.1', tr('green/1.1')), + }}) + + # add unbuilt lightgreen; should run tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)], + } + ) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run should not trigger any new requests + self.do_test([], {'green': (False, {}), 'lightgreen': (False, {})}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # now lightgreen 2 gets built, should trigger a new test run + self.data.remove_all(True) + self.do_test( + [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '2'}, 'autopkgtest')], + {}) + self.assertEqual(self.amqp_requests, + set(['debci-series-amd64:lightgreen {"triggers": ["lightgreen/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["lightgreen/2"]}'])) + + # next run collects the results + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100200@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_102000@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + self.do_test( + [], + # green hasn't changed, the above re-run was for trigger lightgreen/2 + {'green': (False, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (True, {'lightgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2')], + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_rdepends_unbuilt_unstable_only(self): + '''Unbuilt reverse dependency which is not in testing''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + }}) + # run britney once to pick up previous results + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}})}) + + # add new uninstallable brokengreen; should not run test at all + exc = self.do_test( + [('brokengreen', {'Version': '1', 'Depends': 'libgreen1, nonexisting'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + 'brokengreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'brokengreen': [('old-version', '-'), ('new-version', '1'), + ('reason', 'depends'), + ('excuses', 'brokengreen/amd64 unsatisfiable Depends: nonexisting')], + })[1] + # autopkgtest should not be triggered for uninstallable pkg + self.assertEqual(exc['brokengreen']['policy_info']['autopkgtest'], {}) + + self.assertEqual(self.amqp_requests, set()) + + def test_rdepends_unbuilt_new_version_result(self): + '''Unbuilt reverse dependency gets test result for newer version + + This might happen if the autopkgtest infrastructure runs the unstable + source tests against the testing binaries. Even if that gets done + properly it might still happen that at the time of the britney run the + package isn't built yet, but it is once the test gets run. + ''' + # old lightgreen fails, thus new green should be held back + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/1.1')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('green/1.1')), + 'series/i386/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 1.1', tr('green/1.1')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 1.1', tr('green/1.1')), + }}) + + # add unbuilt lightgreen; should run tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '1.1', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)] + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # lightgreen 2 stays unbuilt in britney, but we get a test result for it + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100200@': (0, 'lightgreen 2', tr('green/1.1')), + 'series/amd64/l/lightgreen/20150101_102000@': (0, 'lightgreen 2', tr('green/1.1')), + }}) + self.do_test( + [], + {'green': (True, {'green/1.1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '1.1')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)] + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run should not trigger any new requests + self.do_test([], {'green': (True, {}), 'lightgreen': (False, {})}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_rdepends_unbuilt_new_version_fail(self): + '''Unbuilt reverse dependency gets failure for newer version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')), + }}) + + # add unbuilt lightgreen; should request tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)], + } + ) + self.assertEqual(len(self.amqp_requests), 6) + + # we only get a result for lightgreen 2, not for the requested 1 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 0.5', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 0.5', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100200@': (4, 'lightgreen 2', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100200@': (4, 'lightgreen 2', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + }}) + self.do_test( + [], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + 'lightgreen': (False, {}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '2'), + ('missing-builds', ON_ALL_ARCHES)], + } + ) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run should not trigger any new requests + self.do_test([], {'green': (False, {}), 'lightgreen': (False, {})}) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + def test_same_version_binary_in_unstable(self): + '''binary from new architecture in unstable with testing version''' + + # i386 is in testing already, but amd64 just recently built and is in unstable + self.data.add_src('brown', False, {'Testsuite': 'autopkgtest'}) + self.data.add('brown', False, {'Architecture': 'i386'}, add_src=False) + self.data.add('brown', True, {'Architecture': 'amd64'}, add_src=False) + + exc = self.do_test( + # we need some other package to create unstable Sources + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'brown': (True, {})} + )[1] + self.assertEqual(exc['brown']['item-name'], 'brown/amd64') + + def test_package_pair_running(self): + '''Two packages in unstable that need to go in together (running)''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + 'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}}), + }, + {'green': [('old-version', '1'), ('new-version', '2')], + 'lightgreen': [('old-version', '1'), ('new-version', '2')], + }) + + # we expect the package's and its reverse dependencies' tests to get + # triggered; lightgreen should be triggered for each trigger + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:green {"triggers": ["green/2"]}', + 'debci-series-amd64:green {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["lightgreen/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["lightgreen/2"]}', + 'debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}'])) + + # ... and that they get recorded as pending + self.assertEqual(self.pending_requests, + {'lightgreen/2': {'lightgreen': ['amd64', 'i386']}, + 'green/2': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386']}}) + + def test_binary_from_new_source_package_running(self): + '''building an existing binary for a new source package (running)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'newgreen': (True, {'newgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'newgreen': [('old-version', '-'), ('new-version', '2')]}) + + self.assertEqual(len(self.amqp_requests), 8) + self.assertEqual(self.pending_requests, + {'newgreen/2': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386'], + 'newgreen': ['amd64', 'i386']}}) + + def test_binary_from_new_source_package_pass(self): + '''building an existing binary for a new source package (pass)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('newgreen/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('newgreen/2')), + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('newgreen/2')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 1', tr('newgreen/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('newgreen/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('newgreen/2')), + 'series/i386/n/newgreen/20150101_100200@': (0, 'newgreen 2', tr('newgreen/2')), + 'series/amd64/n/newgreen/20150101_100201@': (0, 'newgreen 2', tr('newgreen/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + {'newgreen': (True, {'newgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'green/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'newgreen': [('old-version', '-'), ('new-version', '2')]}) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_result_from_older_version(self): + '''test result from older version than the uploaded one''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('darkgreen/1')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('darkgreen/1')), + }}) + + self.do_test( + [('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], + {'darkgreen': (False, {'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})}) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:darkgreen {"triggers": ["darkgreen/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["darkgreen/2"]}'])) + self.assertEqual(self.pending_requests, + {'darkgreen/2': {'darkgreen': ['amd64', 'i386']}}) + + # second run gets the results for darkgreen 2 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100010@': (0, 'darkgreen 2', tr('darkgreen/2')), + 'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 2', tr('darkgreen/2')), + }}) + self.do_test( + [], + {'darkgreen': (True, {'darkgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}})}) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + # next run sees a newer darkgreen, should re-run tests + self.data.remove_all(True) + self.do_test( + [('darkgreen', {'Version': '3', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], + {'darkgreen': (False, {'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})}) + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:darkgreen {"triggers": ["darkgreen/3"]}', + 'debci-series-amd64:darkgreen {"triggers": ["darkgreen/3"]}'])) + self.assertEqual(self.pending_requests, + {'darkgreen/3': {'darkgreen': ['amd64', 'i386']}}) + + def test_old_result_from_rdep_version(self): + '''re-runs reverse dependency test on new versions''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100010@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100010@': (0, 'green 2', tr('green/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + self.data.remove_all(True) + + # second run: new version re-triggers all tests + self.do_test( + [('libgreen1', {'Version': '3', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + 'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + }), + }) + + self.assertEqual(len(self.amqp_requests), 6) + self.assertEqual(self.pending_requests, + {'green/3': {'darkgreen': ['amd64', 'i386'], + 'green': ['amd64', 'i386'], + 'lightgreen': ['amd64', 'i386']}}) + + # third run gets the results for green and lightgreen, darkgreen is + # still running + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100020@': (0, 'green 3', tr('green/3')), + 'series/amd64/g/green/20150101_100020@': (0, 'green 3', tr('green/3')), + 'series/i386/l/lightgreen/20150101_100010@': (0, 'lightgreen 1', tr('green/3')), + 'series/amd64/l/lightgreen/20150101_100010@': (0, 'lightgreen 1', tr('green/3')), + }}) + self.do_test( + [], + {'green': (False, {'green/3': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}, + }), + }) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, + {'green/3': {'darkgreen': ['amd64', 'i386']}}) + + # fourth run finally gets the new darkgreen result + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100010@': (0, 'darkgreen 1', tr('green/3')), + 'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 1', tr('green/3')), + }}) + self.do_test( + [], + {'green': (True, {'green/3': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_different_versions_on_arches(self): + '''different tested package versions on different architectures''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('passedbefore/1')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('passedbefore/1')), + }}) + + # first run: no results yet + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')], + {'green': (False, {'darkgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})}) + + # second run: i386 result has version 1.1 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100010@': (0, 'darkgreen 1.1', tr('green/2')) + }}) + self.do_test( + [], + {'green': (False, {'darkgreen': {'amd64': 'RUNNING'}, + 'darkgreen/1.1': {'i386': 'PASS'}, + })}) + + # third run: amd64 result has version 1.2 + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/d/darkgreen/20150101_100010@': (0, 'darkgreen 1.2', tr('green/2')), + }}) + self.do_test( + [], + {'green': (True, {'darkgreen/1.2': {'amd64': 'PASS'}, + 'darkgreen/1.1': {'i386': 'PASS'}, + })}) + + def test_tmpfail(self): + '''tmpfail results''' + + # one tmpfail result without testpkg-version, should be ignored + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'series/i386/l/lightgreen/20150101_100101@': (16, None, tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('lightgreen/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (16, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}})}) + self.assertEqual(self.pending_requests, + {'lightgreen/2': {'lightgreen': ['i386']}}) + + # one more tmpfail result, should not confuse britney with None version + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100201@': (16, None, tr('lightgreen/2')), + }}) + self.do_test( + [], + {'lightgreen': (False, {'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}})}) + with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f: + contents = f.read() + self.assertNotIn('null', contents) + self.assertNotIn('None', contents) + + def test_rerun_failure(self): + '''manually re-running failed tests gets picked up''' + + # first run fails + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 2', tr('green/1')), + 'series/i386/g/green/20150101_100101@': (4, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 2', tr('green/1')), + 'series/amd64/g/green/20150101_100101@': (4, 'green 2', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + + # re-running test manually succeeded (note: darkgreen result should be + # cached already) + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100201@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100201@': (0, 'lightgreen 1', tr('green/2')), + }}) + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + + def test_new_runs_dont_clobber_pass(self): + '''passing once is sufficient + + If a test succeeded once for a particular version and trigger, + subsequent failures (which might be triggered by other unstable + uploads) should not invalidate the PASS, as that new failure is the + fault of the new upload, not the original one. + ''' + # new libc6 works fine with green + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('libc6/2')), + 'series/amd64/g/green/20150101_100000@': (0, 'green 1', tr('libc6/2')), + }}) + + self.do_test( + [('libc6', {'Version': '2'}, None)], + {'libc6': (True, {'green/1': {'amd64': 'PASS', 'i386': 'PASS'}})}) + self.assertEqual(self.pending_requests, {}) + + # new green fails; that's not libc6's fault though, so it should stay + # valid + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100100@': (4, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100100@': (4, 'green 2', tr('green/2')), + }}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}}), + 'libc6': (True, {'green/1': {'amd64': 'PASS', 'i386': 'PASS'}}), + }) + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:darkgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:darkgreen {"triggers": ["green/2"]}', + 'debci-series-i386:lightgreen {"triggers": ["green/2"]}', + 'debci-series-amd64:lightgreen {"triggers": ["green/2"]}', + ])) + + def test_remove_from_unstable(self): + '''broken package gets removed from unstable''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100101@': (0, 'green 1', tr('green/1')), + 'series/amd64/g/green/20150101_100101@': (0, 'green 1', tr('green/1')), + 'series/i386/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100201@': (4, 'lightgreen 2', tr('green/2 lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100201@': (4, 'lightgreen 2', tr('green/2 lightgreen/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1', tr('green/2')), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), + ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + # remove new lightgreen by resetting archive indexes, and re-adding + # green + self.data.remove_all(True) + + self.swift.set_results({'autopkgtest-series': { + # add new result for lightgreen 1 + 'series/i386/l/lightgreen/20150101_100301@': (0, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100301@': (0, 'lightgreen 1', tr('green/2')), + }}) + + # next run should re-trigger lightgreen 1 to test against green/2 + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + })[1] + self.assertNotIn('lightgreen 2', exc['green']['policy_info']['autopkgtest']) + + # should not trigger new requests + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + # but the next run should not trigger anything new + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + def test_multiarch_dep(self): + '''multi-arch dependency''' + + # lightgreen has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + }}) + + self.data.add('rainbow', False, {'Depends': 'lightgreen:any'}, + testsuite='autopkgtest') + + self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'rainbow': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'lightgreen': [('old-version', '1'), ('new-version', '2')]} + ) + + def test_nbs(self): + '''source-less binaries do not cause harm''' + + # NBS in testing + self.data.add('liboldgreen0', False, add_src=False) + # NBS in unstable + self.data.add('liboldgreen1', True, add_src=False) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')], + {'green': (True, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')]}) + + def test_newer_version_in_testing(self): + '''Testing version is newer than in unstable''' + + exc = self.do_test( + [('lightgreen', {'Version': '0.9~beta'}, 'autopkgtest')], + {'lightgreen': (False, {})}, + {'lightgreen': [('old-version', '1'), ('new-version', '0.9~beta'), + ('reason', 'newerintesting'), + ('excuses', 'ALERT: lightgreen is newer in testing (1 0.9~beta)') + ] + })[1] + + # autopkgtest should not be triggered + self.assertNotIn('autopkgtest', exc['lightgreen'].get('policy_info', {})) + self.assertEqual(self.pending_requests, {}) + self.assertEqual(self.amqp_requests, set()) + + def test_testsuite_triggers(self): + '''Testsuite-Triggers''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/r/rainbow/20150101_100000@': (0, 'rainbow 1', tr('passedbefore/1')), + }}) + + self.data.add('rainbow', False, testsuite='autopkgtest', + srcfields={'Testsuite-Triggers': 'unicorn, lightgreen, sugar'}) + + self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'rainbow': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + }), + } + ) + + def test_huge_number_of_tests(self): + '''package triggers huge number of tests''' + + for i in range(30): + self.data.add('green%i' % i, False, {'Depends': 'libgreen1'}, testsuite='autopkgtest') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')], + {'green': (True, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'green0': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'green29': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }) + }, + ) + + # requests should all go into the -huge queues + self.assertEqual([x for x in self.amqp_requests if 'huge' not in x], []) + for i in range(30): + for arch in ['i386', 'amd64']: + self.assertIn('debci-huge-series-%s:green%i {"triggers": ["green/2"]}' % + (arch, i), self.amqp_requests) + + ################################################################ + # Tests for hint processing + ################################################################ + + def test_hint_force_badtest(self): + '''force-badtest hint''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_badtest_multi_version(self): + '''force-badtest hint''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'i386': 'IGNORE-FAIL'}, + 'lightgreen/2': {'amd64': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + # hint the version on amd64 too + self.create_hint('pitti', 'force-badtest lightgreen/2') + + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'i386': 'IGNORE-FAIL'}, + 'lightgreen/2': {'amd64': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_badtest_different_version(self): + '''force-badtest hint with non-matching version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + # lower hint version should not apply + self.create_hint('pitti', 'force-badtest lightgreen/0.1') + + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('reason', 'autopkgtest')]} + )[1] + self.assertNotIn('forced-reason', exc['green']) + + # higher hint version should apply + self.create_hint('pitti', 'force-badtest lightgreen/3') + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {} + ) + + def test_hint_force_badtest_arch(self): + '''force-badtest hint for architecture instead of version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1', tr('green/2')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/amd64/all') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'REGRESSION'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + # hint i386 too, then it should become valid + self.create_hint('pitti', 'force-badtest lightgreen/i386/all') + + self.do_test( + [], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen/1': {'amd64': 'IGNORE-FAIL', 'i386': 'IGNORE-FAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_badtest_running(self): + '''force-badtest hint on running test''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('green/1')), + 'series/i386/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2', tr('green/2')), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'PASS', 'i386': 'PASS'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2')] + }) + + def test_hint_force_skiptest(self): + '''force-skiptest hint''' + + self.create_hint('pitti', 'force-skiptest green/2') + + # regression of green, darkgreen ok, lightgreen running + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + 'series/i386/g/green/20150101_100200@': (4, 'green 2', tr('green/2')), + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1', tr('green/2')), + }}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {'green/2': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'REGRESSION'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen/1': {'amd64': 'PASS', 'i386': 'PASS'}, + }), + }, + {'green': [('old-version', '1'), ('new-version', '2'), + ('forced-reason', 'skiptest'), + ('excuses', 'Should wait for tests relating to green 2, but forced by pitti')] + }) + + def test_hint_force_skiptest_different_version(self): + '''force-skiptest hint with non-matching version''' + + # green has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/g/green/20150101_100000@': (0, 'green 1', tr('passedbefore/1')), + }}) + + self.create_hint('pitti', 'force-skiptest green/1') + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (False, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'darkgreen': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + }, + {'green': [('reason', 'autopkgtest')]} + )[1] + self.assertNotIn('forced-reason', exc['green']) + + def test_hint_blockall_runs_tests(self): + '''block-all hint still runs tests''' + + self.create_hint('freeze', 'block-all source') + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (False, {'lightgreen': {'amd64': 'RUNNING', 'i386': 'RUNNING'}})} + ) + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [], + {'lightgreen': (False, {'lightgreen/2': {'amd64': 'PASS', 'i386': 'PASS'}})}, + {'lightgreen': [('reason', 'block')]} + ) + + + ################################################################ + # Tests for non-hint policies + ################################################################ + + def test_lp_bug_block(self): + with open(os.path.join(self.data.path, 'data/series-proposed/Blocks'), 'w') as f: + f.write('darkgreen 12345 1471505000\ndarkgreen 98765 1471500000\n') + + exc = self.do_test( + [('darkgreen', {'Version': '2'}, 'autopkgtest')], + {'darkgreen': (False, {'darkgreen': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}})}, + {'darkgreen': [('reason', 'block'), + ('excuses', 'Not touching package as requested in bug 12345 on Thu Aug 18 07:23:20 2016'), + ('is-candidate', False), + ] + } + )[1] + self.assertEqual(exc['darkgreen']['policy_info']['block-bugs'], + {'12345': 1471505000, '98765': 1471500000}) + + + ################################################################ + # Kernel related tests + ################################################################ + + def test_detect_dkms_autodep8(self): + '''DKMS packages are autopkgtested (via autodep8)''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/f/fancy/20150101_100101@': (0, 'fancy 0.1', tr('passedbefore/1')) + }}) + + self.do_test( + [('dkms', {'Version': '2'}, None)], + {'dkms': (False, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}})}, + {'dkms': [('old-version', '1'), ('new-version', '2')]}) + + def test_kernel_triggers_dkms(self): + '''DKMS packages get triggered by kernel uploads''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + self.do_test( + [('linux-image-generic', {'Source': 'linux-meta'}, None), + ('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ], + {'linux-meta': (True, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}}), + 'linux-meta-lts-grumpy': (True, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}}), + 'linux-meta-64only': (True, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL'}}), + }) + + # one separate test should be triggered for each kernel + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:fancy {"triggers": ["linux-meta/1"]}', + 'debci-series-amd64:fancy {"triggers": ["linux-meta/1"]}', + 'debci-series-i386:fancy {"triggers": ["linux-meta-lts-grumpy/1"]}', + 'debci-series-amd64:fancy {"triggers": ["linux-meta-lts-grumpy/1"]}', + 'debci-series-amd64:fancy {"triggers": ["linux-meta-64only/1"]}'])) + + # ... and that they get recorded as pending + self.assertEqual(self.pending_requests, + {'linux-meta-lts-grumpy/1': {'fancy': ['amd64', 'i386']}, + 'linux-meta/1': {'fancy': ['amd64', 'i386']}, + 'linux-meta-64only/1': {'fancy': ['amd64']}}) + + def test_dkms_results_per_kernel(self): + '''DKMS results get mapped to the triggering kernel version''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + # works against linux-meta and -64only, fails against grumpy i386, no + # result yet for grumpy amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/f/fancy/20150101_100301@': (0, 'fancy 0.5', tr('passedbefore/1')), + 'series/i386/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100201@': (0, 'fancy 1', tr('linux-meta-64only/1')), + 'series/i386/f/fancy/20150101_100301@': (4, 'fancy 1', tr('linux-meta-lts-grumpy/1')), + }}) + + self.do_test( + [('linux-image-generic', {'Source': 'linux-meta'}, None), + ('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ], + {'linux-meta': (True, {'fancy/1': {'amd64': 'PASS', 'i386': 'PASS'}}), + 'linux-meta-lts-grumpy': (False, {'fancy/1': {'amd64': 'RUNNING', 'i386': 'ALWAYSFAIL'}}), + 'linux-meta-64only': (True, {'fancy/1': {'amd64': 'PASS'}}), + }) + + self.assertEqual(self.pending_requests, + {'linux-meta-lts-grumpy/1': {'fancy': ['amd64']}}) + + def test_dkms_results_per_kernel_old_results(self): + '''DKMS results get mapped to the triggering kernel version, old results''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + + # works against linux-meta and -64only, fails against grumpy i386, no + # result yet for grumpy amd64 + self.swift.set_results({'autopkgtest-series': { + # old results without trigger info + 'series/i386/f/fancy/20140101_100101@': (0, 'fancy 1', {}), + 'series/amd64/f/fancy/20140101_100101@': (8, 'fancy 1', {}), + # current results with triggers + 'series/i386/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100101@': (0, 'fancy 1', tr('linux-meta/1')), + 'series/amd64/f/fancy/20150101_100201@': (0, 'fancy 1', tr('linux-meta-64only/1')), + 'series/i386/f/fancy/20150101_100301@': (4, 'fancy 1', tr('linux-meta-lts-grumpy/1')), + }}) + + self.do_test( + [('linux-image-generic', {'Source': 'linux-meta'}, None), + ('linux-image-grumpy-generic', {'Source': 'linux-meta-lts-grumpy'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ], + {'linux-meta': (True, {'fancy/1': {'amd64': 'PASS', 'i386': 'PASS'}}), + # we don't have an explicit result for amd64 + 'linux-meta-lts-grumpy': (False, {'fancy/1': {'amd64': 'RUNNING', 'i386': 'ALWAYSFAIL'}}), + 'linux-meta-64only': (True, {'fancy/1': {'amd64': 'PASS'}}), + }) + + self.assertEqual(self.pending_requests, + {'linux-meta-lts-grumpy/1': {'fancy': ['amd64']}}) + + def test_kernel_triggered_tests(self): + '''linux, lxc, glibc tests get triggered by linux-meta* uploads''' + + self.data.remove_all(False) + self.data.add('libc6-dev', False, {'Source': 'glibc', 'Depends': 'linux-libc-dev'}, + testsuite='autopkgtest') + self.data.add('lxc', False, {'Testsuite-Triggers': 'linux-generic'}, + testsuite='autopkgtest') + self.data.add('systemd', False, {'Testsuite-Triggers': 'linux-generic'}, + testsuite='autopkgtest') + self.data.add('linux-image-1', False, {'Source': 'linux'}, testsuite='autopkgtest') + self.data.add('linux-libc-dev', False, {'Source': 'linux'}, testsuite='autopkgtest') + self.data.add('linux-image', False, {'Source': 'linux-meta', 'Depends': 'linux-image-1'}) + + self.swift.set_results({'autopkgtest-series': { + 'series/amd64/l/lxc/20150101_100101@': (0, 'lxc 0.1', tr('passedbefore/1')) + }}) + + exc = self.do_test( + [('linux-image', {'Version': '2', 'Depends': 'linux-image-2', 'Source': 'linux-meta'}, None), + ('linux-image-64only', {'Source': 'linux-meta-64only', 'Architecture': 'amd64'}, None), + ('linux-image-2', {'Version': '2', 'Source': 'linux'}, 'autopkgtest'), + ('linux-libc-dev', {'Version': '2', 'Source': 'linux'}, 'autopkgtest'), + ], + {'linux-meta': (False, {'lxc': {'amd64': 'RUNNING', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'glibc': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'linux': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + 'systemd': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}, + }), + 'linux-meta-64only': (False, {'lxc': {'amd64': 'RUNNING'}}), + 'linux': (False, {}), + })[1] + # the kernel itself should not trigger tests; we want to trigger + # everything from -meta + self.assertEqual(exc['linux']['policy_info']['autopkgtest'], {}) + + def test_kernel_waits_on_meta(self): + '''linux waits on linux-meta''' + + self.data.add('dkms', False, {}) + self.data.add('fancy-dkms', False, {'Source': 'fancy', 'Depends': 'dkms (>= 1)'}) + self.data.add('linux-image-generic', False, {'Version': '0.1', 'Source': 'linux-meta', 'Depends': 'linux-image-1'}) + self.data.add('linux-image-1', False, {'Source': 'linux'}, testsuite='autopkgtest') + self.data.add('linux-firmware', False, {'Source': 'linux-firmware'}, testsuite='autopkgtest') + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/f/fancy/20150101_090000@': (0, 'fancy 0.5', tr('passedbefore/1')), + 'series/i386/l/linux/20150101_100000@': (0, 'linux 2', tr('linux-meta/0.2')), + 'series/amd64/l/linux/20150101_100000@': (0, 'linux 2', tr('linux-meta/0.2')), + 'series/i386/l/linux-firmware/20150101_100000@': (0, 'linux-firmware 2', tr('linux-firmware/2')), + 'series/amd64/l/linux-firmware/20150101_100000@': (0, 'linux-firmware 2', tr('linux-firmware/2')), + }}) + + self.do_test( + [('linux-image-generic', {'Version': '0.2', 'Source': 'linux-meta', 'Depends': 'linux-image-2'}, None), + ('linux-image-2', {'Version': '2', 'Source': 'linux'}, 'autopkgtest'), + ('linux-firmware', {'Version': '2', 'Source': 'linux-firmware'}, 'autopkgtest'), + ], + {'linux-meta': (False, {'fancy': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'linux/2': {'amd64': 'PASS', 'i386': 'PASS'} + }), + # no tests, but should wait on linux-meta + 'linux': (False, {}), + # this one does not have a -meta, so don't wait + 'linux-firmware': (True, {'linux-firmware/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + }, + {'linux': [('reason', 'depends'), + ('excuses', 'Invalidated by dependency'), + ('dependencies', {'blocked-by': ['linux-meta']})] + } + ) + + # now linux-meta is ready to go + self.swift.set_results({'autopkgtest-series': { + 'series/i386/f/fancy/20150101_100000@': (0, 'fancy 1', tr('linux-meta/0.2')), + 'series/amd64/f/fancy/20150101_100000@': (0, 'fancy 1', tr('linux-meta/0.2')), + }}) + self.do_test( + [], + {'linux-meta': (True, {'fancy/1': {'amd64': 'PASS', 'i386': 'PASS'}, + 'linux/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + 'linux': (True, {}), + 'linux-firmware': (True, {'linux-firmware/2': {'amd64': 'PASS', 'i386': 'PASS'}}), + }, + {'linux': [('dependencies', {'migrate-after': ['linux-meta']})] + } + ) + + ################################################################ + # Tests for special-cased packages + ################################################################ + + def test_gcc(self): + '''gcc only triggers some key packages''' + + self.data.add('binutils', False, {}, testsuite='autopkgtest') + self.data.add('linux', False, {}, testsuite='autopkgtest') + self.data.add('notme', False, {'Depends': 'libgcc1'}, testsuite='autopkgtest') + + # binutils has passed before on i386 only, therefore ALWAYSFAIL on amd64 + self.swift.set_results({'autopkgtest-series': { + 'series/i386/b/binutils/20150101_100000@': (0, 'binutils 1', tr('passedbefore/1')), + }}) + + exc = self.do_test( + [('libgcc1', {'Source': 'gcc-5', 'Version': '2'}, None)], + {'gcc-5': (False, {'binutils': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING'}, + 'linux': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'}})})[1] + self.assertNotIn('notme 1', exc['gcc-5']['policy_info']['autopkgtest']) + + def test_alternative_gcc(self): + '''alternative gcc does not trigger anything''' + + self.data.add('binutils', False, {}, testsuite='autopkgtest') + self.data.add('notme', False, {'Depends': 'libgcc1'}, testsuite='autopkgtest') + + exc = self.do_test( + [('libgcc1', {'Source': 'gcc-snapshot', 'Version': '2'}, None)], + {'gcc-snapshot': (True, {})})[1] + self.assertEqual(exc['gcc-snapshot']['policy_info']['autopkgtest'], {}) + + ################################################################ + # Tests for non-default ADT_* configuration modes + ################################################################ + + def test_disable_adt(self): + '''Run without autopkgtest requests''' + + # Disable AMQP server config, to ensure we don't touch them with ADT + # disabled + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('ADT_ENABLE'): + print('ADT_ENABLE = no') + elif not line.startswith('ADT_AMQP') and not line.startswith('ADT_SWIFT_URL'): + sys.stdout.write(line) + + exc = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {'green': (True, {})}, + {'green': [('old-version', '1'), ('new-version', '2')]})[1] + self.assertNotIn('autopkgtest', exc['green']['policy_info']) + + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, None) + + def test_ppas(self): + '''Run test requests with additional PPAs''' + + for line in fileinput.input(self.britney_conf, inplace=True): + if line.startswith('ADT_PPAS'): + print('ADT_PPAS = joe/foo awesome-developers/staging') + else: + sys.stdout.write(line) + + exc = self.do_test( + [('lightgreen', {'Version': '2'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen': {'amd64': 'RUNNING-ALWAYSFAIL'}})}, + {'lightgreen': [('old-version', '1'), ('new-version', '2')]} + )[1] + self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], + {'lightgreen': { + 'amd64': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + None, + None, + None], + 'i386': ['RUNNING-ALWAYSFAIL', + 'http://autopkgtest.ubuntu.com/running', + None, + None, + None]} + }) + + for arch in ['i386', 'amd64']: + self.assertTrue('debci-ppa-series-%s:lightgreen {"triggers": ["lightgreen/2"], "ppas": ["joe/foo", "awesome-developers/staging"]}' % arch in self.amqp_requests or + 'debci-ppa-series-%s:lightgreen {"ppas": ["joe/foo", "awesome-developers/staging"], "triggers": ["lightgreen/2"]}' % arch in self.amqp_requests, + self.amqp_requests) + self.assertEqual(len(self.amqp_requests), 2) + + # add results to PPA specific swift container + self.swift.set_results({'autopkgtest-series-awesome-developers-staging': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1', tr('passedbefore/1')), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + exc = self.do_test( + [], + {'lightgreen': (False, {'lightgreen/2': {'i386': 'REGRESSION', 'amd64': 'PASS'}})}, + {'lightgreen': [('old-version', '1'), ('new-version', '2')]} + )[1] + self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], + {'lightgreen/2': { + 'amd64': ['PASS', + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/amd64/l/lightgreen/20150101_100101@/log.gz', + None, + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/amd64/l/lightgreen/20150101_100101@/artifacts.tar.gz', + None], + 'i386': ['REGRESSION', + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/i386/l/lightgreen/20150101_100100@/log.gz', + None, + 'http://localhost:18085/autopkgtest-series-awesome-developers-staging/series/i386/l/lightgreen/20150101_100100@/artifacts.tar.gz', + 'https://autopkgtest.ubuntu.com/request.cgi?release=series&arch=i386&package=lightgreen&' + 'trigger=lightgreen%2F2&ppa=joe%2Ffoo&ppa=awesome-developers%2Fstaging']} + }) + self.assertEqual(self.amqp_requests, set()) + self.assertEqual(self.pending_requests, {}) + + def test_disable_upgrade_tester(self): + '''Run without second stage upgrade tester''' + + for line in fileinput.input(self.britney_conf, inplace=True): + if not line.startswith('UPGRADE_OUTPUT'): + sys.stdout.write(line) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + {})[1] + + self.assertFalse(os.path.exists(os.path.join(self.data.path, 'output', 'series', 'output.txt'))) + + def test_shared_results_cache(self): + '''Run with shared r/o results.cache''' + + # first run to create results.cache + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 2', tr('lightgreen/2')), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 2', tr('lightgreen/2')), + }}) + + self.do_test( + [('lightgreen', {'Version': '2', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen/2': {'i386': 'PASS', 'amd64': 'PASS'}})}, + ) + + # move and remember original contents + local_path = os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache') + shared_path = os.path.join(self.data.path, 'shared_results.cache') + os.rename(local_path, shared_path) + with open(shared_path) as f: + orig_contents = f.read() + + # enable shared cache + for line in fileinput.input(self.britney_conf, inplace=True): + if 'ADT_SHARED_RESULTS_CACHE' in line: + print('ADT_SHARED_RESULTS_CACHE = %s' % shared_path) + else: + sys.stdout.write(line) + + # second run, should now not update cache + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 3', tr('lightgreen/3')), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 3', tr('lightgreen/3')), + }}) + + self.data.remove_all(True) + self.do_test( + [('lightgreen', {'Version': '3', 'Depends': 'libc6'}, 'autopkgtest')], + {'lightgreen': (True, {'lightgreen/3': {'i386': 'PASS', 'amd64': 'PASS'}})}, + ) + + # leaves results.cache untouched + self.assertFalse(os.path.exists(local_path)) + with open(shared_path) as f: + self.assertEqual(orig_contents, f.read()) + + +if __name__ == '__main__': + unittest.main()