diff --git a/autopkgtest.py b/autopkgtest.py index e7e68a3..e1bf838 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (C) 2013 Canonical Ltd. -# Author: Colin Watson -# Partly based on code in auto-package-testing by -# Jean-Baptiste Lallement +# Authors: +# Colin Watson +# Jean-Baptiste Lallement +# Martin Pitt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,12 +18,7 @@ from __future__ import print_function -from collections import defaultdict -from contextlib import closing import os -import subprocess -import tempfile -from textwrap import dedent import time import json import tarfile @@ -37,9 +33,6 @@ import kombu from consts import (AUTOPKGTEST, BINARIES, DEPENDS, RDEPENDS, SOURCE, VERSION) -adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") - -ADT_PASS = ["PASS", "ALWAYSFAIL"] ADT_EXCUSES_LABELS = { "PASS": 'Pass', "ALWAYSFAIL": 'Always failed', @@ -84,8 +77,6 @@ class AutoPackageTest(object): self.distribution = distribution self.series = series self.debug = debug - self.read() - self.rc_path = None # for adt-britney, obsolete self.test_state_dir = os.path.join(britney.options.unstable, 'autopkgtest') # map of requested tests from request() @@ -397,92 +388,6 @@ class AutoPackageTest(object): result.add((src, arch)) return result - # - # obsolete adt-britney helpers - # - - def _ensure_rc_file(self): - if self.rc_path: - return - self.rc_path = os.path.expanduser( - "~/proposed-migration/autopkgtest/rc.%s" % self.series) - with open(self.rc_path, "w") as rc_file: - home = os.path.expanduser("~") - print(dedent("""\ - release: %s - aptroot: ~/.chdist/%s-proposed-amd64/ - apturi: file:%s/mirror/%s - components: main restricted universe multiverse - rsync_host: rsync://tachash.ubuntu-ci/adt/ - datadir: ~/proposed-migration/autopkgtest/data""" % - (self.series, self.series, home, self.distribution)), - file=rc_file) - - @property - def _request_path(self): - return os.path.expanduser( - "~/proposed-migration/autopkgtest/work/adt.request.%s" % - self.series) - - @property - def _result_path(self): - return os.path.expanduser( - "~/proposed-migration/autopkgtest/work/adt.result.%s" % - self.series) - - def _parse(self, path): - if os.path.exists(path): - with open(path) as f: - for line in f: - line = line.strip() - if line.startswith("Suite:") or line.startswith("Date:"): - continue - linebits = line.split() - if len(linebits) < 2: - print("W: Invalid line format: '%s', skipped" % line) - continue - yield linebits - - def read(self): - '''Loads a list of results - - This function loads a list of results returned by __parse() and builds - 2 lists: - - a list of source package/version with all the causes that - triggered a test and the result of the test for this trigger. - - a list of packages/version that triggered a test with the source - package/version and result triggered by this package. - These lists will be used in result() called from britney.py to generate - excuses and now which uploads passed, caused regression or which tests - have always been failing - ''' - self.pkglist = defaultdict(dict) - self.pkgcauses = defaultdict(lambda: defaultdict(list)) - for linebits in self._parse(self._result_path): - (src, ver, status) = linebits[:3] - - if not (src in self.pkglist and ver in self.pkglist[src]): - self.pkglist[src][ver] = { - "status": status, - "causes": {} - } - - i = iter(linebits[3:]) - for trigsrc, trigver in zip(i, i): - self.pkglist[src][ver]['causes'].setdefault( - trigsrc, []).append((trigver, status)) - self.pkgcauses[trigsrc][trigver].append((status, src, ver)) - - def _adt_britney(self, *args): - command = [ - adt_britney, - "-c", self.rc_path, "-r", self.series, "-PU", - ] - if self.debug: - command.append("-d") - command.extend(args) - subprocess.check_call(command) - # # Public API # @@ -506,60 +411,6 @@ class AutoPackageTest(object): self.log_verbose('Requesting %s/%s/%s autopkgtest to verify %s' % (src, ver, arch, ', '.join(['%s/%s' % i for i in triggers]))) - # deprecated requests for old Jenkins/lp:auto-package-testing, will go - # away - - self._ensure_rc_file() - request_path = self._request_path - if os.path.exists(request_path): - os.unlink(request_path) - with closing(tempfile.NamedTemporaryFile(mode="w")) as request_file: - for src, ver in packages: - if src in self.pkglist and ver in self.pkglist[src]: - continue - print("%s %s" % (src, ver), file=request_file) - request_file.flush() - self._adt_britney("request", "-O", request_path, request_file.name) - - # Remove packages that have been identified as invalid candidates for - # testing from the request file i.e run_autopkgtest = False - with open(request_path, 'r') as request_file: - lines = request_file.readlines() - with open(request_path, 'w') as request_file: - for line in lines: - src = line.split()[0] - if src not in excludes: - request_file.write(line) - else: - if self.britney.options.verbose: - self.log_verbose("Requested autopkgtest for %s but " - "run_autopkgtest set to False" % src) - - for linebits in self._parse(request_path): - # Make sure that there's an entry in pkgcauses for each new - # request, so that results() gives useful information without - # relying on the submit/collect cycle. This improves behaviour - # in dry-run mode. - src = linebits.pop(0) - ver = linebits.pop(0) - if self.britney.options.verbose: - self.log_verbose("Requested autopkgtest for %s_%s (%s)" % - (src, ver, " ".join(linebits))) - try: - status = linebits.pop(0).upper() - while True: - trigsrc = linebits.pop(0) - trigver = linebits.pop(0) - for status, csrc, cver in self.pkgcauses[trigsrc][trigver]: - if csrc == trigsrc and cver == trigver: - break - else: - self.pkgcauses[trigsrc][trigver].append( - (status, src, ver)) - except IndexError: - # End of the list - pass - def submit(self): # send AMQP requests for new test requests # TODO: Once we support version constraints in AMQP requests, add them @@ -601,89 +452,81 @@ class AutoPackageTest(object): # mark them as pending now self.update_pending_tests() - # deprecated requests for old Jenkins/lp:auto-package-testing, will go - # away - self._ensure_rc_file() - request_path = self._request_path - if os.path.exists(request_path): - self._adt_britney("submit", request_path) - def collect(self, packages): # fetch results from swift try: swift_url = self.britney.options.adt_swift_url except AttributeError: self.log_error('ADT_SWIFT_URL not set, cannot collect results') - swift_url = None + return try: self.britney.options.adt_amqp except AttributeError: self.log_error('ADT_AMQP not set, not collecting results from swift') - swift_url = None - - if swift_url: - # update results from swift for all packages that we are waiting - # for, and remove pending tests that we have results for on all - # arches - for pkg, verinfo in copy.deepcopy(self.pending_tests.items()): - for archinfo in verinfo.values(): - for arch in archinfo: - self.fetch_swift_results(swift_url, pkg, arch) - # also update results for excuses whose tests failed, in case a - # manual retry worked - for (trigpkg, trigver) in packages: - if trigpkg not in self.pending_tests: - for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver): - self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' % - (pkg, arch, trigpkg, trigver)) - self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver)) - - # update the results cache - with open(self.results_cache_file + '.new', 'w') as f: - json.dump(self.test_results, f, indent=2) - os.rename(self.results_cache_file + '.new', self.results_cache_file) - self.log_verbose('Updated results cache') - - # new results remove pending requests, update the on-disk cache - self.update_pending_tests() - - # deprecated results for old Jenkins/lp:auto-package-testing, will go - # away - self._ensure_rc_file() - result_path = self._result_path - self._adt_britney("collect", "-O", result_path) - self.read() - if self.britney.options.verbose: - for src in sorted(self.pkglist): - for ver in sorted(self.pkglist[src], - cmp=apt_pkg.version_compare): - for trigsrc in sorted(self.pkglist[src][ver]['causes']): - for trigver, status \ - in self.pkglist[src][ver]['causes'][trigsrc]: - self.log_verbose("Collected autopkgtest status " - "for %s_%s/%s_%s: " "%s" % - (src, ver, trigsrc, trigver, status)) + return + + # update results from swift for all packages that we are waiting + # for, and remove pending tests that we have results for on all + # arches + for pkg, verinfo in copy.deepcopy(self.pending_tests.items()): + for archinfo in verinfo.values(): + for arch in archinfo: + self.fetch_swift_results(swift_url, pkg, arch) + # also update results for excuses whose tests failed, in case a + # manual retry worked + for (trigpkg, trigver) in packages: + if trigpkg not in self.pending_tests: + for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver): + self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' % + (pkg, arch, trigpkg, trigver)) + self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver)) + + # update the results cache + with open(self.results_cache_file + '.new', 'w') as f: + json.dump(self.test_results, f, indent=2) + os.rename(self.results_cache_file + '.new', self.results_cache_file) + self.log_verbose('Updated results cache') + + # new results remove pending requests, update the on-disk cache + self.update_pending_tests() def results(self, trigsrc, trigver): '''Return test results for triggering package - Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests - that got triggered by trigsrc/trigver. + Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING) + iterator for all package tests that got triggered by trigsrc/trigver. ''' - # deprecated results for old Jenkins/lp:auto-package-testing, will go - # away - for status, src, ver in self.pkgcauses[trigsrc][trigver]: - # Check for regression - if status == 'FAIL': - passed_once = False - for lver in self.pkglist[src]: - for trigsrc in self.pkglist[src][lver]['causes']: - for trigver, status \ - in self.pkglist[src][lver]['causes'][trigsrc]: - if status == 'PASS': - passed_once = True - if not passed_once: - status = 'ALWAYSFAIL' - else: - status = 'REGRESSION' - yield status, src, ver + for testsrc, testver in self.tests_for_source(trigsrc, trigver): + passed = True + arch_status = {} + for arch in self.britney.options.adt_arches.split(): + try: + (_, ver_map, ever_passed) = self.test_results[testsrc][arch] + (status, triggers) = ver_map[testver] + # triggers might contain tuples or lists + if (trigsrc, trigver) not in triggers and [trigsrc, trigver] not in triggers: + raise KeyError('No result for trigger %s/%s yet' % (trigsrc, trigver)) + if status: + arch_status[arch] = 'PASS' + else: + # test failed, check ever_passed flag for that src/arch + if ever_passed: + arch_status[arch] = 'REGRESSION' + passed = False + else: + arch_status[arch] = 'ALWAYSFAIL' + except KeyError: + # no result for testsrc/testver/arch; still running? + try: + self.pending_tests[testsrc][testver][arch] + arch_status[arch] = 'RUNNING' + passed = False + except KeyError: + # neither done nor pending -> exclusion, or disabled + continue + + # disabled or ignored? + if not arch_status: + continue + + yield (passed, testsrc, testver, arch_status) diff --git a/britney.py b/britney.py index fe22287..5b518d2 100755 --- a/britney.py +++ b/britney.py @@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes, from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) -from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash +from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash from boottest import BootTest @@ -1849,30 +1849,21 @@ class Britney(object): if not self.options.dry_run: autopkgtest.submit() autopkgtest.collect(autopkgtest_packages) - jenkins_public = "https://jenkins.qa.ubuntu.com/job" - jenkins_private = ( - "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" % - self.options.series.title()) cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" for e in autopkgtest_excuses: adtpass = True - adt_jenkins_sources = set() # temporary: drop when switching to cloud based tests - for status, adtsrc, adtver in autopkgtest.results( + for passed, adtsrc, adtver, arch_status in autopkgtest.results( e.name, e.ver[1]): - adt_jenkins_sources.add(adtsrc) - public_url = "%s/%s-adt-%s/lastBuild" % ( - jenkins_public, self.options.series, - adtsrc.replace("+", "-")) - private_url = "%s/%s-adt-%s/lastBuild" % ( - jenkins_private, self.options.series, - adtsrc.replace("+", "-")) - adt_label = ADT_EXCUSES_LABELS.get(status, status) - e.addhtml( - "autopkgtest for %s %s: %s (Jenkins: " - "public, " - "private)" % - (adtsrc, adtver, adt_label, public_url, private_url)) - if status not in ADT_PASS: + archmsg = [] + for arch in sorted(arch_status): + url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc, + 'r': self.options.series, 'a': arch} + archmsg.append('%s: %s' % + (url, arch, ADT_EXCUSES_LABELS[arch_status[arch]])) + e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg))) + + # hints can override failures + if not passed: hints = self.hints.search( 'force-badtest', package=adtsrc) hints.extend( @@ -1884,54 +1875,10 @@ class Britney(object): e.addhtml( "Should wait for %s %s test, but forced by " "%s" % (adtsrc, adtver, forces[0].user)) - else: - adtpass = False - - # temporary: also show results from cloud based tests, - # until that becomes the primary mechanism - adt_cloud_sources = set() - for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]): - adt_cloud_sources.add(testsrc) - msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver) - archmsg = [] - for arch in self.options.adt_arches.split(): - url = cloud_url % {'h': srchash(testsrc), 's': testsrc, - 'r': self.options.series, 'a': arch} - try: - (_, ver_map, ever_passed) = autopkgtest.test_results[testsrc][arch] - (passed, triggers) = ver_map[testver] - # triggers might contain tuples or lists - if (e.name, e.ver[1]) not in triggers and [e.name, e.ver[1]] not in triggers: - raise KeyError('No result for trigger %s/%s yet' % (e.name, e.ver[1])) - - if passed: - status = 'PASS' - else: - if ever_passed: - status = 'REGRESSION' - else: - status = 'ALWAYSFAIL' - except KeyError: - try: - autopkgtest.pending_tests[testsrc][testver][arch] - status = 'RUNNING' - except KeyError: - # neither done nor pending -> exclusion, or disabled - continue + passed = True - archmsg.append('%s: %s' % - (url, arch, ADT_EXCUSES_LABELS[status])) - - if archmsg: - e.addhtml(msg + ', '.join(archmsg)) - - extra_sources = adt_cloud_sources - adt_jenkins_sources - missing_sources = adt_jenkins_sources - adt_cloud_sources - if missing_sources: - e.addhtml('(informational) Missing sources in cloud tests: %s' % ' '.join(missing_sources)) - if extra_sources: - e.addhtml('(informational) Extra sources in cloud tests: %s' % ' '.join(extra_sources)) - # end of temporary code + if not passed: + adtpass = False if not adtpass and e.is_valid: hints = self.hints.search('force-skiptest', package=e.name) diff --git a/tests/__init__.py b/tests/__init__.py index c682229..c2f320b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -172,3 +172,11 @@ class TestBase(unittest.TestCase): excuses = f.read() return (excuses, out) + + def create_hint(self, username, content): + '''Create a hint file for the given username and content''' + + hints_path = os.path.join( + self.data.path, 'data', self.data.series + '-proposed', 'Hints', username) + with open(hints_path, 'w') as fd: + fd.write(content) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 6dbccb9..1ee49e6 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -7,10 +7,8 @@ # (at your option) any later version. import apt_pkg -import operator import os import sys -import subprocess import fileinput import unittest import json @@ -18,7 +16,6 @@ import json PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, PROJECT_DIR) -from autopkgtest import ADT_EXCUSES_LABELS from tests import TestBase, mock_swift NOT_CONSIDERED = False @@ -46,16 +43,6 @@ class TestAutoPkgTest(TestBase): else: sys.stdout.write(line) - # fake adt-britney script; necessary until we drop that code - self.adt_britney = os.path.join( - self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney') - os.makedirs(os.path.dirname(self.adt_britney)) - with open(self.adt_britney, 'w') as f: - f.write('''#!/bin/sh -e -touch $HOME/proposed-migration/autopkgtest/work/adt.request.series -echo "$@" >> /%s/adt-britney.log ''' % self.data.path) - os.chmod(self.adt_britney, 0o755) - # add a bunch of packages to testing to avoid repetition self.data.add('libc6', False) self.data.add('libgreen1', False, {'Source': 'green', @@ -89,7 +76,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) (excuses, out) = self.run_britney() self.swift.stop() - #print('-------\nexcuses: %s\n-----' % excuses) + if 'SHOW_EXCUSES' in os.environ: + print('-------\nexcuses: %s\n-----' % excuses) if 'SHOW_OUTPUT' in os.environ: print('-------\nout: %s\n-----' % out) if considered: @@ -120,14 +108,27 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) return out + def test_no_request_for_uninstallable(self): + '''Does not request a test for an uninstallable package''' + + self.do_test( + # uninstallable unstable version + [('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\blightgreen\b.*>1 to .*>1.1~beta<', + 'lightgreen/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'], + # autopkgtest should not be triggered for uninstallable pkg + ['autopkgtest']) + + self.assertEqual(self.pending_requests, '') + self.assertEqual(self.amqp_requests, set()) + def test_multi_rdepends_with_tests_all_running(self): '''Multiple reverse dependencies with tests (all running)''' self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -153,7 +154,7 @@ lightgreen 1 i386 green 2 self.assertEqual(self.pending_requests, expected_pending) # if we run britney again this should *not* trigger any new tests - self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<']) + self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<']) self.assertEqual(self.amqp_requests, set()) # but the set of pending tests doesn't change self.assertEqual(self.pending_requests, expected_pending) @@ -164,9 +165,7 @@ lightgreen 1 i386 green 2 # first run requests tests and marks them as pending self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -233,9 +232,7 @@ lightgreen 1 i386 green 2 # first run requests tests and marks them as pending self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -252,9 +249,7 @@ lightgreen 1 i386 green 2 out = self.do_test( [], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass', r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress', @@ -267,15 +262,174 @@ lightgreen 1 i386 green 2 self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) + def test_multi_rdepends_with_tests_regression(self): + '''Multiple reverse dependencies with tests (regression)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_regression_last_pass(self): + '''Multiple reverse dependencies with tests (regression), last one passes + + This ensures that we don't just evaluate the test result of the last + test, but all of them. + ''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_always_failed(self): + '''Multiple reverse dependencies with tests (always failed)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (4, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_hint_force_badtest(self): + '''force-badtest hint''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass', + r'Should wait for lightgreen 1 test, but forced by pitti']) + + def test_hint_force_badtest_different_version(self): + '''force-badtest hint with non-matching version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/0.1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'], + ['Should wait']) + + def test_hint_force_skiptest(self): + '''force-skiptest hint''' + + self.create_hint('pitti', 'force-skiptest green/2') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress', + r'Should wait for.*tests.*green 2.*forced by pitti']) + + def test_hint_force_skiptest_different_version(self): + '''force-skiptest hint with non-matching version''' + + self.create_hint('pitti', 'force-skiptest green/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'], + ['Should wait']) + def test_package_pair_running(self): '''Two packages in unstable that need to go in together (running)''' self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'\blightgreen\b.*>1 to .*>2<']) @@ -300,6 +454,58 @@ lightgreen 2 i386 lightgreen 2 ''' self.assertEqual(self.pending_requests, expected_pending) + def test_binary_from_new_source_package_running(self): + '''building an existing binary for a new source package (running)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bnewgreen\b.*\(- to .*>2<', + r'autopkgtest for newgreen 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen', + 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) + expected_pending = '''darkgreen 1 amd64 newgreen 2 +darkgreen 1 i386 newgreen 2 +lightgreen 1 amd64 newgreen 2 +lightgreen 1 i386 newgreen 2 +newgreen 2 amd64 newgreen 2 +newgreen 2 i386 newgreen 2 +''' + self.assertEqual(self.pending_requests, expected_pending) + + def test_binary_from_new_source_package_pass(self): + '''building an existing binary for a new source package (pass)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/n/newgreen/20150101_100200@': (0, 'newgreen 2'), + 'series/amd64/n/newgreen/20150101_100201@': (0, 'newgreen 2'), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bnewgreen\b.*\(- to .*>2<', + r'autopkgtest for newgreen 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen', + 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) + self.assertEqual(self.pending_requests, '') + def test_result_from_older_version(self): '''test result from older version than the uploaded one''' @@ -310,9 +516,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bdarkgreen\b.*>1 to .*>2<', r'autopkgtest for darkgreen 2: .*amd64.*in progress.*i386.*in progress']) @@ -342,9 +546,7 @@ lightgreen 2 i386 lightgreen 2 self.data.remove_all(True) self.do_test( [('darkgreen', {'Version': '3', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bdarkgreen\b.*>1 to .*>3<', r'autopkgtest for darkgreen 3: .*amd64.*in progress.*i386.*in progress']) self.assertEqual( @@ -386,9 +588,7 @@ lightgreen 2 i386 lightgreen 2 # second run: new version re-triggers all tests self.do_test( [('libgreen1', {'Version': '3', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\green\b.*>1 to .*>3<', r'autopkgtest for green 3: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -421,10 +621,7 @@ lightgreen 1 i386 green 3 'series/amd64/l/lightgreen/20150101_100010@': (0, 'lightgreen 1'), }}) self.do_test( - [], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + [], NOT_CONSIDERED, [r'\green\b.*>1 to .*>3<', r'autopkgtest for green 3: .*amd64.*Pass.*i386.*Pass', r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', @@ -460,9 +657,7 @@ lightgreen 1 i386 green 3 self.do_test( [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'], ['in progress']) @@ -488,9 +683,7 @@ lightgreen 1 i386 green 3 self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression', r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression']) @@ -539,9 +732,7 @@ lightgreen 1 i386 green 3 self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', @@ -603,9 +794,7 @@ lightgreen 1 i386 green 3 self.do_test( [('lightgreen', {'Version': '2'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for lightgreen 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for rainbow 1: .*amd64.*in progress.*i386.*in progress']) @@ -618,9 +807,7 @@ lightgreen 1 i386 green 3 self.do_test( [('dkms', {'Version': '2'}, None)], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bdkms\b.*>1 to .*>2<', r'autopkgtest for fancy 1: .*amd64.*in progress.*i386.*in progress']) @@ -641,448 +828,5 @@ lightgreen 1 i386 green 3 self.assertEqual(self.pending_requests, None) -class TestAdtBritney(TestBase): - '''Legacy adt-britney/lp:auto-package-testing interface''' - - def setUp(self): - super(TestAdtBritney, self).setUp() - - # Mofify configuration according to the test context. - with open(self.britney_conf, 'r') as fp: - original_config = fp.read() - # Disable boottests. - new_config = original_config.replace( - 'BOOTTEST_ENABLE = yes', 'BOOTTEST_ENABLE = no') - with open(self.britney_conf, 'w') as fp: - fp.write(new_config) - - # fake adt-britney script - self.adt_britney = os.path.join( - self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney') - os.makedirs(os.path.dirname(self.adt_britney)) - - with open(self.adt_britney, 'w') as f: - f.write('''#!/bin/sh -e -echo "$@" >> /%s/adt-britney.log ''' % self.data.path) - os.chmod(self.adt_britney, 0o755) - - # add a bunch of packages to testing to avoid repetition - self.data.add('libc6', False) - self.data.add('libgreen1', False, {'Source': 'green', - 'Depends': 'libc6 (>= 0.9)'}) - self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', - 'Conflicts': 'blue'}) - self.data.add('lightgreen', False, {'Depends': 'libgreen1'}) - self.data.add('darkgreen', False, {'Depends': 'libgreen1'}) - self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', - 'Conflicts': 'green'}) - self.data.add('justdata', False, {'Architecture': 'all'}) - - def __merge_records(self, results, history=""): - '''Merges a list of results with records in history. - - This function merges results from a collect with records already in - history and sort records by version/name of causes and version/name of - source packages with tests. This should be done in the fake - adt-britney but it is more convenient to just pass a static list of - records and make adt-britney just return this list. - ''' - - if history is None: - history = "" - records = [x.split() for x in (results.strip() + '\n' + - history.strip()).split('\n') if x] - - records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(4)) - records.sort(key=operator.itemgetter(3)) - records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(1)) - records.sort() - - return "\n".join([' '.join(x) for x in records]) - - def make_adt_britney(self, request, history=""): - with open(self.adt_britney, 'w') as f: - f.write('''#!%(py)s -import argparse, shutil,sys - -def request(): - if args.req: - shutil.copy(args.req, '%(path)s/adt-britney.requestarg') - with open(args.output, 'w') as f: - f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW').replace('RUNNING', 'NEW')) - -def submit(): - with open(args.req, 'w') as f: - f.write("""%(rq)s""".replace('PASS', 'RUNNING'). - replace('FAIL', 'RUNNING')) - -def collect(): - with open(args.output, 'w') as f: - f.write("""%(res)s""") - -p = argparse.ArgumentParser() -p.add_argument('-c', '--config') -p.add_argument('-a', '--arch') -p.add_argument('-r', '--release') -p.add_argument('-P', '--use-proposed', action='store_true') -p.add_argument('-d', '--debug', action='store_true') -p.add_argument('-U', '--no-update', action='store_true') -sp = p.add_subparsers() - -prequest = sp.add_parser('request') -prequest.add_argument('-O', '--output') -prequest.add_argument('req', nargs='?') -prequest.set_defaults(func=request) - -psubmit = sp.add_parser('submit') -psubmit.add_argument('req') -psubmit.set_defaults(func=submit) - -pcollect = sp.add_parser('collect') -pcollect.add_argument('-O', '--output') -pcollect.add_argument('-n', '--new-only', action='store_true', default=False) -pcollect.set_defaults(func=collect) - -args = p.parse_args() -args.func() - ''' % {'py': sys.executable, 'path': self.data.path, - 'rq': request, - 'res': self.__merge_records(request, history)}) - - def do_test(self, unstable_add, adt_request, considered, expect=None, - no_expect=None, history=""): - for (pkg, fields) in unstable_add: - self.data.add(pkg, True, fields) - - self.make_adt_britney(adt_request, history) - - (excuses, out) = self.run_britney() - #print('-------\nexcuses: %s\n-----' % excuses) - #print('-------\nout: %s\n-----' % out) - #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) - #subprocess.call(['bash', '-i'], cwd=self.data.path) - if considered: - self.assertIn('Valid candidate', excuses) - else: - self.assertIn('Not considered', excuses) - - if expect: - for re in expect: - self.assertRegexpMatches(excuses, re) - if no_expect: - for re in no_expect: - self.assertNotRegexpMatches(excuses, re) - - def test_no_request_for_uninstallable(self): - '''Does not request a test for an uninstallable package''' - - self.do_test( - # uninstallable unstable version - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'})], - 'green 1.1~beta RUNNING green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - 'green/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'], - # autopkgtest should not be triggered for uninstallable pkg - ['autopkgtest']) - - def test_request_for_installable_running(self): - '''Requests a test for an installable package, test still running''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta RUNNING green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_request_for_installable_first_fail(self): - '''Requests a test for an installable package. No history and first result is a failure''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta FAIL green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_request_for_installable_fail_regression(self): - '''Requests a test for an installable package, test fail''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta FAIL green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='green 1.0~beta PASS green 1.0~beta\n') - - def test_request_for_installable_pass(self): - '''Requests a test for an installable package, test pass''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta PASS green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_multi_rdepends_with_tests_running(self): - '''Multiple reverse dependencies with tests (still running)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 RUNNING green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_multi_rdepends_with_tests_fail_always(self): - '''Multiple reverse dependencies with tests (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_multi_rdepends_with_tests_fail_regression(self): - '''Multiple reverse dependencies with tests (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='darkgreen 1 PASS green 1\n') - - def test_multi_rdepends_with_tests_pass(self): - '''Multiple reverse dependencies with tests (pass)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 PASS green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_multi_rdepends_with_some_tests_running(self): - '''Multiple reverse dependencies with some tests (running)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 RUNNING green 2\n' - 'darkgreen 1 RUNNING green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_multi_rdepends_with_some_tests_fail_always(self): - '''Multiple reverse dependencies with some tests (fail)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_multi_rdepends_with_some_tests_fail_regression(self): - '''Multiple reverse dependencies with some tests (fail)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='darkgreen 1 PASS green 1\n') - - def test_multi_rdepends_with_some_tests_pass(self): - '''Multiple reverse dependencies with some tests (pass)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 PASS green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_binary_from_new_source_package_running(self): - '''building an existing binary for a new source package (running)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 RUNNING newgreen 2\n', - NOT_CONSIDERED, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_binary_from_new_source_package_fail_always(self): - '''building an existing binary for a new source package (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 FAIL newgreen 2\n', - VALID_CANDIDATE, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_binary_from_new_source_package_fail_regression(self): - '''building an existing binary for a new source package (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 FAIL newgreen 2\n', - NOT_CONSIDERED, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='darkgreen 1 PASS green 1\n') - - def test_binary_from_new_source_package_pass(self): - '''building an existing binary for a new source package (pass)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 PASS newgreen 2\n', - VALID_CANDIDATE, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_binary_from_new_source_package_uninst(self): - '''building an existing binary for a new source package (uninstallable)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6, nosuchpkg'})], - 'darkgreen 1 FAIL newgreen 2\n', - NOT_CONSIDERED, - [r'\bnewgreen\b.*\(- to .*>2<', - 'libgreen1/amd64 unsatisfiable Depends: nosuchpkg'], - # autopkgtest should not be triggered for uninstallable pkg - ['autopkgtest']) - - @unittest.expectedFailure - def test_result_from_older_version(self): - '''test result from older version than the uploaded one''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~alpha PASS green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - # it's not entirely clear what precisely it should say here - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_request_for_installable_fail_regression_promoted(self): - '''Requests a test for an installable package, test fail, is a regression. - - This test verifies a bug in britney where a package was promoted if latest test - appeared before previous result in history, only the last result in - alphabetic order was taken into account. For example: - A 1 FAIL B 1 - A 1 PASS A 1 - In this case results for A 1 didn't appear in the list of results - triggered by the upload of B 1 and B 1 was promoted - ''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 FAIL green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history="lightgreen 1 PASS lightgreen 1" - ) - - def test_history_always_passed(self): - '''All the results in history are PASS, and test passed - - ''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 PASS green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']], - history="lightgreen 1 PASS lightgreen 1" - ) - - def test_history_always_failed(self): - '''All the results in history are FAIL, test fails. not a regression. - - ''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 FAIL green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']], - history="lightgreen 1 FAIL lightgreen 1" - ) - - def test_history_regression(self): - '''All the results in history are PASS, test fails. Blocked. - - ''' - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 FAIL green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history="lightgreen 1 PASS lightgreen 1" - ) - - def shell(self): - # uninstallable unstable version - self.data.add('yellow', True, {'Version': '1.1~beta', - 'Depends': 'libc6 (>= 0.9), nosuchpkg'}) - - self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n', - 'purple 2 FAIL pink 3.0.~britney\n') - - print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) - subprocess.call(['bash', '-i'], cwd=self.data.path) - - if __name__ == '__main__': unittest.main() diff --git a/tests/test_boottest.py b/tests/test_boottest.py index 63a901c..6452fbb 100644 --- a/tests/test_boottest.py +++ b/tests/test_boottest.py @@ -11,6 +11,7 @@ import os import shutil import sys import tempfile +import fileinput import unittest @@ -236,10 +237,11 @@ args.func() def do_test(self, context, expect=None, no_expect=None): """Process the given package context and assert britney results.""" for (pkg, fields) in context: - self.data.add(pkg, True, fields) + self.data.add(pkg, True, fields, testsuite='autopkgtest') self.make_boottest() (excuses, out) = self.run_britney() # print('-------\nexcuses: %s\n-----' % excuses) + # print('-------\nout: %s\n-----' % out) if expect: for re in expect: self.assertRegexpMatches(excuses, re) @@ -329,14 +331,6 @@ args.func() r'
  • Boottest result: UNKNOWN STATUS \(Jenkins: .*\)', '
  • Not considered']) - def create_hint(self, username, content): - """Populates a hint file for the given 'username' with 'content'.""" - hints_path = os.path.join( - self.data.path, - 'data/{}-proposed/Hints/{}'.format(self.data.series, username)) - with open(hints_path, 'w') as fd: - fd.write(content) - def test_skipped_by_hints(self): # `Britney` allows boottests to be skipped by hinting the # corresponding source with 'force-skiptest'. The boottest @@ -420,37 +414,27 @@ args.func() def test_with_adt(self): # Boottest can run simultaneously with autopkgtest (adt). - # Enable ADT in britney configuration. - with open(self.britney_conf, 'r') as fp: - original_config = fp.read() - new_config = original_config.replace( - 'ADT_ENABLE = no', 'ADT_ENABLE = yes') - with open(self.britney_conf, 'w') as fp: - fp.write(new_config) + fake_amqp = os.path.join(self.data.path, 'amqp') - # Create a fake 'adt-britney' that reports a RUNNING job for - # the testing source ('purple_1.1'). - script_path = os.path.expanduser( - "~/auto-package-testing/jenkins/adt-britney") - os.makedirs(os.path.dirname(script_path)) - with open(script_path, 'w') as f: - f.write('''#!/bin/sh -e -mkdir -p ~/proposed-migration/autopkgtest/work -touch ~/proposed-migration/autopkgtest/work/adt.request.series -echo "purple 1.1 RUNNING purple 1.1" >> ~/proposed-migration/autopkgtest/work/adt.result.series''') - os.chmod(script_path, 0o755) + # Enable ADT in britney configuration. + for line in fileinput.input(self.britney_conf, inplace=True): + if 'ADT_ENABLE' in line: + print('ADT_ENABLE = yes') + elif 'ADT_AMQP' in line: + print('ADT_AMQP = file://%s' % fake_amqp) + else: + sys.stdout.write(line) # Britney blocks testing source promotion while ADT and boottest # are running. - self.data.add('purple', False, {'Version': '1.0'}) + self.data.add('purple', False, {'Version': '1.0'}, testsuite='autopkgtest') context = [ ('purple', {'Version': '1.1'}), ] self.do_test( context, [r'\bpurple\b.*>1.0<.* to .*>1.1<', - '
  • autopkgtest for purple 1.1: {}'.format( - boottest.BootTest.EXCUSE_LABELS['RUNNING']), + '
  • autopkgtest for purple 1.1: .*amd64.*in progress.*i386.*in progress', '
  • Boottest result: {}'.format( boottest.BootTest.EXCUSE_LABELS['RUNNING']), '
  • Not considered'])