From 2bf6eb5652aec2c30a00e06410f44fb52d9d4210 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 28 Jul 2015 11:04:34 +0200 Subject: [PATCH 1/4] Switch autopkgtest evaluation to cloud results Change AutoPackageTest.results() to evaluate the Swift results instead of the adt-britney ones. TODO: - Add more tests (like for adt-britney) - Drop triggering of adt-britney tests - Drop adt-britney tests (which fail now) --- autopkgtest.py | 49 ++++++++++++--------- britney.py | 62 ++++++--------------------- tests/test_autopkgtest.py | 90 ++++++++++++++++++++++++++++----------- 3 files changed, 107 insertions(+), 94 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index 91618e8..d197e00 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -39,7 +39,6 @@ from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION) adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") -ADT_PASS = ["PASS", "ALWAYSFAIL"] ADT_EXCUSES_LABELS = { "PASS": 'Pass', "ALWAYSFAIL": 'Always failed', @@ -647,23 +646,33 @@ class AutoPackageTest(object): def results(self, trigsrc, trigver): '''Return test results for triggering package - Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests - that got triggered by trigsrc/trigver. + Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING) + iterator for all package tests that got triggered by trigsrc/trigver. ''' - # deprecated results for old Jenkins/lp:auto-package-testing, will go - # away - for status, src, ver in self.pkgcauses[trigsrc][trigver]: - # Check for regression - if status == 'FAIL': - passed_once = False - for lver in self.pkglist[src]: - for trigsrc in self.pkglist[src][lver]['causes']: - for trigver, status \ - in self.pkglist[src][lver]['causes'][trigsrc]: - if status == 'PASS': - passed_once = True - if not passed_once: - status = 'ALWAYSFAIL' - else: - status = 'REGRESSION' - yield status, src, ver + for testsrc, testver in self.tests_for_source(trigsrc, trigver): + passed = True + arch_status = {} + for arch in self.britney.options.adt_arches.split(): + try: + if self.test_results[testsrc][arch][1][testver][0]: + arch_status[arch] = 'PASS' + else: + if self.test_results[testsrc][arch][2]: + arch_status[arch] = 'REGRESSION' + passed = False + else: + arch_status[arch] = 'ALWAYSFAIL' + except KeyError: + try: + self.pending_tests[testsrc][testver][arch] + arch_status[arch] = 'RUNNING' + passed = False + except KeyError: + # neither done nor pending -> exclusion, or disabled + continue + + # disabled or ignored? + if not arch_status: + continue + + yield (passed, testsrc, testver, arch_status) diff --git a/britney.py b/britney.py index 23455c5..46531f8 100755 --- a/britney.py +++ b/britney.py @@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes, from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) -from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash +from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash from boottest import BootTest @@ -1849,28 +1849,21 @@ class Britney(object): if not self.options.dry_run: autopkgtest.submit() autopkgtest.collect(autopkgtest_packages) - jenkins_public = "https://jenkins.qa.ubuntu.com/job" - jenkins_private = ( - "http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" % - self.options.series.title()) cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s" + adtpass = True for e in autopkgtest_excuses: - adtpass = True - for status, adtsrc, adtver in autopkgtest.results( + for adtpass, adtsrc, adtver, arch_status in autopkgtest.results( e.name, e.ver[1]): - public_url = "%s/%s-adt-%s/lastBuild" % ( - jenkins_public, self.options.series, - adtsrc.replace("+", "-")) - private_url = "%s/%s-adt-%s/lastBuild" % ( - jenkins_private, self.options.series, - adtsrc.replace("+", "-")) - adt_label = ADT_EXCUSES_LABELS.get(status, status) - e.addhtml( - "autopkgtest for %s %s: %s (Jenkins: " - "public, " - "private)" % - (adtsrc, adtver, adt_label, public_url, private_url)) - if status not in ADT_PASS: + archmsg = [] + for arch in sorted(arch_status): + url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc, + 'r': self.options.series, 'a': arch} + archmsg.append('%s: %s' % + (url, arch, ADT_EXCUSES_LABELS[arch_status[arch]])) + e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg))) + + # hints can override failures + if not adtpass: hints = self.hints.search( 'force-badtest', package=adtsrc) hints.extend( @@ -1882,34 +1875,7 @@ class Britney(object): e.addhtml( "Should wait for %s %s test, but forced by " "%s" % (adtsrc, adtver, forces[0].user)) - else: - adtpass = False - - # temporary: also show results from cloud based tests, - # until that becomes the primary mechanism - for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]): - msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver) - archmsg = [] - for arch in self.options.adt_arches.split(): - url = cloud_url % {'h': srchash(testsrc), 's': testsrc, - 'r': self.options.series, 'a': arch} - try: - r = autopkgtest.test_results[testsrc][arch][1][testver][0] - status = r and 'PASS' or 'REGRESSION' - except KeyError: - try: - autopkgtest.pending_tests[testsrc][testver][arch] - status = 'RUNNING' - except KeyError: - # neither done nor pending -> exclusion, or disabled - continue - - archmsg.append('%s: %s' % - (url, arch, ADT_EXCUSES_LABELS[status])) - - if archmsg: - e.addhtml(msg + ', '.join(archmsg)) - # end of temporary code + adtpass = True if not adtpass and e.is_valid: hints = self.hints.search('force-skiptest', package=e.name) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 8d04ede..95376ef 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -125,9 +125,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -153,7 +151,7 @@ lightgreen 1 i386 green 2 self.assertEqual(self.pending_requests, expected_pending) # if we run britney again this should *not* trigger any new tests - self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<']) + self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<']) self.assertEqual(self.amqp_requests, set()) # but the set of pending tests doesn't change self.assertEqual(self.pending_requests, expected_pending) @@ -164,9 +162,7 @@ lightgreen 1 i386 green 2 # first run requests tests and marks them as pending self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -233,9 +229,7 @@ lightgreen 1 i386 green 2 # first run requests tests and marks them as pending self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', @@ -252,11 +246,9 @@ lightgreen 1 i386 green 2 out = self.do_test( [], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', - r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass', r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress', r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*Pass']) @@ -267,15 +259,67 @@ lightgreen 1 i386 green 2 self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests) self.assertIn('lightgreen 1 i386 green 2', self.pending_requests) + def test_multi_rdepends_with_tests_regression(self): + '''Multiple reverse dependencies with tests (regression)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + + def test_multi_rdepends_with_tests_always_failed(self): + '''Multiple reverse dependencies with tests (always failed)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (4, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (4, 'green 2'), + }}) + + out = self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual(self.pending_requests, '') + # not expecting any failures to retrieve from swift + self.assertNotIn('Failure', out, out) + def test_package_pair_running(self): '''Two packages in unstable that need to go in together (running)''' self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'\blightgreen\b.*>1 to .*>2<']) @@ -313,9 +357,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'], ['in progress']) @@ -341,9 +383,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression', r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression']) @@ -392,9 +432,7 @@ lightgreen 2 i386 lightgreen 2 self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'), ('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')], - # FIXME: while we only submit requests through AMQP, but don't consider - # their results, we don't expect this to hold back stuff. - VALID_CANDIDATE, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', r'\blightgreen\b.*>1 to .*>2<', r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', From fc141215e949f2086a842113b5c6289059d48cc4 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 28 Jul 2015 11:32:03 +0200 Subject: [PATCH 2/4] Add more autopkgtest tests, drop TestAdtBritney Drop TestAdtBritney tests which now fail as we switched results evaluation to swift. Port relevant tests to TestAutoPkgTest. --- tests/test_autopkgtest.py | 529 ++++++-------------------------------- 1 file changed, 82 insertions(+), 447 deletions(-) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 95376ef..c4f28b3 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -7,10 +7,8 @@ # (at your option) any later version. import apt_pkg -import operator import os import sys -import subprocess import fileinput import unittest import json @@ -18,7 +16,6 @@ import json PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, PROJECT_DIR) -from autopkgtest import ADT_EXCUSES_LABELS from tests import TestBase, mock_swift NOT_CONSIDERED = False @@ -89,7 +86,7 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) (excuses, out) = self.run_britney() self.swift.stop() - #print('-------\nexcuses: %s\n-----' % excuses) + # print('-------\nexcuses: %s\n-----' % excuses) if 'SHOW_OUTPUT' in os.environ: print('-------\nout: %s\n-----' % out) if considered: @@ -120,6 +117,21 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) return out + def test_no_request_for_uninstallable(self): + '''Does not request a test for an uninstallable package''' + + self.do_test( + # uninstallable unstable version + [('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\blightgreen\b.*>1 to .*>1.1~beta<', + 'lightgreen/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'], + # autopkgtest should not be triggered for uninstallable pkg + ['autopkgtest']) + + self.assertEqual(self.pending_requests, '') + self.assertEqual(self.amqp_requests, set()) + def test_multi_rdepends_with_tests_all_running(self): '''Multiple reverse dependencies with tests (all running)''' @@ -344,6 +356,72 @@ lightgreen 2 i386 lightgreen 2 ''' self.assertEqual(self.pending_requests, expected_pending) + def test_binary_from_new_source_package_running(self): + '''building an existing binary for a new source package (running)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bnewgreen\b.*\(- to .*>2<', + r'autopkgtest for newgreen 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress']) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen', + 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) + expected_pending = '''darkgreen 1 amd64 newgreen 2 +darkgreen 1 i386 newgreen 2 +lightgreen 1 amd64 newgreen 2 +lightgreen 1 i386 newgreen 2 +newgreen 2 amd64 newgreen 2 +newgreen 2 i386 newgreen 2 +''' + self.assertEqual(self.pending_requests, expected_pending) + + def test_binary_from_new_source_package_pass(self): + '''building an existing binary for a new source package (pass)''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/n/newgreen/20150101_100200@': (0, 'newgreen 2'), + 'series/amd64/n/newgreen/20150101_100201@': (0, 'newgreen 2'), + }}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bnewgreen\b.*\(- to .*>2<', + r'autopkgtest for newgreen 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass']) + + self.assertEqual( + self.amqp_requests, + set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen', + 'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen', + 'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen'])) + self.assertEqual(self.pending_requests, '') + + def test_result_from_older_version(self): + '''test result from older version than the uploaded one''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + }}) + + self.do_test( + [('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bdarkgreen\b.*>1 to .*>2<', + r'autopkgtest for darkgreen 2: .*amd64.*in progress.*i386.*in progress']) + def test_tmpfail(self): '''tmpfail result is considered a failure''' @@ -487,448 +565,5 @@ lightgreen 2 i386 lightgreen 2 self.assertEqual(self.pending_requests, None) -class TestAdtBritney(TestBase): - '''Legacy adt-britney/lp:auto-package-testing interface''' - - def setUp(self): - super(TestAdtBritney, self).setUp() - - # Mofify configuration according to the test context. - with open(self.britney_conf, 'r') as fp: - original_config = fp.read() - # Disable boottests. - new_config = original_config.replace( - 'BOOTTEST_ENABLE = yes', 'BOOTTEST_ENABLE = no') - with open(self.britney_conf, 'w') as fp: - fp.write(new_config) - - # fake adt-britney script - self.adt_britney = os.path.join( - self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney') - os.makedirs(os.path.dirname(self.adt_britney)) - - with open(self.adt_britney, 'w') as f: - f.write('''#!/bin/sh -e -echo "$@" >> /%s/adt-britney.log ''' % self.data.path) - os.chmod(self.adt_britney, 0o755) - - # add a bunch of packages to testing to avoid repetition - self.data.add('libc6', False) - self.data.add('libgreen1', False, {'Source': 'green', - 'Depends': 'libc6 (>= 0.9)'}) - self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', - 'Conflicts': 'blue'}) - self.data.add('lightgreen', False, {'Depends': 'libgreen1'}) - self.data.add('darkgreen', False, {'Depends': 'libgreen1'}) - self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', - 'Conflicts': 'green'}) - self.data.add('justdata', False, {'Architecture': 'all'}) - - def __merge_records(self, results, history=""): - '''Merges a list of results with records in history. - - This function merges results from a collect with records already in - history and sort records by version/name of causes and version/name of - source packages with tests. This should be done in the fake - adt-britney but it is more convenient to just pass a static list of - records and make adt-britney just return this list. - ''' - - if history is None: - history = "" - records = [x.split() for x in (results.strip() + '\n' + - history.strip()).split('\n') if x] - - records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(4)) - records.sort(key=operator.itemgetter(3)) - records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(1)) - records.sort() - - return "\n".join([' '.join(x) for x in records]) - - def make_adt_britney(self, request, history=""): - with open(self.adt_britney, 'w') as f: - f.write('''#!%(py)s -import argparse, shutil,sys - -def request(): - if args.req: - shutil.copy(args.req, '%(path)s/adt-britney.requestarg') - with open(args.output, 'w') as f: - f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW').replace('RUNNING', 'NEW')) - -def submit(): - with open(args.req, 'w') as f: - f.write("""%(rq)s""".replace('PASS', 'RUNNING'). - replace('FAIL', 'RUNNING')) - -def collect(): - with open(args.output, 'w') as f: - f.write("""%(res)s""") - -p = argparse.ArgumentParser() -p.add_argument('-c', '--config') -p.add_argument('-a', '--arch') -p.add_argument('-r', '--release') -p.add_argument('-P', '--use-proposed', action='store_true') -p.add_argument('-d', '--debug', action='store_true') -p.add_argument('-U', '--no-update', action='store_true') -sp = p.add_subparsers() - -prequest = sp.add_parser('request') -prequest.add_argument('-O', '--output') -prequest.add_argument('req', nargs='?') -prequest.set_defaults(func=request) - -psubmit = sp.add_parser('submit') -psubmit.add_argument('req') -psubmit.set_defaults(func=submit) - -pcollect = sp.add_parser('collect') -pcollect.add_argument('-O', '--output') -pcollect.add_argument('-n', '--new-only', action='store_true', default=False) -pcollect.set_defaults(func=collect) - -args = p.parse_args() -args.func() - ''' % {'py': sys.executable, 'path': self.data.path, - 'rq': request, - 'res': self.__merge_records(request, history)}) - - def do_test(self, unstable_add, adt_request, considered, expect=None, - no_expect=None, history=""): - for (pkg, fields) in unstable_add: - self.data.add(pkg, True, fields) - - self.make_adt_britney(adt_request, history) - - (excuses, out) = self.run_britney() - #print('-------\nexcuses: %s\n-----' % excuses) - #print('-------\nout: %s\n-----' % out) - #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) - #subprocess.call(['bash', '-i'], cwd=self.data.path) - if considered: - self.assertIn('Valid candidate', excuses) - else: - self.assertIn('Not considered', excuses) - - if expect: - for re in expect: - self.assertRegexpMatches(excuses, re) - if no_expect: - for re in no_expect: - self.assertNotRegexpMatches(excuses, re) - - def test_no_request_for_uninstallable(self): - '''Does not request a test for an uninstallable package''' - - self.do_test( - # uninstallable unstable version - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'})], - 'green 1.1~beta RUNNING green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - 'green/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'], - # autopkgtest should not be triggered for uninstallable pkg - ['autopkgtest']) - - def test_request_for_installable_running(self): - '''Requests a test for an installable package, test still running''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta RUNNING green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_request_for_installable_first_fail(self): - '''Requests a test for an installable package. No history and first result is a failure''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta FAIL green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_request_for_installable_fail_regression(self): - '''Requests a test for an installable package, test fail''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta FAIL green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='green 1.0~beta PASS green 1.0~beta\n') - - def test_request_for_installable_pass(self): - '''Requests a test for an installable package, test pass''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~beta PASS green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_multi_rdepends_with_tests_running(self): - '''Multiple reverse dependencies with tests (still running)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 RUNNING green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_multi_rdepends_with_tests_fail_always(self): - '''Multiple reverse dependencies with tests (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_multi_rdepends_with_tests_fail_regression(self): - '''Multiple reverse dependencies with tests (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='darkgreen 1 PASS green 1\n') - - def test_multi_rdepends_with_tests_pass(self): - '''Multiple reverse dependencies with tests (pass)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 PASS green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_multi_rdepends_with_some_tests_running(self): - '''Multiple reverse dependencies with some tests (running)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 RUNNING green 2\n' - 'darkgreen 1 RUNNING green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_multi_rdepends_with_some_tests_fail_always(self): - '''Multiple reverse dependencies with some tests (fail)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_multi_rdepends_with_some_tests_fail_regression(self): - '''Multiple reverse dependencies with some tests (fail)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='darkgreen 1 PASS green 1\n') - - def test_multi_rdepends_with_some_tests_pass(self): - '''Multiple reverse dependencies with some tests (pass)''' - - # add a third reverse dependency to libgreen1 which does not have a test - self.data.add('mint', False, {'Depends': 'libgreen1'}) - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 PASS green 2\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_binary_from_new_source_package_running(self): - '''building an existing binary for a new source package (running)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 RUNNING newgreen 2\n', - NOT_CONSIDERED, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_binary_from_new_source_package_fail_always(self): - '''building an existing binary for a new source package (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 FAIL newgreen 2\n', - VALID_CANDIDATE, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']]) - - def test_binary_from_new_source_package_fail_regression(self): - '''building an existing binary for a new source package (fail)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 FAIL newgreen 2\n', - NOT_CONSIDERED, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history='darkgreen 1 PASS green 1\n') - - def test_binary_from_new_source_package_pass(self): - '''building an existing binary for a new source package (pass)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS newgreen 2\n' - 'darkgreen 1 PASS newgreen 2\n', - VALID_CANDIDATE, - [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'], - '
  • autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']]) - - def test_binary_from_new_source_package_uninst(self): - '''building an existing binary for a new source package (uninstallable)''' - - self.do_test( - [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6, nosuchpkg'})], - 'darkgreen 1 FAIL newgreen 2\n', - NOT_CONSIDERED, - [r'\bnewgreen\b.*\(- to .*>2<', - 'libgreen1/amd64 unsatisfiable Depends: nosuchpkg'], - # autopkgtest should not be triggered for uninstallable pkg - ['autopkgtest']) - - @unittest.expectedFailure - def test_result_from_older_version(self): - '''test result from older version than the uploaded one''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'green 1.1~alpha PASS green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - # it's not entirely clear what precisely it should say here - '
  • autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']]) - - def test_request_for_installable_fail_regression_promoted(self): - '''Requests a test for an installable package, test fail, is a regression. - - This test verifies a bug in britney where a package was promoted if latest test - appeared before previous result in history, only the last result in - alphabetic order was taken into account. For example: - A 1 FAIL B 1 - A 1 PASS A 1 - In this case results for A 1 didn't appear in the list of results - triggered by the upload of B 1 and B 1 was promoted - ''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 FAIL green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history="lightgreen 1 PASS lightgreen 1" - ) - - def test_history_always_passed(self): - '''All the results in history are PASS, and test passed - - ''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 PASS green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']], - history="lightgreen 1 PASS lightgreen 1" - ) - - def test_history_always_failed(self): - '''All the results in history are FAIL, test fails. not a regression. - - ''' - - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 FAIL green 1.1~beta\n', - VALID_CANDIDATE, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']], - history="lightgreen 1 FAIL lightgreen 1" - ) - - def test_history_regression(self): - '''All the results in history are PASS, test fails. Blocked. - - ''' - self.do_test( - [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], - 'lightgreen 1 FAIL green 1.1~beta\n', - NOT_CONSIDERED, - [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']], - history="lightgreen 1 PASS lightgreen 1" - ) - - def shell(self): - # uninstallable unstable version - self.data.add('yellow', True, {'Version': '1.1~beta', - 'Depends': 'libc6 (>= 0.9), nosuchpkg'}) - - self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n', - 'purple 2 FAIL pink 3.0.~britney\n') - - print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) - subprocess.call(['bash', '-i'], cwd=self.data.path) - - if __name__ == '__main__': unittest.main() From 269b156def86783322c323f84f0d252992413d40 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 28 Jul 2015 11:46:17 +0200 Subject: [PATCH 3/4] Drop obsolete adt-britney autopkgtest code Now that we look at autopkgtest results from swift we can drop the adt-britney/lp:auto-package-testing code from autopkgtest.py. Note that we still need it for boottest.py. --- autopkgtest.py | 180 +------------------------------------- tests/test_autopkgtest.py | 10 --- 2 files changed, 4 insertions(+), 186 deletions(-) diff --git a/autopkgtest.py b/autopkgtest.py index d197e00..2d15ec4 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (C) 2013 Canonical Ltd. -# Author: Colin Watson -# Partly based on code in auto-package-testing by -# Jean-Baptiste Lallement +# Authors: +# Colin Watson +# Jean-Baptiste Lallement +# Martin Pitt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,12 +18,7 @@ from __future__ import print_function -from collections import defaultdict -from contextlib import closing import os -import subprocess -import tempfile -from textwrap import dedent import time import json import tarfile @@ -37,8 +33,6 @@ import kombu from consts import (AUTOPKGTEST, BINARIES, RDEPENDS, SOURCE, VERSION) -adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") - ADT_EXCUSES_LABELS = { "PASS": 'Pass', "ALWAYSFAIL": 'Always failed', @@ -83,8 +77,6 @@ class AutoPackageTest(object): self.distribution = distribution self.series = series self.debug = debug - self.read() - self.rc_path = None # for adt-britney, obsolete self.test_state_dir = os.path.join(britney.options.unstable, 'autopkgtest') # map of requested tests from request() @@ -376,92 +368,6 @@ class AutoPackageTest(object): result.add((src, arch)) return result - # - # obsolete adt-britney helpers - # - - def _ensure_rc_file(self): - if self.rc_path: - return - self.rc_path = os.path.expanduser( - "~/proposed-migration/autopkgtest/rc.%s" % self.series) - with open(self.rc_path, "w") as rc_file: - home = os.path.expanduser("~") - print(dedent("""\ - release: %s - aptroot: ~/.chdist/%s-proposed-amd64/ - apturi: file:%s/mirror/%s - components: main restricted universe multiverse - rsync_host: rsync://tachash.ubuntu-ci/adt/ - datadir: ~/proposed-migration/autopkgtest/data""" % - (self.series, self.series, home, self.distribution)), - file=rc_file) - - @property - def _request_path(self): - return os.path.expanduser( - "~/proposed-migration/autopkgtest/work/adt.request.%s" % - self.series) - - @property - def _result_path(self): - return os.path.expanduser( - "~/proposed-migration/autopkgtest/work/adt.result.%s" % - self.series) - - def _parse(self, path): - if os.path.exists(path): - with open(path) as f: - for line in f: - line = line.strip() - if line.startswith("Suite:") or line.startswith("Date:"): - continue - linebits = line.split() - if len(linebits) < 2: - print("W: Invalid line format: '%s', skipped" % line) - continue - yield linebits - - def read(self): - '''Loads a list of results - - This function loads a list of results returned by __parse() and builds - 2 lists: - - a list of source package/version with all the causes that - triggered a test and the result of the test for this trigger. - - a list of packages/version that triggered a test with the source - package/version and result triggered by this package. - These lists will be used in result() called from britney.py to generate - excuses and now which uploads passed, caused regression or which tests - have always been failing - ''' - self.pkglist = defaultdict(dict) - self.pkgcauses = defaultdict(lambda: defaultdict(list)) - for linebits in self._parse(self._result_path): - (src, ver, status) = linebits[:3] - - if not (src in self.pkglist and ver in self.pkglist[src]): - self.pkglist[src][ver] = { - "status": status, - "causes": {} - } - - i = iter(linebits[3:]) - for trigsrc, trigver in zip(i, i): - self.pkglist[src][ver]['causes'].setdefault( - trigsrc, []).append((trigver, status)) - self.pkgcauses[trigsrc][trigver].append((status, src, ver)) - - def _adt_britney(self, *args): - command = [ - adt_britney, - "-c", self.rc_path, "-r", self.series, "-PU", - ] - if self.debug: - command.append("-d") - command.extend(args) - subprocess.check_call(command) - # # Public API # @@ -485,60 +391,6 @@ class AutoPackageTest(object): self.log_verbose('Requesting %s/%s/%s autopkgtest to verify %s' % (src, ver, arch, ', '.join(['%s/%s' % i for i in triggers]))) - # deprecated requests for old Jenkins/lp:auto-package-testing, will go - # away - - self._ensure_rc_file() - request_path = self._request_path - if os.path.exists(request_path): - os.unlink(request_path) - with closing(tempfile.NamedTemporaryFile(mode="w")) as request_file: - for src, ver in packages: - if src in self.pkglist and ver in self.pkglist[src]: - continue - print("%s %s" % (src, ver), file=request_file) - request_file.flush() - self._adt_britney("request", "-O", request_path, request_file.name) - - # Remove packages that have been identified as invalid candidates for - # testing from the request file i.e run_autopkgtest = False - with open(request_path, 'r') as request_file: - lines = request_file.readlines() - with open(request_path, 'w') as request_file: - for line in lines: - src = line.split()[0] - if src not in excludes: - request_file.write(line) - else: - if self.britney.options.verbose: - self.log_verbose("Requested autopkgtest for %s but " - "run_autopkgtest set to False" % src) - - for linebits in self._parse(request_path): - # Make sure that there's an entry in pkgcauses for each new - # request, so that results() gives useful information without - # relying on the submit/collect cycle. This improves behaviour - # in dry-run mode. - src = linebits.pop(0) - ver = linebits.pop(0) - if self.britney.options.verbose: - self.log_verbose("Requested autopkgtest for %s_%s (%s)" % - (src, ver, " ".join(linebits))) - try: - status = linebits.pop(0).upper() - while True: - trigsrc = linebits.pop(0) - trigver = linebits.pop(0) - for status, csrc, cver in self.pkgcauses[trigsrc][trigver]: - if csrc == trigsrc and cver == trigver: - break - else: - self.pkgcauses[trigsrc][trigver].append( - (status, src, ver)) - except IndexError: - # End of the list - pass - def submit(self): # send AMQP requests for new test requests # TODO: Once we support version constraints in AMQP requests, add them @@ -580,13 +432,6 @@ class AutoPackageTest(object): # mark them as pending now self.update_pending_tests() - # deprecated requests for old Jenkins/lp:auto-package-testing, will go - # away - self._ensure_rc_file() - request_path = self._request_path - if os.path.exists(request_path): - self._adt_britney("submit", request_path) - def collect(self, packages): # fetch results from swift try: @@ -626,23 +471,6 @@ class AutoPackageTest(object): # new results remove pending requests, update the on-disk cache self.update_pending_tests() - # deprecated results for old Jenkins/lp:auto-package-testing, will go - # away - self._ensure_rc_file() - result_path = self._result_path - self._adt_britney("collect", "-O", result_path) - self.read() - if self.britney.options.verbose: - for src in sorted(self.pkglist): - for ver in sorted(self.pkglist[src], - cmp=apt_pkg.version_compare): - for trigsrc in sorted(self.pkglist[src][ver]['causes']): - for trigver, status \ - in self.pkglist[src][ver]['causes'][trigsrc]: - self.log_verbose("Collected autopkgtest status " - "for %s_%s/%s_%s: " "%s" % - (src, ver, trigsrc, trigver, status)) - def results(self, trigsrc, trigver): '''Return test results for triggering package diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index c4f28b3..fdef494 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -43,16 +43,6 @@ class TestAutoPkgTest(TestBase): else: sys.stdout.write(line) - # fake adt-britney script; necessary until we drop that code - self.adt_britney = os.path.join( - self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney') - os.makedirs(os.path.dirname(self.adt_britney)) - with open(self.adt_britney, 'w') as f: - f.write('''#!/bin/sh -e -touch $HOME/proposed-migration/autopkgtest/work/adt.request.series -echo "$@" >> /%s/adt-britney.log ''' % self.data.path) - os.chmod(self.adt_britney, 0o755) - # add a bunch of packages to testing to avoid repetition self.data.add('libc6', False) self.data.add('libgreen1', False, {'Source': 'green', From 1d4eab951e92f624fff159dc9d604cc3949b64db Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 31 Jul 2015 09:50:38 +0200 Subject: [PATCH 4/4] Add autopkgtest tests for hints --- tests/__init__.py | 8 ++++ tests/test_autopkgtest.py | 78 +++++++++++++++++++++++++++++++++++++++ tests/test_boottest.py | 8 ---- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index c682229..c2f320b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -172,3 +172,11 @@ class TestBase(unittest.TestCase): excuses = f.read() return (excuses, out) + + def create_hint(self, username, content): + '''Create a hint file for the given username and content''' + + hints_path = os.path.join( + self.data.path, 'data', self.data.series + '-proposed', 'Hints', username) + with open(hints_path, 'w') as fd: + fd.write(content) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 8cf626c..0b7c9f3 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -345,6 +345,84 @@ lightgreen 1 i386 green 2 # not expecting any failures to retrieve from swift self.assertNotIn('Failure', out, out) + def test_hint_force_badtest(self): + '''force-badtest hint''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass', + r'Should wait for lightgreen 1 test, but forced by pitti']) + + def test_hint_force_badtest_different_version(self): + '''force-badtest hint with non-matching version''' + + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100200@': (0, 'green 2'), + }}) + + self.create_hint('pitti', 'force-badtest lightgreen/0.1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'], + ['Should wait']) + + def test_hint_force_skiptest(self): + '''force-skiptest hint''' + + self.create_hint('pitti', 'force-skiptest green/2') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress', + r'Should wait for.*tests.*green 2.*forced by pitti']) + + def test_hint_force_skiptest_different_version(self): + '''force-skiptest hint with non-matching version''' + + self.create_hint('pitti', 'force-skiptest green/1') + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress', + r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'], + ['Should wait']) + def test_package_pair_running(self): '''Two packages in unstable that need to go in together (running)''' diff --git a/tests/test_boottest.py b/tests/test_boottest.py index 3d48f72..6452fbb 100644 --- a/tests/test_boottest.py +++ b/tests/test_boottest.py @@ -331,14 +331,6 @@ args.func() r'
  • Boottest result: UNKNOWN STATUS \(Jenkins: .*\)', '
  • Not considered']) - def create_hint(self, username, content): - """Populates a hint file for the given 'username' with 'content'.""" - hints_path = os.path.join( - self.data.path, - 'data/{}-proposed/Hints/{}'.format(self.data.series, username)) - with open(hints_path, 'w') as fd: - fd.write(content) - def test_skipped_by_hints(self): # `Britney` allows boottests to be skipped by hinting the # corresponding source with 'force-skiptest'. The boottest