From acd4de78682a331b2c57a61d60c4da28ffba68ab Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Mon, 5 May 2014 18:08:40 +0200 Subject: [PATCH] tests/autopkgtest.py: Fixed testsuite and added tests to match new labels and rules for regression/always failing tests --- tests/autopkgtest.py | 144 +++++++++++++++++++++++++++++++++---------- 1 file changed, 112 insertions(+), 32 deletions(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 26f7fb5..46d6924 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -12,7 +12,10 @@ import os import sys import subprocess import unittest +import apt_pkg +import operator +apt_pkg.init() architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'] my_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -20,6 +23,13 @@ my_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) NOT_CONSIDERED = False VALID_CANDIDATE = True +ADT_EXCUSES_LABELS = { + "PASS": 'Pass', + "ALWAYSFAIL": 'Always failed', + "REGRESSION": 'Regression', + "RUNNING": 'Test in progress', +} + class TestData: def __init__(self): @@ -157,7 +167,20 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path) def tearDown(self): del self.data - def make_adt_britney(self, request): + def __merge_records(self, results, history=""): + if history is None: + history = "" + records = [x.split() for x in (results.strip() + '\n' + + history.strip()).split('\n') if x] + + records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(4)) + records.sort(key=operator.itemgetter(3)) + records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(1)) + records.sort(key=operator.itemgetter(1)) + + return "\n".join([' '.join(x) for x in records]) + + def make_adt_britney(self, request, history=""): with open(self.adt_britney, 'w') as f: f.write('''#!%(py)s import argparse, shutil,sys @@ -175,7 +198,7 @@ def submit(): def collect(): with open(args.output, 'w') as f: - f.write("""%(rq)s""") + f.write("""%(res)s""") p = argparse.ArgumentParser() p.add_argument('-c', '--config') @@ -202,7 +225,9 @@ pcollect.set_defaults(func=collect) args = p.parse_args() args.func() -''' % {'py': sys.executable, 'path': self.data.path, 'rq': request}) + ''' % {'py': sys.executable, 'path': self.data.path, 'rq': + request, 'res': self.__merge_records(request, + history)}) def run_britney(self, args=[]): '''Run britney. @@ -245,9 +270,19 @@ args.func() 'green 1.1~beta RUNNING green 1.1~beta\n', NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: RUNNING']) + '
  • autopkgtest for green 1.1~beta: {}'.format(ADT_EXCUSES_LABELS['RUNNING'])]) - def test_request_for_installable_fail(self): + def test_request_for_installable_fail_always(self): + '''Requests a test for an installable package, test fail''' + + self.do_test( + [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], + 'green 1.1~beta FAIL green 1.1~beta\n', + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>1.1~beta<', + '
  • autopkgtest for green 1.1~beta: {}'.format(ADT_EXCUSES_LABELS['ALWAYSFAIL'])]) + + def test_request_for_installable_fail_regression(self): '''Requests a test for an installable package, test fail''' self.do_test( @@ -255,7 +290,8 @@ args.func() 'green 1.1~beta FAIL green 1.1~beta\n', NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: FAIL']) + '
  • autopkgtest for green 1.1~beta: {}'.format(ADT_EXCUSES_LABELS['REGRESSION'])], + history='green 1.0~beta PASS green 1.0~beta\n') def test_request_for_installable_pass(self): '''Requests a test for an installable package, test pass''' @@ -265,7 +301,7 @@ args.func() 'green 1.1~beta PASS green 1.1~beta\n', VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>1.1~beta<', - '
  • autopkgtest for green 1.1~beta: PASS']) + '
  • autopkgtest for green 1.1~beta: {}'.format(ADT_EXCUSES_LABELS['PASS'])]) def test_multi_rdepends_with_tests_running(self): '''Multiple reverse dependencies with tests (still running)''' @@ -276,10 +312,22 @@ args.func() 'darkgreen 1 RUNNING green 2\n', NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: RUNNING']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['RUNNING'])]) + + def test_multi_rdepends_with_tests_fail_always(self): + '''Multiple reverse dependencies with tests (fail)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 FAIL green 2\n', + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['ALWAYSFAIL'])]) - def test_multi_rdepends_with_tests_fail(self): + def test_multi_rdepends_with_tests_fail_regression(self): '''Multiple reverse dependencies with tests (fail)''' self.do_test( @@ -288,8 +336,9 @@ args.func() 'darkgreen 1 FAIL green 2\n', NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: FAIL']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['REGRESSION'])], + history='darkgreen 1 PASS green 1\n') def test_multi_rdepends_with_tests_pass(self): '''Multiple reverse dependencies with tests (pass)''' @@ -300,8 +349,8 @@ args.func() 'darkgreen 1 PASS green 2\n', VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: PASS']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS'])]) def test_multi_rdepends_with_some_tests_running(self): '''Multiple reverse dependencies with some tests (running)''' @@ -315,10 +364,25 @@ args.func() 'darkgreen 1 RUNNING green 2\n', NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: RUNNING', - '
  • autopkgtest for darkgreen 1: RUNNING']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['RUNNING']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['RUNNING'])]) - def test_multi_rdepends_with_some_tests_fail(self): + def test_multi_rdepends_with_some_tests_fail_always(self): + '''Multiple reverse dependencies with some tests (fail)''' + + # add a third reverse dependency to libgreen1 which does not have a test + self.data.add('mint', False, {'Depends': 'libgreen1'}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 FAIL green 2\n', + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['ALWAYSFAIL'])]) + + def test_multi_rdepends_with_some_tests_fail_regression(self): '''Multiple reverse dependencies with some tests (fail)''' # add a third reverse dependency to libgreen1 which does not have a test @@ -330,8 +394,9 @@ args.func() 'darkgreen 1 FAIL green 2\n', NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: FAIL']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['REGRESSION'])], + history='darkgreen 1 PASS green 1\n') def test_multi_rdepends_with_some_tests_pass(self): '''Multiple reverse dependencies with some tests (pass)''' @@ -345,8 +410,8 @@ args.func() 'darkgreen 1 PASS green 2\n', VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: PASS']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS'])]) def test_binary_from_new_source_package_running(self): '''building an existing binary for a new source package (running)''' @@ -357,10 +422,22 @@ args.func() 'darkgreen 1 RUNNING newgreen 2\n', NOT_CONSIDERED, [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: RUNNING']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['RUNNING'])]) + + def test_binary_from_new_source_package_fail_always(self): + '''building an existing binary for a new source package (fail)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], + 'lightgreen 1 PASS newgreen 2\n' + 'darkgreen 1 FAIL newgreen 2\n', + VALID_CANDIDATE, + [r'\bnewgreen\b.*\(- to .*>2<', + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['ALWAYSFAIL'])]) - def test_binary_from_new_source_package_fail(self): + def test_binary_from_new_source_package_fail_regression(self): '''building an existing binary for a new source package (fail)''' self.do_test( @@ -369,8 +446,9 @@ args.func() 'darkgreen 1 FAIL newgreen 2\n', NOT_CONSIDERED, [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: FAIL']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['REGRESSION'])], + history='darkgreen 1 PASS green 1\n') def test_binary_from_new_source_package_pass(self): '''building an existing binary for a new source package (pass)''' @@ -381,8 +459,8 @@ args.func() 'darkgreen 1 PASS newgreen 2\n', VALID_CANDIDATE, [r'\bnewgreen\b.*\(- to .*>2<', - '
  • autopkgtest for lightgreen 1: PASS', - '
  • autopkgtest for darkgreen 1: PASS']) + '
  • autopkgtest for lightgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS']), + '
  • autopkgtest for darkgreen 1: {}'.format(ADT_EXCUSES_LABELS['PASS'])]) def test_binary_from_new_source_package_uninst(self): '''building an existing binary for a new source package (uninstallable)''' @@ -405,14 +483,14 @@ args.func() NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', # it's not entirely clear what precisely it should say here - '
  • autopkgtest for green 1.1~beta: RUNNING']) + '
  • autopkgtest for green 1.1~beta: {}'.format(ADT_EXCUSES_LABELS['RUNNING'])]) def do_test(self, unstable_add, adt_request, considered, expect=None, - no_expect=None): + no_expect=None, history=""): for (pkg, fields) in unstable_add: self.data.add(pkg, True, fields) - self.make_adt_britney(adt_request) + self.make_adt_britney(adt_request, history) (excuses, out) = self.run_britney() #print('-------\nexcuses: %s\n-----' % excuses) @@ -436,7 +514,9 @@ args.func() self.data.add('yellow', True, {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), nosuchpkg'}) - self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n') + self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n', "") + #self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n', + # 'purple 2 FAIL pink 3.0.~britney\n') print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) subprocess.call(['bash', '-i'], cwd=self.data.path)