From e1dc047c7e92722a5b29b009a7da0eba6af1de4d Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 18 Feb 2014 16:59:59 -0800 Subject: [PATCH 01/13] Add first tests for the autopkgtest integration This currently checks that an autopkgtest is triggered for an installable package, and that no autopkgtest is triggered for an uninstallable one (failing, demonstrating a bug). --- tests/autopkgtest.py | 262 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100755 tests/autopkgtest.py diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py new file mode 100755 index 0000000..23847bc --- /dev/null +++ b/tests/autopkgtest.py @@ -0,0 +1,262 @@ +#!/usr/bin/python +# (C) 2014 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import tempfile +import shutil +import os +import sys +import subprocess +import unittest + +architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'] + +my_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class TestData: + def __init__(self): + '''Construct local test package indexes. + + The archive is initially empty. You can create new packages with + create_deb(). self.path contains the path of the archive, and + self.apt_source provides an apt source "deb" line. + + It is kept in a temporary directory which gets removed when the Archive + object gets deleted. + ''' + self.path = tempfile.mkdtemp(prefix='testarchive.') + self.apt_source = 'deb file://%s /' % self.path + self.dirs = {False: os.path.join(self.path, 'data', 'testing'), + True: os.path.join(self.path, 'data', 'unstable')} + os.makedirs(self.dirs[False]) + os.mkdir(self.dirs[True]) + self.added_sources = {False: set(), True: set()} + self.added_binaries = {False: set(), True: set()} + + # pre-create all files for all architectures + for arch in architectures: + for dir in self.dirs.values(): + with open(os.path.join(dir, 'Packages_' + arch), 'w'): + pass + for dir in self.dirs.values(): + for fname in ['Dates', 'Blocks']: + with open(os.path.join(dir, fname), 'w'): + pass + for dname in ['Hints']: + os.mkdir(os.path.join(dir, dname)) + + os.mkdir(os.path.join(self.path, 'output')) + + # create temporary home dir for proposed-migration autopktest status + self.home = os.path.join(self.path, 'home') + os.environ['HOME'] = self.home + os.makedirs(os.path.join(self.home, 'proposed-migration', + 'autopkgtest', 'work')) + + def __del__(self): + shutil.rmtree(self.path) + + def add(self, name, unstable, fields={}, add_src=True): + '''Add a binary package to the index file. + + You need to specify at least the package name and in which list to put + it (unstable==True for unstable/proposed, or False for + testing/release). fields specifies all additional entries, e. g. + {'Depends': 'foo, bar', 'Conflicts: baz'}. There are defaults for most + fields. + + Unless add_src is set to False, this will also automatically create a + source record, based on fields['Source'] and name. + ''' + assert (name not in self.added_binaries[unstable]) + self.added_binaries[unstable].add(name) + + fields.setdefault('Architecture', architectures[0]) + fields.setdefault('Version', '1') + fields.setdefault('Priority', 'optional') + fields.setdefault('Section', 'devel') + fields.setdefault('Description', 'test pkg') + if fields['Architecture'] == 'all': + for a in architectures: + self._append(name, unstable, 'Packages_' + a, fields) + else: + self._append(name, unstable, 'Packages_' + fields['Architecture'], + fields) + + if add_src: + src = fields.get('Source', name) + if src not in self.added_sources[unstable]: + self.add_src(src, unstable, {'Version': fields['Version'], + 'Section': fields['Section']}) + + def add_src(self, name, unstable, fields={}): + '''Add a source package to the index file. + + You need to specify at least the package name and in which list to put + it (unstable==True for unstable/proposed, or False for + testing/release). fields specifies all additional entries, which can be + Version (default: 1), Section (default: devel), and Extra-Source-Only. + ''' + assert (name not in self.added_sources[unstable]) + self.added_sources[unstable].add(name) + + fields.setdefault('Version', '1') + fields.setdefault('Section', 'devel') + self._append(name, unstable, 'Sources', fields) + + def _append(self, name, unstable, file_name, fields): + with open(os.path.join(self.dirs[unstable], file_name), 'a') as f: + f.write('''Package: %s +Maintainer: Joe +''' % name) + + for k, v in fields.items(): + f.write('%s: %s\n' % (k, v)) + f.write('\n') + + +class Test(unittest.TestCase): + def setUp(self): + self.data = TestData() + + # add a bunch of packages to testing to avoid repetition + self.data.add('libc6', False) + self.data.add('libgreen1', False, {'Source': 'green', + 'Depends': 'libc6 (>= 0.9)'}) + self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', + 'Conflicts': 'green'}) + self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', + 'Conflicts': 'green'}) + self.data.add('justdata', False, {'Architecture': 'all'}) + + self.britney = os.path.join(my_dir, 'britney.py') + self.britney_conf = os.path.join(my_dir, 'britney.conf') + assert os.path.exists(self.britney) + assert os.path.exists(self.britney_conf) + + # fake adt-britney script + self.adt_britney = os.path.join(self.data.home, 'auto-package-testing', + 'jenkins', 'adt-britney') + os.makedirs(os.path.dirname(self.adt_britney)) + + with open(self.adt_britney, 'w') as f: + f.write('''#!/bin/sh -e +echo "$@" >> /%s/adt-britney.log ''' % self.data.path) + os.chmod(self.adt_britney, 0o755) + + def tearDown(self): + del self.data + + def make_adt_britney(self, request): + with open(self.adt_britney, 'w') as f: + f.write('''#!%(py)s +import argparse, shutil,sys + +def request(): + if args.req: + shutil.copy(args.req, '%(path)s/adt-britney.requestarg') + with open(args.output, 'w') as f: + f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW')) + +def submit(): + with open(args.req, 'w') as f: + f.write("""%(rq)s""".replace('PASS', 'RUNNING'). + replace('FAIL', 'RUNNING')) + +def collect(): + with open(args.output, 'w') as f: + f.write("""%(rq)s""") + +p = argparse.ArgumentParser() +p.add_argument('-c', '--config') +p.add_argument('-a', '--arch') +p.add_argument('-r', '--release') +p.add_argument('-P', '--use-proposed', action='store_true') +p.add_argument('-U', '--no-update', action='store_true') +sp = p.add_subparsers() + +prequest = sp.add_parser('request') +prequest.add_argument('-O', '--output') +prequest.add_argument('req', nargs='?') +prequest.set_defaults(func=request) + +psubmit = sp.add_parser('submit') +psubmit.add_argument('req') +psubmit.set_defaults(func=submit) + +pcollect = sp.add_parser('collect') +pcollect.add_argument('-O', '--output') +pcollect.add_argument('-n', '--new-only', action='store_true', default=False) +pcollect.set_defaults(func=collect) + +args = p.parse_args() +args.func() +''' % {'py': sys.executable, 'path': self.data.path, 'rq': request}) + + def run_britney(self, args=[]): + '''Run britney and return (exit, out, err)''' + + britney = subprocess.Popen([self.britney, '-c', self.britney_conf], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.data.path, + universal_newlines=True) + (out, err) = britney.communicate() + return (britney.returncode, out, err) + + def test_no_request_for_uninstallable(self): + '''Does not request a test for an uninstallable package''' + + # uninstallable unstable version + self.data.add('green', True, + {'Version': '1.1~beta', + 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}) + + self.make_adt_britney('green 1.1~beta FAIL green 1.1~beta\n') + + (c, o, e) = self.run_britney() + self.assertEqual(e, '', e) + self.assertEqual(c, 0, o + e) + + with open(os.path.join(self.data.path, 'output', 'excuses.html')) as f: + excuses = f.read() + self.assertIn('green/amd64 unsatisfiable Depends: libgreen1 (>= 2)', excuses) + # does not request autopkgtest for uninstallable package + self.assertNotIn('autopkgtest', excuses) + + def test_request_for_installable(self): + '''Requests a test for an installable package''' + + # installable unstable version + self.data.add('green', True, + {'Version': '1.1~beta', + 'Depends': 'libc6 (>= 0.9), libgreen1'}) + + self.make_adt_britney('green 1.1~beta RUNNING green 1.1~beta\n') + + (c, o, e) = self.run_britney() + self.assertEqual(c, 0, o + e) + self.assertEqual(e, '') + + with open(os.path.join(self.data.path, 'output', 'excuses.html')) as f: + excuses = f.read() + self.assertRegexpMatches(excuses, r'\bgreen\b.*>1 to .*>1.1~beta<') + self.assertIn('
  • autopkgtest for green 1.1~beta: RUNNING', excuses) + + def shell(self): + # uninstallable unstable version + self.data.add('yellow', True, {'Version': '1.1~beta', + 'Depends': 'libc6 (>= 0.9), nosuchpkg'}) + + self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n') + + print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) + subprocess.call(['bash', '-i'], cwd=self.data.path) + + +unittest.main() From fecf3be8115192cfed95f74c1ec2835247ad314d Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 19 Feb 2014 13:36:20 -0800 Subject: [PATCH 02/13] refactoring, add tests for passing and failing adt test --- tests/autopkgtest.py | 100 +++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 33 deletions(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 23847bc..66c6250 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -199,54 +199,88 @@ args.func() ''' % {'py': sys.executable, 'path': self.data.path, 'rq': request}) def run_britney(self, args=[]): - '''Run britney and return (exit, out, err)''' + '''Run britney. + Assert that it succeeds and does not produce anything on stderr. + Return generated excuses.html output. + ''' britney = subprocess.Popen([self.britney, '-c', self.britney_conf], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.data.path, universal_newlines=True) (out, err) = britney.communicate() - return (britney.returncode, out, err) - - def test_no_request_for_uninstallable(self): - '''Does not request a test for an uninstallable package''' - - # uninstallable unstable version - self.data.add('green', True, - {'Version': '1.1~beta', - 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}) - - self.make_adt_britney('green 1.1~beta FAIL green 1.1~beta\n') - - (c, o, e) = self.run_britney() - self.assertEqual(e, '', e) - self.assertEqual(c, 0, o + e) + self.assertEqual(britney.returncode, 0, out + err) + self.assertEqual(err, '') with open(os.path.join(self.data.path, 'output', 'excuses.html')) as f: excuses = f.read() - self.assertIn('green/amd64 unsatisfiable Depends: libgreen1 (>= 2)', excuses) - # does not request autopkgtest for uninstallable package - self.assertNotIn('autopkgtest', excuses) - def test_request_for_installable(self): - '''Requests a test for an installable package''' + return excuses - # installable unstable version - self.data.add('green', True, - {'Version': '1.1~beta', - 'Depends': 'libc6 (>= 0.9), libgreen1'}) + def test_no_request_for_uninstallable(self): + '''Does not request a test for an uninstallable package''' - self.make_adt_britney('green 1.1~beta RUNNING green 1.1~beta\n') + self.do_test( + # uninstallable unstable version + [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'})], + 'green 1.1~beta RUNNING green 1.1~beta\n', + False, + [r'\bgreen\b.*>1 to .*>1.1~beta<', + 'green/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'], + # autopkgtest should not be triggered for uninstallable pkg + ['autopkgtest']) + + def test_request_for_installable_running(self): + '''Requests a test for an installable package, test still running''' + + self.do_test( + [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], + 'green 1.1~beta RUNNING green 1.1~beta\n', + False, + [r'\bgreen\b.*>1 to .*>1.1~beta<', + '
  • autopkgtest for green 1.1~beta: RUNNING']) + + def test_request_for_installable_fail(self): + '''Requests a test for an installable package, test fail''' + + self.do_test( + [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], + 'green 1.1~beta FAIL green 1.1~beta\n', + False, + [r'\bgreen\b.*>1 to .*>1.1~beta<', + '
  • autopkgtest for green 1.1~beta: FAIL']) + + def test_request_for_installable_pass(self): + '''Requests a test for an installable package, test pass''' + + self.do_test( + [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], + 'green 1.1~beta PASS green 1.1~beta\n', + True, + [r'\bgreen\b.*>1 to .*>1.1~beta<', + '
  • autopkgtest for green 1.1~beta: PASS']) + + def do_test(self, unstable_add, adt_request, considered, expect=None, + no_expect=None): + for (pkg, fields) in unstable_add: + self.data.add(pkg, True, fields) + + self.make_adt_britney(adt_request) + + excuses = self.run_britney() + if considered: + self.assertIn('Valid candidate', excuses) + else: + self.assertIn('Not considered', excuses) - (c, o, e) = self.run_britney() - self.assertEqual(c, 0, o + e) - self.assertEqual(e, '') + if expect: + for re in expect: + self.assertRegexpMatches(excuses, re) + if no_expect: + for re in no_expect: + self.assertNotRegexpMatches(excuses, re) - with open(os.path.join(self.data.path, 'output', 'excuses.html')) as f: - excuses = f.read() - self.assertRegexpMatches(excuses, r'\bgreen\b.*>1 to .*>1.1~beta<') - self.assertIn('
  • autopkgtest for green 1.1~beta: RUNNING', excuses) def shell(self): # uninstallable unstable version From 0f5d8cfdcc68b00597a2815cd23e2d022de4c7cb Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 19 Feb 2014 15:01:00 -0800 Subject: [PATCH 03/13] fix mock adt-britney request for RUNNING --- tests/autopkgtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 66c6250..20704c1 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -161,7 +161,7 @@ def request(): if args.req: shutil.copy(args.req, '%(path)s/adt-britney.requestarg') with open(args.output, 'w') as f: - f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW')) + f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW').replace('RUNNING', 'NEW')) def submit(): with open(args.req, 'w') as f: From 899417b78d468ec225e06fa4ed77efc286790e6d Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 19 Feb 2014 15:02:11 -0800 Subject: [PATCH 04/13] return britney output as well, and add some commented-out extra debugging --- tests/autopkgtest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 20704c1..b15cdf9 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -202,7 +202,7 @@ args.func() '''Run britney. Assert that it succeeds and does not produce anything on stderr. - Return generated excuses.html output. + Return (excuses.html, britney_out). ''' britney = subprocess.Popen([self.britney, '-c', self.britney_conf], stdout=subprocess.PIPE, @@ -216,7 +216,7 @@ args.func() with open(os.path.join(self.data.path, 'output', 'excuses.html')) as f: excuses = f.read() - return excuses + return (excuses, out) def test_no_request_for_uninstallable(self): '''Does not request a test for an uninstallable package''' @@ -268,7 +268,11 @@ args.func() self.make_adt_britney(adt_request) - excuses = self.run_britney() + (excuses, out) = self.run_britney() + #print('-------\nexcuses: %s\n-----' % excuses) + #print('-------\nout: %s\n-----' % out) + #print('run:\n%s -c %s\n' % (self.britney, self.britney_conf)) + #subprocess.call(['bash', '-i'], cwd=self.data.path) if considered: self.assertIn('Valid candidate', excuses) else: From d59d78f718ef4707117560c636677e0cc55328c7 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 19 Feb 2014 15:02:19 -0800 Subject: [PATCH 05/13] Add tests for multiple reverse dependencies with tests This reproduces the failure to hold back gccgo-4.9 due to the broken libgcc1 package. This only seems to happen when a new source package takes over an existing binary package. --- tests/autopkgtest.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index b15cdf9..5d85dc7 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -129,7 +129,9 @@ class Test(unittest.TestCase): self.data.add('libgreen1', False, {'Source': 'green', 'Depends': 'libc6 (>= 0.9)'}) self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1', - 'Conflicts': 'green'}) + 'Conflicts': 'blue'}) + self.data.add('lightgreen', False, {'Depends': 'libgreen1'}) + self.data.add('darkgreen', False, {'Depends': 'libgreen1'}) self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)', 'Conflicts': 'green'}) self.data.add('justdata', False, {'Architecture': 'all'}) @@ -261,6 +263,45 @@ args.func() [r'\bgreen\b.*>1 to .*>1.1~beta<', '
  • autopkgtest for green 1.1~beta: PASS']) + def test_multi_rdepends_with_tests(self): + '''Multiple reverse dependencies with tests''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 RUNNING green 2\n', + False, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: RUNNING']) + + def test_multi_rdepends_with_some_tests(self): + '''Multiple reverse dependencies with some tests''' + + # add a third reverse dependency to libgreen1 which does not have a test + self.data.add('mint', False, {'Depends': 'libgreen1'}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 RUNNING green 2\n' + 'darkgreen 1 RUNNING green 2\n', + False, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: RUNNING', + '
  • autopkgtest for darkgreen 1: RUNNING']) + + def test_binary_from_new_source_package(self): + '''building an existing binary for a new source package''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 RUNNING green 2\n', + False, + [r'\bnewgreen\b.*>- to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: RUNNING']) + def do_test(self, unstable_add, adt_request, considered, expect=None, no_expect=None): for (pkg, fields) in unstable_add: From a6a086c4b51abcaba798034dcb7cf4d506ab875e Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 19 Feb 2014 17:25:19 -0800 Subject: [PATCH 06/13] add failing test for test results from a previous version --- tests/autopkgtest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 5d85dc7..d213d0b 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -302,6 +302,17 @@ args.func() '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: RUNNING']) + def test_result_from_older_version(self): + '''test result from older version than the uploaded one''' + + self.do_test( + [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], + 'green 1.1~alpha PASS green 1.1~beta\n', + False, + [r'\bgreen\b.*>1 to .*>1.1~beta<', + # it's not entirely clear what precisely it should say here + '
  • autopkgtest for green 1.1~beta: RUNNING']) + def do_test(self, unstable_add, adt_request, considered, expect=None, no_expect=None): for (pkg, fields) in unstable_add: From 319bb8260b79424c242fe89a9d41d3650ec45461 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 25 Feb 2014 08:05:39 +0100 Subject: [PATCH 07/13] use symbolic constants instead of True/False for considered status --- tests/autopkgtest.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index d213d0b..1aebc68 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -17,6 +17,9 @@ architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el'] my_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +NOT_CONSIDERED = False +VALID_CANDIDATE = True + class TestData: def __init__(self): @@ -227,7 +230,7 @@ args.func() # uninstallable unstable version [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'})], 'green 1.1~beta RUNNING green 1.1~beta\n', - False, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', 'green/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'], # autopkgtest should not be triggered for uninstallable pkg @@ -239,7 +242,7 @@ args.func() self.do_test( [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], 'green 1.1~beta RUNNING green 1.1~beta\n', - False, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', '
  • autopkgtest for green 1.1~beta: RUNNING']) @@ -249,7 +252,7 @@ args.func() self.do_test( [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], 'green 1.1~beta FAIL green 1.1~beta\n', - False, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', '
  • autopkgtest for green 1.1~beta: FAIL']) @@ -259,7 +262,7 @@ args.func() self.do_test( [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], 'green 1.1~beta PASS green 1.1~beta\n', - True, + VALID_CANDIDATE, [r'\bgreen\b.*>1 to .*>1.1~beta<', '
  • autopkgtest for green 1.1~beta: PASS']) @@ -270,7 +273,7 @@ args.func() [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], 'lightgreen 1 PASS green 2\n' 'darkgreen 1 RUNNING green 2\n', - False, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: RUNNING']) @@ -285,7 +288,7 @@ args.func() [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], 'lightgreen 1 RUNNING green 2\n' 'darkgreen 1 RUNNING green 2\n', - False, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>2<', '
  • autopkgtest for lightgreen 1: RUNNING', '
  • autopkgtest for darkgreen 1: RUNNING']) @@ -297,7 +300,7 @@ args.func() [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], 'lightgreen 1 PASS green 2\n' 'darkgreen 1 RUNNING green 2\n', - False, + NOT_CONSIDERED, [r'\bnewgreen\b.*>- to .*>2<', '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: RUNNING']) @@ -308,7 +311,7 @@ args.func() self.do_test( [('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})], 'green 1.1~alpha PASS green 1.1~beta\n', - False, + NOT_CONSIDERED, [r'\bgreen\b.*>1 to .*>1.1~beta<', # it's not entirely clear what precisely it should say here '
  • autopkgtest for green 1.1~beta: RUNNING']) @@ -337,7 +340,6 @@ args.func() for re in no_expect: self.assertNotRegexpMatches(excuses, re) - def shell(self): # uninstallable unstable version self.data.add('yellow', True, {'Version': '1.1~beta', From 5e54ba901803b06f51eb516ed88dbc906f3cdf74 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 25 Feb 2014 08:14:11 +0100 Subject: [PATCH 08/13] some more tests --- tests/autopkgtest.py | 90 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 6 deletions(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 1aebc68..0992d1c 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -266,8 +266,8 @@ args.func() [r'\bgreen\b.*>1 to .*>1.1~beta<', '
  • autopkgtest for green 1.1~beta: PASS']) - def test_multi_rdepends_with_tests(self): - '''Multiple reverse dependencies with tests''' + def test_multi_rdepends_with_tests_running(self): + '''Multiple reverse dependencies with tests (still running)''' self.do_test( [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], @@ -278,8 +278,32 @@ args.func() '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: RUNNING']) - def test_multi_rdepends_with_some_tests(self): - '''Multiple reverse dependencies with some tests''' + def test_multi_rdepends_with_tests_fail(self): + '''Multiple reverse dependencies with tests (fail)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 FAIL green 2\n', + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: FAIL']) + + def test_multi_rdepends_with_tests_pass(self): + '''Multiple reverse dependencies with tests (pass)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 PASS green 2\n', + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: PASS']) + + def test_multi_rdepends_with_some_tests_running(self): + '''Multiple reverse dependencies with some tests (running)''' # add a third reverse dependency to libgreen1 which does not have a test self.data.add('mint', False, {'Depends': 'libgreen1'}) @@ -293,8 +317,38 @@ args.func() '
  • autopkgtest for lightgreen 1: RUNNING', '
  • autopkgtest for darkgreen 1: RUNNING']) - def test_binary_from_new_source_package(self): - '''building an existing binary for a new source package''' + def test_multi_rdepends_with_some_tests_fail(self): + '''Multiple reverse dependencies with some tests (fail)''' + + # add a third reverse dependency to libgreen1 which does not have a test + self.data.add('mint', False, {'Depends': 'libgreen1'}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 FAIL green 2\n', + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: FAIL']) + + def test_multi_rdepends_with_some_tests_pass(self): + '''Multiple reverse dependencies with some tests (pass)''' + + # add a third reverse dependency to libgreen1 which does not have a test + self.data.add('mint', False, {'Depends': 'libgreen1'}) + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 PASS green 2\n', + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: PASS']) + + def test_binary_from_new_source_package_running(self): + '''building an existing binary for a new source package (running)''' self.do_test( [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], @@ -305,6 +359,30 @@ args.func() '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: RUNNING']) + def test_binary_from_new_source_package_fail(self): + '''building an existing binary for a new source package (fail)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 FAIL green 2\n', + NOT_CONSIDERED, + [r'\bnewgreen\b.*>- to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: FAIL']) + + def test_binary_from_new_source_package_pass(self): + '''building an existing binary for a new source package (pass)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], + 'lightgreen 1 PASS green 2\n' + 'darkgreen 1 PASS green 2\n', + VALID_CANDIDATE, + [r'\bnewgreen\b.*>- to .*>2<', + '
  • autopkgtest for lightgreen 1: PASS', + '
  • autopkgtest for darkgreen 1: PASS']) + def test_result_from_older_version(self): '''test result from older version than the uploaded one''' From 882a985966dc43b2a42e430561dd4f1b47070df3 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 27 Feb 2014 16:01:34 +0100 Subject: [PATCH 09/13] add --debug option mock adt-britney --- tests/autopkgtest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 0992d1c..0ebf337 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -182,6 +182,7 @@ p.add_argument('-c', '--config') p.add_argument('-a', '--arch') p.add_argument('-r', '--release') p.add_argument('-P', '--use-proposed', action='store_true') +p.add_argument('-d', '--debug', action='store_true') p.add_argument('-U', '--no-update', action='store_true') sp = p.add_subparsers() From 2d73f30d64e95e2e0cdaadc18f5437aa8e0ec0b0 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 27 Feb 2014 18:38:20 +0100 Subject: [PATCH 10/13] run britney in verbose mode --- tests/autopkgtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 0ebf337..283d30c 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -210,7 +210,7 @@ args.func() Assert that it succeeds and does not produce anything on stderr. Return (excuses.html, britney_out). ''' - britney = subprocess.Popen([self.britney, '-c', self.britney_conf], + britney = subprocess.Popen([self.britney, '-v', '-c', self.britney_conf], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.data.path, From 2b7e93515fed8cb0abdd45f107f4c3db9524a614 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 17 Mar 2014 08:43:25 +0100 Subject: [PATCH 11/13] add test for uninstallable binary built from new source package --- tests/autopkgtest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 283d30c..cde2032 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -384,6 +384,18 @@ args.func() '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: PASS']) + def test_binary_from_new_source_package_uninst(self): + '''building an existing binary for a new source package (uninstallable)''' + + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6, nosuchpkg'})], + '', + NOT_CONSIDERED, + [r'\bnewgreen\b.*- to .*>2<', + 'libgreen1/amd64 unsatisfiable Depends: nosuchpkg'], + # autopkgtest should not be triggered for uninstallable pkg + ['autopkgtest']) + def test_result_from_older_version(self): '''test result from older version than the uploaded one''' From 0cd14d1394a027034e3e392dfa9a229fe8505f2b Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 17 Mar 2014 09:35:20 +0100 Subject: [PATCH 12/13] fix source package name in cause of new_source tests --- tests/autopkgtest.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index cde2032..26f7fb5 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -353,10 +353,10 @@ args.func() self.do_test( [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 RUNNING green 2\n', + 'lightgreen 1 PASS newgreen 2\n' + 'darkgreen 1 RUNNING newgreen 2\n', NOT_CONSIDERED, - [r'\bnewgreen\b.*>- to .*>2<', + [r'\bnewgreen\b.*\(- to .*>2<', '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: RUNNING']) @@ -365,10 +365,10 @@ args.func() self.do_test( [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 FAIL green 2\n', + 'lightgreen 1 PASS newgreen 2\n' + 'darkgreen 1 FAIL newgreen 2\n', NOT_CONSIDERED, - [r'\bnewgreen\b.*>- to .*>2<', + [r'\bnewgreen\b.*\(- to .*>2<', '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: FAIL']) @@ -377,10 +377,10 @@ args.func() self.do_test( [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})], - 'lightgreen 1 PASS green 2\n' - 'darkgreen 1 PASS green 2\n', + 'lightgreen 1 PASS newgreen 2\n' + 'darkgreen 1 PASS newgreen 2\n', VALID_CANDIDATE, - [r'\bnewgreen\b.*>- to .*>2<', + [r'\bnewgreen\b.*\(- to .*>2<', '
  • autopkgtest for lightgreen 1: PASS', '
  • autopkgtest for darkgreen 1: PASS']) @@ -389,9 +389,9 @@ args.func() self.do_test( [('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6, nosuchpkg'})], - '', + 'darkgreen 1 FAIL newgreen 2\n', NOT_CONSIDERED, - [r'\bnewgreen\b.*- to .*>2<', + [r'\bnewgreen\b.*\(- to .*>2<', 'libgreen1/amd64 unsatisfiable Depends: nosuchpkg'], # autopkgtest should not be triggered for uninstallable pkg ['autopkgtest']) From 9b4eeafef20ca2f3a4c3b4ff430b3039da34ae2a Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 12 May 2014 14:04:55 +0200 Subject: [PATCH 13/13] Mark test_result_from_older_version as XFAIL --- tests/autopkgtest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/autopkgtest.py b/tests/autopkgtest.py index 26f7fb5..308767a 100755 --- a/tests/autopkgtest.py +++ b/tests/autopkgtest.py @@ -396,6 +396,7 @@ args.func() # autopkgtest should not be triggered for uninstallable pkg ['autopkgtest']) + @unittest.expectedFailure def test_result_from_older_version(self): '''test result from older version than the uploaded one'''