From 380e3fca646adfb5d05404132389edbb69679388 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Mon, 17 Aug 2015 21:55:18 +0200 Subject: [PATCH] autopkgtest: Check for test results from newer package version than the requested one There are scenarios when britney requests a package test for a particular version but we actually get a result for a later version: * When britney runs the later version is not built yet and thus it is in excludes; but at the time when the test actually runs the package is built. * We don't support running tests for a given older (source) version yet, tests always get run from the latest unstable source even if that isn't built yet. Thus we need to consider results >= the requested version. However, we prefer a succesful result for the originally requested version so that we can continue to remove a broken version from unstable. This is already covered by TestAutoPkgTest.test_remove_from_unstable. --- autopkgtest.py | 33 +++++++++++++++++++++++- tests/test_autopkgtest.py | 53 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/autopkgtest.py b/autopkgtest.py index 9650486..7606841 100644 --- a/autopkgtest.py +++ b/autopkgtest.py @@ -63,6 +63,26 @@ def merge_triggers(trigs1, trigs2): return list(pkgvers.items()) +def latest_item(ver_map, min_version=None): + '''Return (ver, value) from version -> value map with latest version number + + If min_version is given, version has to be >= that, otherwise a KeyError is + raised. + ''' + latest = None + for ver in ver_map: + if latest is None or apt_pkg.version_compare(ver, latest) > 0: + latest = ver + if min_version is not None and latest is not None and \ + apt_pkg.version_compare(latest, min_version) < 0: + latest = None + + if latest is not None: + return (latest, ver_map[latest]) + else: + raise KeyError('no version >= %s' % min_version) + + class AutoPackageTest(object): """autopkgtest integration @@ -499,7 +519,18 @@ class AutoPackageTest(object): for arch in self.britney.options.adt_arches.split(): try: (_, ver_map, ever_passed) = self.test_results[testsrc][arch] - (status, triggers) = ver_map[testver] + + # we prefer passing tests for the specified testver; but if + # they fail, we also check for a result > testver, as test + # runs might see built versions which we didn't see in + # britney yet + try: + (status, triggers) = ver_map[testver] + if not status: + raise KeyError + except KeyError: + (testver, (status, triggers)) = latest_item(ver_map, testver) + # triggers might contain tuples or lists if (trigsrc, trigver) not in triggers and [trigsrc, trigver] not in triggers: raise KeyError('No result for trigger %s/%s yet' % (trigsrc, trigver)) diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 243074e..7e2e698 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -446,6 +446,59 @@ lightgreen 1 i386 green 2 ['autopkgtest for brokengreen']) self.assertEqual(self.amqp_requests, set()) + def test_rdepends_unbuilt_new_version_result(self): + '''Unbuilt reverse dependency gets test result for newer version + + This might happen if the autopkgtest infrastructure runs the unstable + source tests against the testing binaries. Even if that gets done + properly it might still happen that at the time of the britney run the + package isn't built yet, but it is once the test gets run. + ''' + # old lightgreen fails, thus new green should be held back + self.swift.set_results({'autopkgtest-series': { + 'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'), + 'series/amd64/d/darkgreen/20150101_100001@': (0, 'darkgreen 1'), + 'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), + 'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'), + 'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'), + 'series/i386/g/green/20150101_020000@': (0, 'green 1'), + 'series/amd64/g/green/20150101_020000@': (0, 'green 1'), + 'series/i386/g/green/20150101_100200@': (0, 'green 2'), + 'series/amd64/g/green/20150101_100201@': (0, 'green 2'), + }}) + + # add unbuilt lightgreen; should run tests against the old version + self.data.add_src('lightgreen', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.do_test( + [('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')], + NOT_CONSIDERED, + [r'\bgreen\b.*>1 to .*>2<', + r'\blightgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 1 \(2 is unbuilt/uninstallable\): .*amd64.*Regression.*i386.*Regression', + r'lightgreen has no up-to-date binaries on any arch'], + ['Valid candidate']) + self.assertEqual(self.pending_requests, '') + + # lightgreen 2 stays unbuilt in britney, but we get a test result for it + self.swift.set_results({'autopkgtest-series': { + 'series/i386/l/lightgreen/20150101_100200@': (0, 'lightgreen 2'), + 'series/amd64/l/lightgreen/20150101_102000@': (0, 'lightgreen 2'), + }}) + self.do_test( + [], + VALID_CANDIDATE, + [r'\bgreen\b.*>1 to .*>2<', + r'\blightgreen\b.*>1 to .*>2<', + r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass', + r'autopkgtest for lightgreen 2.*: .*amd64.*Pass.*i386.*Pass', + r'lightgreen has no up-to-date binaries on any arch']) + + self.assertEqual(self.pending_requests, '') + def test_hint_force_badtest(self): '''force-badtest hint'''