Lubuntu
/
britney2-ubuntu
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch autopkgtest evaluation to cloud results

Browse Source 
- Change AutoPackageTest.results() to evaluate the Swift results instead of
   the adt-britney ones.
 - Drop TestAdtBritney tests which now fail as we switched results evaluation
   to swift. Port relevant tests to TestAutoPkgTest.
 - Drop obsolete adt-britney autopkgtest code.
 - Adjust TestBoottestEnd2End.test_with_adt() for cloud results.
bzr-import-20160707
Martin Pitt 10 years ago
parent 027404b6e7 4e5ed1739d
commit ee4450b671
5 changed files with 349 additions and 823 deletions
Show Stats Download Patch File Download Diff File

291
autopkgtest.py
Unescape View File

@ -1,9 +1,10 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Canonical Ltd.
# Author: Colin Watson <cjwatson@ubuntu.com>
# Partly based on code in auto-package-testing by
# Jean-Baptiste Lallement <jean-baptiste.lallement@canonical.com>
# Authors:
# Colin Watson <cjwatson@ubuntu.com>
# Jean-Baptiste Lallement <jean-baptiste.lallement@canonical.com>
# Martin Pitt <martin.pitt@ubuntu.com>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -17,12 +18,7 @@
from __future__ import print_function
from collections import defaultdict
from contextlib import closing
import os
import subprocess
import tempfile
from textwrap import dedent
import time
import json
import tarfile
@ -37,9 +33,6 @@ import kombu
from consts import (AUTOPKGTEST, BINARIES, DEPENDS, RDEPENDS, SOURCE, VERSION)
adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney")
ADT_PASS = ["PASS", "ALWAYSFAIL"]
ADT_EXCUSES_LABELS = {
"PASS": '<span style="background:#87d96c">Pass</span>',
"ALWAYSFAIL": '<span style="background:#e5c545">Always failed</span>',
@ -84,8 +77,6 @@ class AutoPackageTest(object):
self.distribution = distribution
self.series = series
self.debug = debug
self.read()
self.rc_path = None # for adt-britney, obsolete
self.test_state_dir = os.path.join(britney.options.unstable,
'autopkgtest')
# map of requested tests from request()
@ -397,92 +388,6 @@ class AutoPackageTest(object):
result.add((src, arch))
return result
#
# obsolete adt-britney helpers
#
def _ensure_rc_file(self):
if self.rc_path:
return
self.rc_path = os.path.expanduser(
"~/proposed-migration/autopkgtest/rc.%s" % self.series)
with open(self.rc_path, "w") as rc_file:
home = os.path.expanduser("~")
print(dedent("""\
release: %s
aptroot: ~/.chdist/%s-proposed-amd64/
apturi: file:%s/mirror/%s
components: main restricted universe multiverse
rsync_host: rsync://tachash.ubuntu-ci/adt/
datadir: ~/proposed-migration/autopkgtest/data""" %
(self.series, self.series, home, self.distribution)),
file=rc_file)
@property
def _request_path(self):
return os.path.expanduser(
"~/proposed-migration/autopkgtest/work/adt.request.%s" %
self.series)
@property
def _result_path(self):
return os.path.expanduser(
"~/proposed-migration/autopkgtest/work/adt.result.%s" %
self.series)
def _parse(self, path):
if os.path.exists(path):
with open(path) as f:
for line in f:
line = line.strip()
if line.startswith("Suite:") or line.startswith("Date:"):
continue
linebits = line.split()
if len(linebits) < 2:
print("W: Invalid line format: '%s', skipped" % line)
continue
yield linebits
def read(self):
'''Loads a list of results
This function loads a list of results returned by __parse() and builds
2 lists:
- a list of source package/version with all the causes that
triggered a test and the result of the test for this trigger.
- a list of packages/version that triggered a test with the source
package/version and result triggered by this package.
These lists will be used in result() called from britney.py to generate
excuses and now which uploads passed, caused regression or which tests
have always been failing
'''
self.pkglist = defaultdict(dict)
self.pkgcauses = defaultdict(lambda: defaultdict(list))
for linebits in self._parse(self._result_path):
(src, ver, status) = linebits[:3]
if not (src in self.pkglist and ver in self.pkglist[src]):
self.pkglist[src][ver] = {
"status": status,
"causes": {}
}
i = iter(linebits[3:])
for trigsrc, trigver in zip(i, i):
self.pkglist[src][ver]['causes'].setdefault(
trigsrc, []).append((trigver, status))
self.pkgcauses[trigsrc][trigver].append((status, src, ver))
def _adt_britney(self, *args):
command = [
adt_britney,
"-c", self.rc_path, "-r", self.series, "-PU",
]
if self.debug:
command.append("-d")
command.extend(args)
subprocess.check_call(command)
#
# Public API
#
@ -506,60 +411,6 @@ class AutoPackageTest(object):
self.log_verbose('Requesting %s/%s/%s autopkgtest to verify %s' %
(src, ver, arch, ', '.join(['%s/%s' % i for i in triggers])))
# deprecated requests for old Jenkins/lp:auto-package-testing, will go
# away
self._ensure_rc_file()
request_path = self._request_path
if os.path.exists(request_path):
os.unlink(request_path)
with closing(tempfile.NamedTemporaryFile(mode="w")) as request_file:
for src, ver in packages:
if src in self.pkglist and ver in self.pkglist[src]:
continue
print("%s %s" % (src, ver), file=request_file)
request_file.flush()
self._adt_britney("request", "-O", request_path, request_file.name)
# Remove packages that have been identified as invalid candidates for
# testing from the request file i.e run_autopkgtest = False
with open(request_path, 'r') as request_file:
lines = request_file.readlines()
with open(request_path, 'w') as request_file:
for line in lines:
src = line.split()[0]
if src not in excludes:
request_file.write(line)
else:
if self.britney.options.verbose:
self.log_verbose("Requested autopkgtest for %s but "
"run_autopkgtest set to False" % src)
for linebits in self._parse(request_path):
# Make sure that there's an entry in pkgcauses for each new
# request, so that results() gives useful information without
# relying on the submit/collect cycle. This improves behaviour
# in dry-run mode.
src = linebits.pop(0)
ver = linebits.pop(0)
if self.britney.options.verbose:
self.log_verbose("Requested autopkgtest for %s_%s (%s)" %
(src, ver, " ".join(linebits)))
try:
status = linebits.pop(0).upper()
while True:
trigsrc = linebits.pop(0)
trigver = linebits.pop(0)
for status, csrc, cver in self.pkgcauses[trigsrc][trigver]:
if csrc == trigsrc and cver == trigver:
break
else:
self.pkgcauses[trigsrc][trigver].append(
(status, src, ver))
except IndexError:
# End of the list
pass
def submit(self):
# send AMQP requests for new test requests
# TODO: Once we support version constraints in AMQP requests, add them
@ -601,89 +452,81 @@ class AutoPackageTest(object):
# mark them as pending now
self.update_pending_tests()
# deprecated requests for old Jenkins/lp:auto-package-testing, will go
# away
self._ensure_rc_file()
request_path = self._request_path
if os.path.exists(request_path):
self._adt_britney("submit", request_path)
def collect(self, packages):
# fetch results from swift
try:
swift_url = self.britney.options.adt_swift_url
except AttributeError:
self.log_error('ADT_SWIFT_URL not set, cannot collect results')
swift_url = None
return
try:
self.britney.options.adt_amqp
except AttributeError:
self.log_error('ADT_AMQP not set, not collecting results from swift')
swift_url = None
if swift_url:
# update results from swift for all packages that we are waiting
# for, and remove pending tests that we have results for on all
# arches
for pkg, verinfo in copy.deepcopy(self.pending_tests.items()):
for archinfo in verinfo.values():
for arch in archinfo:
self.fetch_swift_results(swift_url, pkg, arch)
# also update results for excuses whose tests failed, in case a
# manual retry worked
for (trigpkg, trigver) in packages:
if trigpkg not in self.pending_tests:
for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver):
self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' %
(pkg, arch, trigpkg, trigver))
self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver))
# update the results cache
with open(self.results_cache_file + '.new', 'w') as f:
json.dump(self.test_results, f, indent=2)
os.rename(self.results_cache_file + '.new', self.results_cache_file)
self.log_verbose('Updated results cache')
# new results remove pending requests, update the on-disk cache
self.update_pending_tests()
# deprecated results for old Jenkins/lp:auto-package-testing, will go
# away
self._ensure_rc_file()
result_path = self._result_path
self._adt_britney("collect", "-O", result_path)
self.read()
if self.britney.options.verbose:
for src in sorted(self.pkglist):
for ver in sorted(self.pkglist[src],
cmp=apt_pkg.version_compare):
for trigsrc in sorted(self.pkglist[src][ver]['causes']):
for trigver, status \
in self.pkglist[src][ver]['causes'][trigsrc]:
self.log_verbose("Collected autopkgtest status "
"for %s_%s/%s_%s: " "%s" %
(src, ver, trigsrc, trigver, status))
return
# update results from swift for all packages that we are waiting
# for, and remove pending tests that we have results for on all
# arches
for pkg, verinfo in copy.deepcopy(self.pending_tests.items()):
for archinfo in verinfo.values():
for arch in archinfo:
self.fetch_swift_results(swift_url, pkg, arch)
# also update results for excuses whose tests failed, in case a
# manual retry worked
for (trigpkg, trigver) in packages:
if trigpkg not in self.pending_tests:
for (pkg, arch) in self.failed_tests_for_trigger(trigpkg, trigver):
self.log_verbose('Checking for new results for failed %s on %s for trigger %s/%s' %
(pkg, arch, trigpkg, trigver))
self.fetch_swift_results(swift_url, pkg, arch, (trigpkg, trigver))
# update the results cache
with open(self.results_cache_file + '.new', 'w') as f:
json.dump(self.test_results, f, indent=2)
os.rename(self.results_cache_file + '.new', self.results_cache_file)
self.log_verbose('Updated results cache')
# new results remove pending requests, update the on-disk cache
self.update_pending_tests()
def results(self, trigsrc, trigver):
'''Return test results for triggering package
Return (ALWAYSFAIL|PASS|FAIL, src, ver) iterator for all package tests
that got triggered by trigsrc/trigver.
Return (passed, src, ver, arch -> ALWAYSFAIL|PASS|FAIL|RUNNING)
iterator for all package tests that got triggered by trigsrc/trigver.
'''
# deprecated results for old Jenkins/lp:auto-package-testing, will go
# away
for status, src, ver in self.pkgcauses[trigsrc][trigver]:
# Check for regression
if status == 'FAIL':
passed_once = False
for lver in self.pkglist[src]:
for trigsrc in self.pkglist[src][lver]['causes']:
for trigver, status \
in self.pkglist[src][lver]['causes'][trigsrc]:
if status == 'PASS':
passed_once = True
if not passed_once:
status = 'ALWAYSFAIL'
else:
status = 'REGRESSION'
yield status, src, ver
for testsrc, testver in self.tests_for_source(trigsrc, trigver):
passed = True
arch_status = {}
for arch in self.britney.options.adt_arches.split():
try:
(_, ver_map, ever_passed) = self.test_results[testsrc][arch]
(status, triggers) = ver_map[testver]
# triggers might contain tuples or lists
if (trigsrc, trigver) not in triggers and [trigsrc, trigver] not in triggers:
raise KeyError('No result for trigger %s/%s yet' % (trigsrc, trigver))
if status:
arch_status[arch] = 'PASS'
else:
# test failed, check ever_passed flag for that src/arch
if ever_passed:
arch_status[arch] = 'REGRESSION'
passed = False
else:
arch_status[arch] = 'ALWAYSFAIL'
except KeyError:
# no result for testsrc/testver/arch; still running?
try:
self.pending_tests[testsrc][testver][arch]
arch_status[arch] = 'RUNNING'
passed = False
except KeyError:
# neither done nor pending -> exclusion, or disabled
continue
# disabled or ignored?
if not arch_status:
continue
yield (passed, testsrc, testver, arch_status)

83
britney.py
Unescape View File

@ -225,7 +225,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes,
from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL)
from autopkgtest import AutoPackageTest, ADT_PASS, ADT_EXCUSES_LABELS, srchash
from autopkgtest import AutoPackageTest, ADT_EXCUSES_LABELS, srchash
from boottest import BootTest
@ -1849,30 +1849,21 @@ class Britney(object):
if not self.options.dry_run:
autopkgtest.submit()
autopkgtest.collect(autopkgtest_packages)
jenkins_public = "https://jenkins.qa.ubuntu.com/job"
jenkins_private = (
"http://d-jenkins.ubuntu-ci:8080/view/%s/view/AutoPkgTest/job" %
self.options.series.title())
cloud_url = "http://autopkgtest.ubuntu.com/packages/%(h)s/%(s)s/%(r)s/%(a)s"
for e in autopkgtest_excuses:
adtpass = True
adt_jenkins_sources = set() # temporary: drop when switching to cloud based tests
for status, adtsrc, adtver in autopkgtest.results(
for passed, adtsrc, adtver, arch_status in autopkgtest.results(
e.name, e.ver[1]):
adt_jenkins_sources.add(adtsrc)
public_url = "%s/%s-adt-%s/lastBuild" % (
jenkins_public, self.options.series,
adtsrc.replace("+", "-"))
private_url = "%s/%s-adt-%s/lastBuild" % (
jenkins_private, self.options.series,
adtsrc.replace("+", "-"))
adt_label = ADT_EXCUSES_LABELS.get(status, status)
e.addhtml(
"autopkgtest for %s %s: %s (Jenkins: "
"<a href=\"%s\">public</a>, "
"<a href=\"%s\">private</a>)" %
(adtsrc, adtver, adt_label, public_url, private_url))
if status not in ADT_PASS:
archmsg = []
for arch in sorted(arch_status):
url = cloud_url % {'h': srchash(adtsrc), 's': adtsrc,
'r': self.options.series, 'a': arch}
archmsg.append('<a href="%s">%s: %s</a>' %
(url, arch, ADT_EXCUSES_LABELS[arch_status[arch]]))
e.addhtml('autopkgtest for %s %s: %s' % (adtsrc, adtver, ', '.join(archmsg)))
# hints can override failures
if not passed:
hints = self.hints.search(
'force-badtest', package=adtsrc)
hints.extend(
@ -1884,54 +1875,10 @@ class Britney(object):
e.addhtml(
"Should wait for %s %s test, but forced by "
"%s" % (adtsrc, adtver, forces[0].user))
else:
adtpass = False
# temporary: also show results from cloud based tests,
# until that becomes the primary mechanism
adt_cloud_sources = set()
for testsrc, testver in autopkgtest.tests_for_source(e.name, e.ver[1]):
adt_cloud_sources.add(testsrc)
msg = '(informational) cloud autopkgtest for %s %s: ' % (testsrc, testver)
archmsg = []
for arch in self.options.adt_arches.split():
url = cloud_url % {'h': srchash(testsrc), 's': testsrc,
'r': self.options.series, 'a': arch}
try:
(_, ver_map, ever_passed) = autopkgtest.test_results[testsrc][arch]
(passed, triggers) = ver_map[testver]
# triggers might contain tuples or lists
if (e.name, e.ver[1]) not in triggers and [e.name, e.ver[1]] not in triggers:
raise KeyError('No result for trigger %s/%s yet' % (e.name, e.ver[1]))
if passed:
status = 'PASS'
else:
if ever_passed:
status = 'REGRESSION'
else:
status = 'ALWAYSFAIL'
except KeyError:
try:
autopkgtest.pending_tests[testsrc][testver][arch]
status = 'RUNNING'
except KeyError:
# neither done nor pending -> exclusion, or disabled
continue
passed = True
archmsg.append('<a href="%s">%s: %s</a>' %
(url, arch, ADT_EXCUSES_LABELS[status]))
if archmsg:
e.addhtml(msg + ', '.join(archmsg))
extra_sources = adt_cloud_sources - adt_jenkins_sources
missing_sources = adt_jenkins_sources - adt_cloud_sources
if missing_sources:
e.addhtml('(informational) Missing sources in cloud tests: %s' % ' '.join(missing_sources))
if extra_sources:
e.addhtml('(informational) Extra sources in cloud tests: %s' % ' '.join(extra_sources))
# end of temporary code
if not passed:
adtpass = False
if not adtpass and e.is_valid:
hints = self.hints.search('force-skiptest', package=e.name)

8
tests/__init__.py
Unescape View File

@ -172,3 +172,11 @@ class TestBase(unittest.TestCase):
excuses = f.read()
return (excuses, out)
def create_hint(self, username, content):
'''Create a hint file for the given username and content'''
hints_path = os.path.join(
self.data.path, 'data', self.data.series + '-proposed', 'Hints', username)
with open(hints_path, 'w') as fd:
fd.write(content)

746
tests/test_autopkgtest.py
Unescape View File

@ -7,10 +7,8 @@
# (at your option) any later version.
import apt_pkg
import operator
import os
import sys
import subprocess
import fileinput
import unittest
import json
@ -18,7 +16,6 @@ import json
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, PROJECT_DIR)
from autopkgtest import ADT_EXCUSES_LABELS
from tests import TestBase, mock_swift
NOT_CONSIDERED = False
@ -46,16 +43,6 @@ class TestAutoPkgTest(TestBase):
else:
sys.stdout.write(line)
# fake adt-britney script; necessary until we drop that code
self.adt_britney = os.path.join(
self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney')
os.makedirs(os.path.dirname(self.adt_britney))
with open(self.adt_britney, 'w') as f:
f.write('''#!/bin/sh -e
touch $HOME/proposed-migration/autopkgtest/work/adt.request.series
echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
os.chmod(self.adt_britney, 0o755)
# add a bunch of packages to testing to avoid repetition
self.data.add('libc6', False)
self.data.add('libgreen1', False, {'Source': 'green',
@ -89,7 +76,8 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
(excuses, out) = self.run_britney()
self.swift.stop()
#print('-------\nexcuses: %s\n-----' % excuses)
if 'SHOW_EXCUSES' in os.environ:
print('-------\nexcuses: %s\n-----' % excuses)
if 'SHOW_OUTPUT' in os.environ:
print('-------\nout: %s\n-----' % out)
if considered:
@ -120,14 +108,27 @@ echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
return out
def test_no_request_for_uninstallable(self):
'''Does not request a test for an uninstallable package'''
self.do_test(
# uninstallable unstable version
[('lightgreen', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'}, 'autopkgtest')],
NOT_CONSIDERED,
[r'\blightgreen\b.*>1</a> to .*>1.1~beta<',
'lightgreen/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'],
# autopkgtest should not be triggered for uninstallable pkg
['autopkgtest'])
self.assertEqual(self.pending_requests, '')
self.assertEqual(self.amqp_requests, set())
def test_multi_rdepends_with_tests_all_running(self):
'''Multiple reverse dependencies with tests (all running)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -153,7 +154,7 @@ lightgreen 1 i386 green 2
self.assertEqual(self.pending_requests, expected_pending)
# if we run britney again this should *not* trigger any new tests
self.do_test([], VALID_CANDIDATE, [r'\bgreen\b.*>1</a> to .*>2<'])
self.do_test([], NOT_CONSIDERED, [r'\bgreen\b.*>1</a> to .*>2<'])
self.assertEqual(self.amqp_requests, set())
# but the set of pending tests doesn't change
self.assertEqual(self.pending_requests, expected_pending)
@ -164,9 +165,7 @@ lightgreen 1 i386 green 2
# first run requests tests and marks them as pending
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -233,9 +232,7 @@ lightgreen 1 i386 green 2
# first run requests tests and marks them as pending
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -252,9 +249,7 @@ lightgreen 1 i386 green 2
out = self.do_test(
[],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*in progress',
@ -267,15 +262,174 @@ lightgreen 1 i386 green 2
self.assertIn('darkgreen 1 amd64 green 2', self.pending_requests)
self.assertIn('lightgreen 1 i386 green 2', self.pending_requests)
def test_multi_rdepends_with_tests_regression(self):
'''Multiple reverse dependencies with tests (regression)'''
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
}})
out = self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
self.assertEqual(self.pending_requests, '')
# not expecting any failures to retrieve from swift
self.assertNotIn('Failure', out, out)
def test_multi_rdepends_with_tests_regression_last_pass(self):
'''Multiple reverse dependencies with tests (regression), last one passes
This ensures that we don't just evaluate the test result of the last
test, but all of them.
'''
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
}})
out = self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
self.assertEqual(self.pending_requests, '')
# not expecting any failures to retrieve from swift
self.assertNotIn('Failure', out, out)
def test_multi_rdepends_with_tests_always_failed(self):
'''Multiple reverse dependencies with tests (always failed)'''
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/i386/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100200@': (4, 'green 2'),
'series/amd64/g/green/20150101_100201@': (4, 'green 2'),
}})
out = self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Always failed.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Always failed.*i386.*Always failed',
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
self.assertEqual(self.pending_requests, '')
# not expecting any failures to retrieve from swift
self.assertNotIn('Failure', out, out)
def test_hint_force_badtest(self):
'''force-badtest hint'''
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
}})
self.create_hint('pitti', 'force-badtest lightgreen/1')
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass',
r'Should wait for lightgreen 1 test, but forced by pitti'])
def test_hint_force_badtest_different_version(self):
'''force-badtest hint with non-matching version'''
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 1'),
'series/i386/g/green/20150101_100200@': (0, 'green 2'),
'series/amd64/g/green/20150101_100200@': (0, 'green 2'),
}})
self.create_hint('pitti', 'force-badtest lightgreen/0.1')
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression',
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'],
['Should wait'])
def test_hint_force_skiptest(self):
'''force-skiptest hint'''
self.create_hint('pitti', 'force-skiptest green/2')
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress',
r'Should wait for.*tests.*green 2.*forced by pitti'])
def test_hint_force_skiptest_different_version(self):
'''force-skiptest hint with non-matching version'''
self.create_hint('pitti', 'force-skiptest green/1')
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'],
['Should wait'])
def test_package_pair_running(self):
'''Two packages in unstable that need to go in together (running)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'\blightgreen\b.*>1</a> to .*>2<'])
@ -300,6 +454,58 @@ lightgreen 2 i386 lightgreen 2
'''
self.assertEqual(self.pending_requests, expected_pending)
def test_binary_from_new_source_package_running(self):
'''building an existing binary for a new source package (running)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')],
NOT_CONSIDERED,
[r'\bnewgreen\b.*\(- to .*>2<',
r'autopkgtest for newgreen 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for darkgreen 1: .*amd64.*in progress.*i386.*in progress'])
self.assertEqual(
self.amqp_requests,
set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen',
'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
expected_pending = '''darkgreen 1 amd64 newgreen 2
darkgreen 1 i386 newgreen 2
lightgreen 1 amd64 newgreen 2
lightgreen 1 i386 newgreen 2
newgreen 2 amd64 newgreen 2
newgreen 2 i386 newgreen 2
'''
self.assertEqual(self.pending_requests, expected_pending)
def test_binary_from_new_source_package_pass(self):
'''building an existing binary for a new source package (pass)'''
self.swift.set_results({'autopkgtest-series': {
'series/i386/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/amd64/d/darkgreen/20150101_100000@': (0, 'darkgreen 1'),
'series/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1'),
'series/i386/n/newgreen/20150101_100200@': (0, 'newgreen 2'),
'series/amd64/n/newgreen/20150101_100201@': (0, 'newgreen 2'),
}})
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'}, 'autopkgtest')],
VALID_CANDIDATE,
[r'\bnewgreen\b.*\(- to .*>2<',
r'autopkgtest for newgreen 2: .*amd64.*Pass.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
r'autopkgtest for darkgreen 1: .*amd64.*Pass.*i386.*Pass'])
self.assertEqual(
self.amqp_requests,
set(['debci-series-i386:newgreen', 'debci-series-amd64:newgreen',
'debci-series-i386:lightgreen', 'debci-series-amd64:lightgreen',
'debci-series-i386:darkgreen', 'debci-series-amd64:darkgreen']))
self.assertEqual(self.pending_requests, '')
def test_result_from_older_version(self):
'''test result from older version than the uploaded one'''
@ -310,9 +516,7 @@ lightgreen 2 i386 lightgreen 2
self.do_test(
[('darkgreen', {'Version': '2', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bdarkgreen\b.*>1</a> to .*>2<',
r'autopkgtest for darkgreen 2: .*amd64.*in progress.*i386.*in progress'])
@ -342,9 +546,7 @@ lightgreen 2 i386 lightgreen 2
self.data.remove_all(True)
self.do_test(
[('darkgreen', {'Version': '3', 'Depends': 'libc6 (>= 0.9), libgreen1'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bdarkgreen\b.*>1</a> to .*>3<',
r'autopkgtest for darkgreen 3: .*amd64.*in progress.*i386.*in progress'])
self.assertEqual(
@ -386,9 +588,7 @@ lightgreen 2 i386 lightgreen 2
# second run: new version re-triggers all tests
self.do_test(
[('libgreen1', {'Version': '3', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\green\b.*>1</a> to .*>3<',
r'autopkgtest for green 3: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for lightgreen 1: .*amd64.*in progress.*i386.*in progress',
@ -421,10 +621,7 @@ lightgreen 1 i386 green 3
'series/amd64/l/lightgreen/20150101_100010@': (0, 'lightgreen 1'),
}})
self.do_test(
[],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
[], NOT_CONSIDERED,
[r'\green\b.*>1</a> to .*>3<',
r'autopkgtest for green 3: .*amd64.*Pass.*i386.*Pass',
r'autopkgtest for lightgreen 1: .*amd64.*Pass.*i386.*Pass',
@ -460,9 +657,7 @@ lightgreen 1 i386 green 3
self.do_test(
[('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\blightgreen\b.*>1</a> to .*>2<',
r'autopkgtest for lightgreen 2: .*amd64.*Regression.*i386.*Regression'],
['in progress'])
@ -488,9 +683,7 @@ lightgreen 1 i386 green 3
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Regression.*i386.*Regression',
r'autopkgtest for lightgreen 1: .*amd64.*Regression.*i386.*Regression'])
@ -539,9 +732,7 @@ lightgreen 1 i386 green 3
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 2)'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
r'\blightgreen\b.*>1</a> to .*>2<',
r'autopkgtest for green 2: .*amd64.*Pass.*i386.*Pass',
@ -603,9 +794,7 @@ lightgreen 1 i386 green 3
self.do_test(
[('lightgreen', {'Version': '2'}, 'autopkgtest')],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\blightgreen\b.*>1</a> to .*>2<',
r'autopkgtest for lightgreen 2: .*amd64.*in progress.*i386.*in progress',
r'autopkgtest for rainbow 1: .*amd64.*in progress.*i386.*in progress'])
@ -618,9 +807,7 @@ lightgreen 1 i386 green 3
self.do_test(
[('dkms', {'Version': '2'}, None)],
# FIXME: while we only submit requests through AMQP, but don't consider
# their results, we don't expect this to hold back stuff.
VALID_CANDIDATE,
NOT_CONSIDERED,
[r'\bdkms\b.*>1</a> to .*>2<',
r'autopkgtest for fancy 1: .*amd64.*in progress.*i386.*in progress'])
@ -641,448 +828,5 @@ lightgreen 1 i386 green 3
self.assertEqual(self.pending_requests, None)
class TestAdtBritney(TestBase):
'''Legacy adt-britney/lp:auto-package-testing interface'''
def setUp(self):
super(TestAdtBritney, self).setUp()
# Mofify configuration according to the test context.
with open(self.britney_conf, 'r') as fp:
original_config = fp.read()
# Disable boottests.
new_config = original_config.replace(
'BOOTTEST_ENABLE = yes', 'BOOTTEST_ENABLE = no')
with open(self.britney_conf, 'w') as fp:
fp.write(new_config)
# fake adt-britney script
self.adt_britney = os.path.join(
self.data.home, 'auto-package-testing', 'jenkins', 'adt-britney')
os.makedirs(os.path.dirname(self.adt_britney))
with open(self.adt_britney, 'w') as f:
f.write('''#!/bin/sh -e
echo "$@" >> /%s/adt-britney.log ''' % self.data.path)
os.chmod(self.adt_britney, 0o755)
# add a bunch of packages to testing to avoid repetition
self.data.add('libc6', False)
self.data.add('libgreen1', False, {'Source': 'green',
'Depends': 'libc6 (>= 0.9)'})
self.data.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1',
'Conflicts': 'blue'})
self.data.add('lightgreen', False, {'Depends': 'libgreen1'})
self.data.add('darkgreen', False, {'Depends': 'libgreen1'})
self.data.add('blue', False, {'Depends': 'libc6 (>= 0.9)',
'Conflicts': 'green'})
self.data.add('justdata', False, {'Architecture': 'all'})
def __merge_records(self, results, history=""):
'''Merges a list of results with records in history.
This function merges results from a collect with records already in
history and sort records by version/name of causes and version/name of
source packages with tests. This should be done in the fake
adt-britney but it is more convenient to just pass a static list of
records and make adt-britney just return this list.
'''
if history is None:
history = ""
records = [x.split() for x in (results.strip() + '\n' +
history.strip()).split('\n') if x]
records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(4))
records.sort(key=operator.itemgetter(3))
records.sort(cmp=apt_pkg.version_compare, key=operator.itemgetter(1))
records.sort()
return "\n".join([' '.join(x) for x in records])
def make_adt_britney(self, request, history=""):
with open(self.adt_britney, 'w') as f:
f.write('''#!%(py)s
import argparse, shutil,sys
def request():
if args.req:
shutil.copy(args.req, '%(path)s/adt-britney.requestarg')
with open(args.output, 'w') as f:
f.write("""%(rq)s""".replace('PASS', 'NEW').replace('FAIL', 'NEW').replace('RUNNING', 'NEW'))
def submit():
with open(args.req, 'w') as f:
f.write("""%(rq)s""".replace('PASS', 'RUNNING').
replace('FAIL', 'RUNNING'))
def collect():
with open(args.output, 'w') as f:
f.write("""%(res)s""")
p = argparse.ArgumentParser()
p.add_argument('-c', '--config')
p.add_argument('-a', '--arch')
p.add_argument('-r', '--release')
p.add_argument('-P', '--use-proposed', action='store_true')
p.add_argument('-d', '--debug', action='store_true')
p.add_argument('-U', '--no-update', action='store_true')
sp = p.add_subparsers()
prequest = sp.add_parser('request')
prequest.add_argument('-O', '--output')
prequest.add_argument('req', nargs='?')
prequest.set_defaults(func=request)
psubmit = sp.add_parser('submit')
psubmit.add_argument('req')
psubmit.set_defaults(func=submit)
pcollect = sp.add_parser('collect')
pcollect.add_argument('-O', '--output')
pcollect.add_argument('-n', '--new-only', action='store_true', default=False)
pcollect.set_defaults(func=collect)
args = p.parse_args()
args.func()
''' % {'py': sys.executable, 'path': self.data.path,
'rq': request,
'res': self.__merge_records(request, history)})
def do_test(self, unstable_add, adt_request, considered, expect=None,
no_expect=None, history=""):
for (pkg, fields) in unstable_add:
self.data.add(pkg, True, fields)
self.make_adt_britney(adt_request, history)
(excuses, out) = self.run_britney()
#print('-------\nexcuses: %s\n-----' % excuses)
#print('-------\nout: %s\n-----' % out)
#print('run:\n%s -c %s\n' % (self.britney, self.britney_conf))
#subprocess.call(['bash', '-i'], cwd=self.data.path)
if considered:
self.assertIn('Valid candidate', excuses)
else:
self.assertIn('Not considered', excuses)
if expect:
for re in expect:
self.assertRegexpMatches(excuses, re)
if no_expect:
for re in no_expect:
self.assertNotRegexpMatches(excuses, re)
def test_no_request_for_uninstallable(self):
'''Does not request a test for an uninstallable package'''
self.do_test(
# uninstallable unstable version
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1 (>= 2)'})],
'green 1.1~beta RUNNING green 1.1~beta\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'green/amd64 unsatisfiable Depends: libgreen1 \(>= 2\)'],
# autopkgtest should not be triggered for uninstallable pkg
['autopkgtest'])
def test_request_for_installable_running(self):
'''Requests a test for an installable package, test still running'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'green 1.1~beta RUNNING green 1.1~beta\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']])
def test_request_for_installable_first_fail(self):
'''Requests a test for an installable package. No history and first result is a failure'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'green 1.1~beta FAIL green 1.1~beta\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
def test_request_for_installable_fail_regression(self):
'''Requests a test for an installable package, test fail'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'green 1.1~beta FAIL green 1.1~beta\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
history='green 1.0~beta PASS green 1.0~beta\n')
def test_request_for_installable_pass(self):
'''Requests a test for an installable package, test pass'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'green 1.1~beta PASS green 1.1~beta\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['PASS']])
def test_multi_rdepends_with_tests_running(self):
'''Multiple reverse dependencies with tests (still running)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 RUNNING green 2\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']])
def test_multi_rdepends_with_tests_fail_always(self):
'''Multiple reverse dependencies with tests (fail)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 FAIL green 2\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
def test_multi_rdepends_with_tests_fail_regression(self):
'''Multiple reverse dependencies with tests (fail)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 FAIL green 2\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
history='darkgreen 1 PASS green 1\n')
def test_multi_rdepends_with_tests_pass(self):
'''Multiple reverse dependencies with tests (pass)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 PASS green 2\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']])
def test_multi_rdepends_with_some_tests_running(self):
'''Multiple reverse dependencies with some tests (running)'''
# add a third reverse dependency to libgreen1 which does not have a test
self.data.add('mint', False, {'Depends': 'libgreen1'})
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 RUNNING green 2\n'
'darkgreen 1 RUNNING green 2\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']])
def test_multi_rdepends_with_some_tests_fail_always(self):
'''Multiple reverse dependencies with some tests (fail)'''
# add a third reverse dependency to libgreen1 which does not have a test
self.data.add('mint', False, {'Depends': 'libgreen1'})
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 FAIL green 2\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
def test_multi_rdepends_with_some_tests_fail_regression(self):
'''Multiple reverse dependencies with some tests (fail)'''
# add a third reverse dependency to libgreen1 which does not have a test
self.data.add('mint', False, {'Depends': 'libgreen1'})
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 FAIL green 2\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
history='darkgreen 1 PASS green 1\n')
def test_multi_rdepends_with_some_tests_pass(self):
'''Multiple reverse dependencies with some tests (pass)'''
# add a third reverse dependency to libgreen1 which does not have a test
self.data.add('mint', False, {'Depends': 'libgreen1'})
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'})],
'lightgreen 1 PASS green 2\n'
'darkgreen 1 PASS green 2\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']])
def test_binary_from_new_source_package_running(self):
'''building an existing binary for a new source package (running)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
'lightgreen 1 PASS newgreen 2\n'
'darkgreen 1 RUNNING newgreen 2\n',
NOT_CONSIDERED,
[r'\bnewgreen\b.*\(- to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['RUNNING']])
def test_binary_from_new_source_package_fail_always(self):
'''building an existing binary for a new source package (fail)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
'lightgreen 1 PASS newgreen 2\n'
'darkgreen 1 FAIL newgreen 2\n',
VALID_CANDIDATE,
[r'\bnewgreen\b.*\(- to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']])
def test_binary_from_new_source_package_fail_regression(self):
'''building an existing binary for a new source package (fail)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
'lightgreen 1 PASS newgreen 2\n'
'darkgreen 1 FAIL newgreen 2\n',
NOT_CONSIDERED,
[r'\bnewgreen\b.*\(- to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
history='darkgreen 1 PASS green 1\n')
def test_binary_from_new_source_package_pass(self):
'''building an existing binary for a new source package (pass)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6'})],
'lightgreen 1 PASS newgreen 2\n'
'darkgreen 1 PASS newgreen 2\n',
VALID_CANDIDATE,
[r'\bnewgreen\b.*\(- to .*>2<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS'],
'<li>autopkgtest for darkgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']])
def test_binary_from_new_source_package_uninst(self):
'''building an existing binary for a new source package (uninstallable)'''
self.do_test(
[('libgreen1', {'Version': '2', 'Source': 'newgreen', 'Depends': 'libc6, nosuchpkg'})],
'darkgreen 1 FAIL newgreen 2\n',
NOT_CONSIDERED,
[r'\bnewgreen\b.*\(- to .*>2<',
'libgreen1/amd64 unsatisfiable Depends: nosuchpkg'],
# autopkgtest should not be triggered for uninstallable pkg
['autopkgtest'])
@unittest.expectedFailure
def test_result_from_older_version(self):
'''test result from older version than the uploaded one'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'green 1.1~alpha PASS green 1.1~beta\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
# it's not entirely clear what precisely it should say here
'<li>autopkgtest for green 1.1~beta: %s' % ADT_EXCUSES_LABELS['RUNNING']])
def test_request_for_installable_fail_regression_promoted(self):
'''Requests a test for an installable package, test fail, is a regression.
This test verifies a bug in britney where a package was promoted if latest test
appeared before previous result in history, only the last result in
alphabetic order was taken into account. For example:
A 1 FAIL B 1
A 1 PASS A 1
In this case results for A 1 didn't appear in the list of results
triggered by the upload of B 1 and B 1 was promoted
'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'lightgreen 1 FAIL green 1.1~beta\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
history="lightgreen 1 PASS lightgreen 1"
)
def test_history_always_passed(self):
'''All the results in history are PASS, and test passed
'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'lightgreen 1 PASS green 1.1~beta\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['PASS']],
history="lightgreen 1 PASS lightgreen 1"
)
def test_history_always_failed(self):
'''All the results in history are FAIL, test fails. not a regression.
'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'lightgreen 1 FAIL green 1.1~beta\n',
VALID_CANDIDATE,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['ALWAYSFAIL']],
history="lightgreen 1 FAIL lightgreen 1"
)
def test_history_regression(self):
'''All the results in history are PASS, test fails. Blocked.
'''
self.do_test(
[('green', {'Version': '1.1~beta', 'Depends': 'libc6 (>= 0.9), libgreen1'})],
'lightgreen 1 FAIL green 1.1~beta\n',
NOT_CONSIDERED,
[r'\bgreen\b.*>1</a> to .*>1.1~beta<',
'<li>autopkgtest for lightgreen 1: %s' % ADT_EXCUSES_LABELS['REGRESSION']],
history="lightgreen 1 PASS lightgreen 1"
)
def shell(self):
# uninstallable unstable version
self.data.add('yellow', True, {'Version': '1.1~beta',
'Depends': 'libc6 (>= 0.9), nosuchpkg'})
self.make_adt_britney('yellow 1.1~beta RUNNING yellow 1.1~beta\n',
'purple 2 FAIL pink 3.0.~britney\n')
print('run:\n%s -c %s\n' % (self.britney, self.britney_conf))
subprocess.call(['bash', '-i'], cwd=self.data.path)
if __name__ == '__main__':
unittest.main()

44
tests/test_boottest.py
Unescape View File

@ -11,6 +11,7 @@ import os
import shutil
import sys
import tempfile
import fileinput
import unittest
@ -236,10 +237,11 @@ args.func()
def do_test(self, context, expect=None, no_expect=None):
"""Process the given package context and assert britney results."""
for (pkg, fields) in context:
self.data.add(pkg, True, fields)
self.data.add(pkg, True, fields, testsuite='autopkgtest')
self.make_boottest()
(excuses, out) = self.run_britney()
# print('-------\nexcuses: %s\n-----' % excuses)
# print('-------\nout: %s\n-----' % out)
if expect:
for re in expect:
self.assertRegexpMatches(excuses, re)
@ -329,14 +331,6 @@ args.func()
r'<li>Boottest result: UNKNOWN STATUS \(Jenkins: .*\)',
'<li>Not considered'])
def create_hint(self, username, content):
"""Populates a hint file for the given 'username' with 'content'."""
hints_path = os.path.join(
self.data.path,
'data/{}-proposed/Hints/{}'.format(self.data.series, username))
with open(hints_path, 'w') as fd:
fd.write(content)
def test_skipped_by_hints(self):
# `Britney` allows boottests to be skipped by hinting the
# corresponding source with 'force-skiptest'. The boottest
@ -420,37 +414,27 @@ args.func()
def test_with_adt(self):
# Boottest can run simultaneously with autopkgtest (adt).
# Enable ADT in britney configuration.
with open(self.britney_conf, 'r') as fp:
original_config = fp.read()
new_config = original_config.replace(
'ADT_ENABLE = no', 'ADT_ENABLE = yes')
with open(self.britney_conf, 'w') as fp:
fp.write(new_config)
fake_amqp = os.path.join(self.data.path, 'amqp')
# Create a fake 'adt-britney' that reports a RUNNING job for
# the testing source ('purple_1.1').
script_path = os.path.expanduser(
"~/auto-package-testing/jenkins/adt-britney")
os.makedirs(os.path.dirname(script_path))
with open(script_path, 'w') as f:
f.write('''#!/bin/sh -e
mkdir -p ~/proposed-migration/autopkgtest/work
touch ~/proposed-migration/autopkgtest/work/adt.request.series
echo "purple 1.1 RUNNING purple 1.1" >> ~/proposed-migration/autopkgtest/work/adt.result.series''')
os.chmod(script_path, 0o755)
# Enable ADT in britney configuration.
for line in fileinput.input(self.britney_conf, inplace=True):
if 'ADT_ENABLE' in line:
print('ADT_ENABLE = yes')
elif 'ADT_AMQP' in line:
print('ADT_AMQP = file://%s' % fake_amqp)
else:
sys.stdout.write(line)
# Britney blocks testing source promotion while ADT and boottest
# are running.
self.data.add('purple', False, {'Version': '1.0'})
self.data.add('purple', False, {'Version': '1.0'}, testsuite='autopkgtest')
context = [
('purple', {'Version': '1.1'}),
]
self.do_test(
context,
[r'\bpurple\b.*>1.0<.* to .*>1.1<',
'<li>autopkgtest for purple 1.1: {}'.format(
boottest.BootTest.EXCUSE_LABELS['RUNNING']),
'<li>autopkgtest for purple 1.1: .*amd64.*in progress.*i386.*in progress',
'<li>Boottest result: {}'.format(
boottest.BootTest.EXCUSE_LABELS['RUNNING']),
'<li>Not considered'])

Write Preview
Loading…
Cancel
Save