Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-11-04 02:24:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

autopkgtest: Add new force-reset-test hint type

Browse Source 
This allows hinters to reset the baseline for a package/arch, so that
previous failures are ignore and not considered regressions
This commit is contained in:
Iain Lane 2020-06-10 10:30:13 +01:00 committed by Iain Lane
parent 27c2476e8f
commit 15f431d37c
2 changed files with 282 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
britney2/policies/autopkgtest.py
View File

@ -158,6 +158,7 @@ class AutopkgtestPolicy(BasePolicy):
def register_hints(self, hint_parser):
hint_parser.register_hint_type('force-badtest', britney2.hints.split_into_one_hint_per_package)
hint_parser.register_hint_type('force-skiptest', britney2.hints.split_into_one_hint_per_package)
hint_parser.register_hint_type('force-reset-test', britney2.hints.split_into_one_hint_per_package)
def initialise(self, britney):
super().initialise(britney)
@ -1096,6 +1097,11 @@ class AutopkgtestPolicy(BasePolicy):
# determine current test result status
baseline_result = self.result_in_baseline(src, arch)[0]
# determine current test result status
until = self.find_max_lower_force_reset_test(src, ver, arch)
ever_passed = self.check_ever_passed_before(src, ver, arch, until)
fail_result = 'REGRESSION' if ever_passed else 'ALWAYSFAIL'
url = None
run_id = None
@ -1136,9 +1142,13 @@ class AutopkgtestPolicy(BasePolicy):
else:
result = 'ALWAYSFAIL'
else:
result = 'REGRESSION'
result = fail_result
else:
result = 'REGRESSION'
result = fail_result
if result == 'REGRESSION' and \
self.has_higher_force_reset_test(src, ver, arch):
result = 'ALWAYSFAIL'
if self.has_force_badtest(src, ver, arch):
result = 'IGNORE-FAIL'
@ -1181,6 +1191,59 @@ class AutopkgtestPolicy(BasePolicy):
return (result, ver, run_id, url)
def check_ever_passed_before(self, src, max_ver, arch, min_ver=None):
'''Check if tests for src ever passed on arch for specified range
If min_ver is specified, it checks that all versions in
[min_ver, max_ver) have passed; otherwise it checks that
[min_ver, inf) have passed.'''
# FIXME: add caching
for srcmap in self.test_results.values():
try:
too_high = apt_pkg.version_compare(srcmap[src][arch][1], max_ver) > 0
too_low = apt_pkg.version_compare(srcmap[src][arch][1], min_ver) <= 0 if min_ver else False
if too_high or too_low:
continue
if srcmap[src][arch][0] in (Result.PASS, Result.OLD_PASS):
return True
except KeyError:
pass
return False
def find_max_lower_force_reset_test(self, src, ver, arch):
'''Find the maximum force-reset-test hint before/including ver'''
hints = self.hints.search('force-reset-test', package=src)
found_ver = None
if hints:
for hint in hints:
for mi in hint.packages:
if (mi.architecture in ['source', arch] and
mi.version != 'all' and
apt_pkg.version_compare(mi.version, ver) <= 0 and
(found_ver is None or apt_pkg.version_compare(found_ver, mi.version) < 0)):
found_ver = mi.version
return found_ver
def has_higher_force_reset_test(self, src, ver, arch):
'''Find if there is a minimum force-reset-test hint after/including ver'''
hints = self.hints.search('force-reset-test', package=src)
if hints:
self.logger.info('Checking hints for %s/%s/%s: %s' % (src, ver, arch, [str(h) for h in hints]))
for hint in hints:
for mi in hint.packages:
if (mi.architecture in ['source', arch] and
mi.version != 'all' and
apt_pkg.version_compare(mi.version, ver) >= 0):
return True
return False
def has_force_badtest(self, src, ver, arch):
'''Check if src/ver/arch has a force-badtest hint'''

217
tests/test_autopkgtest.py
View File

@ -1985,6 +1985,223 @@ class AT(TestAutopkgtestBase):
{'lightgreen': [('reason', 'block')]}
)
def test_hint_force_reset_test_goodbad_alwaysfail(self):
'''force-reset-test hint marks as alwaysfail'''
self.data.add_default_packages(lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
self.run_it(
[('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'lightgreen': (True, {
'lightgreen/2': {'amd64': 'ALWAYSFAIL'},
}),
},
{'lightgreen': [('old-version', '1'), ('new-version', '2')]
})
def test_hint_force_reset_test_goodbad_alwaysfail_arch(self):
'''force-reset-test hint marks as alwaysfail per arch'''
self.data.add_default_packages(lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
'testing/i386/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/i386/l/lightgreen/20150101_100101@': (4, 'lightgreen 2', tr('lightgreen/2')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/1/amd64')
self.run_it(
[('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'lightgreen': (False, {
'lightgreen/2': {'amd64': 'ALWAYSFAIL', 'i386': 'REGRESSION'},
}),
},
{'lightgreen': [('old-version', '1'), ('new-version', '2')]
})
def test_hint_force_reset_test_bad_good_pass(self):
'''force-reset-test hint followed by pass is pass'''
self.data.add_default_packages(lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
self.run_it(
[('lightgreen', {'Version': '2', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'lightgreen': (True, {
'lightgreen/2': {'amd64': 'PASS'},
}),
},
{'lightgreen': [('old-version', '1'), ('new-version', '2')]
})
def test_hint_force_reset_test_bad_good_bad_regression(self):
'''force-reset-test hint followed by good, bad is regression'''
self.data.add_default_packages(lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
'testing/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 3', tr('lightgreen/3')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
self.run_it(
[('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'lightgreen': (False, {
'lightgreen/3': {'amd64': 'REGRESSION'},
}),
},
{'lightgreen': [('old-version', '1'), ('new-version', '3')]
})
def test_hint_force_reset_test_bad_good_bad_regression_different_trigger(self):
'''force-reset-test hint followed by good, bad is regression (not self-triggered)'''
self.data.add_default_packages(green=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100100@': (4, 'lightgreen 0.1', tr('lightgreen/0.1')),
'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150103_100101@': (4, 'lightgreen 1', tr('green/2')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/0.1')
self.run_it(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (False, {
'lightgreen/1': {'amd64': 'REGRESSION'},
}),
},
{'green': [('old-version', '1'), ('new-version', '2')]
})
def test_hint_force_reset_test_multiple_hints(self):
'''force-reset-test multiple hints check ranges'''
self.data.add_default_packages(green=False, lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100100@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150102_100100@': (4, 'lightgreen 1', tr('green/2')),
'testing/amd64/l/lightgreen/20150103_100101@': (0, 'lightgreen 2', tr('lightgreen/2')),
'testing/amd64/l/lightgreen/20150104_100101@': (4, 'lightgreen 3', tr('lightgreen/3')),
}})
self.run_it(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (False, {
'lightgreen/1': {'amd64': 'REGRESSION'},
}),
},
{'green': [('old-version', '1'), ('new-version', '2')],
'lightgreen': [('old-version', '1'), ('new-version', '3')],
})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/1')
self.run_it(
[],
{'green': (True, {
'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
}),
'lightgreen': (False, {
'lightgreen/3': {'amd64': 'REGRESSION'},
}),
},
{'green': [('old-version', '1'), ('new-version', '2')],
'lightgreen': [('old-version', '1'), ('new-version', '3')],
})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/3')
self.run_it(
[],
{'green': (True, {
'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
}),
'lightgreen': (True, {
'lightgreen/3': {'amd64': 'ALWAYSFAIL'},
}),
},
{'green': [('old-version', '1'), ('new-version', '2')],
'lightgreen': [('old-version', '1'), ('new-version', '3')],
})
def test_hint_force_reset_test_earlier_hints(self):
'''force-reset-test for a later version applies backwards'''
self.data.add_default_packages(green=False, lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150102_100101@': (4, 'lightgreen 1', tr('green/2')),
'testing/amd64/l/lightgreen/20150103_100102@': (0, 'lightgreen 2', tr('lightgreen/2')),
'testing/amd64/l/lightgreen/20150104_100102@': (4, 'lightgreen 3', tr('lightgreen/3')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/3')
self.run_it(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (True, {
'lightgreen/1': {'amd64': 'ALWAYSFAIL'},
}),
'lightgreen': (True, {
'lightgreen/3': {'amd64': 'ALWAYSFAIL'},
}),
},
{'green': [('old-version', '1'), ('new-version', '2')],
'lightgreen': [('old-version', '1'), ('new-version', '3')],
})
def test_hint_force_reset_test_earlier_hints_pass(self):
'''force-reset-test for a later version which is PASS is still PASS'''
self.data.add_default_packages(green=False, lightgreen=False)
self.swift.set_results({'autopkgtest-testing': {
'testing/amd64/l/lightgreen/20150101_100101@': (0, 'lightgreen 1', tr('lightgreen/1')),
'testing/amd64/l/lightgreen/20150102_100101@': (0, 'lightgreen 1', tr('green/2')),
'testing/amd64/l/lightgreen/20150103_100102@': (0, 'lightgreen 2', tr('lightgreen/2')),
'testing/amd64/l/lightgreen/20150104_100102@': (0, 'lightgreen 3', tr('lightgreen/3')),
}})
self.create_hint('autopkgtest', 'force-reset-test lightgreen/3')
self.run_it(
[('libgreen1', {'Version': '2', 'Source': 'green', 'Depends': 'libc6'}, 'autopkgtest'),
('lightgreen', {'Version': '3', 'Source': 'lightgreen', 'Depends': 'libc6'}, 'autopkgtest')],
{'green': (True, {
'lightgreen/1': {'amd64': 'PASS'},
}),
'lightgreen': (True, {
'lightgreen/3': {'amd64': 'PASS'},
}),
},
{'green': [('old-version', '1'), ('new-version', '2')],
'lightgreen': [('old-version', '1'), ('new-version', '3')],
})
################################################################
# Kernel related tests
################################################################