From c436aafda7d54e545b926989dbef0d6da0984dcb Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Wed, 20 May 2020 11:21:27 +0100 Subject: [PATCH] Implement Ubuntu component relationship constraints (ogre model) --- britney2/inputs/suiteloader.py | 20 +++--- britney2/policies/policy.py | 19 +++++- britney2/utils.py | 55 +++++++++++++++- tests/test_autopkgtest.py | 4 +- tests/test_util.py | 115 +++++++++++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 15 deletions(-) create mode 100755 tests/test_util.py diff --git a/britney2/inputs/suiteloader.py b/britney2/inputs/suiteloader.py index 2323e85..c03222f 100644 --- a/britney2/inputs/suiteloader.py +++ b/britney2/inputs/suiteloader.py @@ -7,7 +7,8 @@ import sys from britney2 import SuiteClass, Suite, TargetSuite, Suites, BinaryPackage, BinaryPackageId, SourcePackage from britney2.utils import ( - read_release_file, possibly_compressed, read_sources_file, create_provides_map, parse_provides, parse_builtusing + read_release_file, possibly_compressed, read_sources_file, create_provides_map, parse_provides, parse_builtusing, + UbuntuComponent ) @@ -206,7 +207,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): filename = os.path.join(basedir, component, "source", "Sources") filename = possibly_compressed(filename) self.logger.info("Loading source packages from %s", filename) - read_sources_file(filename, sources, component=component) + read_sources_file(filename, sources) else: filename = os.path.join(basedir, "Sources") self.logger.info("Loading source packages from %s", filename) @@ -287,7 +288,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): """ return separator.join(filter(None, (get_field(x) for x in field_names))) or None - def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern, component=None): + def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern): self.logger.info("Loading binary packages from %s", filename) if packages is None: @@ -302,6 +303,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): while step(): pkg = get_field('Package') version = get_field('Version') + section = get_field('Section') # There may be multiple versions of any arch:all packages # (in unstable) if some architectures have out-of-date @@ -363,6 +365,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): else: builtusing = [] + # XXX: Do the get_component thing in a much nicer way that can be upstreamed dpkg = BinaryPackage(version, intern(get_field('Section')), source, @@ -375,7 +378,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): ess, pkg_id, builtusing, - component, + UbuntuComponent.get_component(section), ) # if the source package is available in the distribution, then register this binary package @@ -390,6 +393,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): srcdist[source].binaries.add(pkg_id) # if the source package doesn't exist, create a fake one else: + # XXX: Do the get_component thing in a much nicer way that can be upstreamed srcdist[source] = SourcePackage(source, source_version, 'faux', @@ -400,7 +404,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): None, [], [], - component) + UbuntuComponent.get_component(section)) # add the resulting dictionary to the package list packages[pkg] = dpkg @@ -470,13 +474,11 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader): self._read_packages_file(filename, arch, suite.sources, - packages, - component=component) + packages) self._read_packages_file(udeb_filename, arch, suite.sources, - packages, - component=component) + packages) # create provides provides = create_provides_map(packages) binaries[arch] = packages diff --git a/britney2/policies/policy.py b/britney2/policies/policy.py index 45da46f..b6c19b6 100644 --- a/britney2/policies/policy.py +++ b/britney2/policies/policy.py @@ -898,7 +898,21 @@ class DependsPolicy(BasePolicy): needed_for_dep = set() for alternative in dep: - if target_suite.is_pkg_in_the_suite(alternative): + alt_bin = self._britney.all_binaries[alternative] + + component = binary_u.component + alt_component = alt_bin.component + + # Don't check components when testing PPAs, as they do not have this concept + if self.options.adt_ppas: + component = None + + # This relationship is good wrt. components if either the binary being + # considered doesn't have a component, or if the ogre model + # permits it (see UbuntuComponent) + relationship_is_allowed = component is None or component.allowed_component(alt_component) + + if target_suite.is_pkg_in_the_suite(alternative) and relationship_is_allowed: # dep can be satisfied in testing - ok is_ok = True elif alternative in my_bins: @@ -1745,6 +1759,7 @@ class LPBlockBugPolicy(BasePolicy): The dates are expressed as the number of seconds from the Unix epoch (1970-01-01 00:00:00 UTC). """ + def __init__(self, options, suite_info): super().__init__('block-bugs', options, suite_info, {SuiteClass.PRIMARY_SOURCE_SUITE}) @@ -1753,7 +1768,7 @@ class LPBlockBugPolicy(BasePolicy): self.blocks = {} # srcpkg -> [(bug, date), ...] filename = os.path.join(self.options.unstable, "Blocks") - self.log("Loading user-supplied block data from %s" % filename) + self.logger.info("Loading user-supplied block data from %s" % filename) for line in open(filename): ln = line.split() if len(ln) != 3: diff --git a/britney2/utils.py b/britney2/utils.py index 96f7654..dfc5c54 100644 --- a/britney2/utils.py +++ b/britney2/utils.py @@ -29,6 +29,7 @@ import sys import time from collections import defaultdict from datetime import datetime +from enum import Enum, unique from functools import partial from itertools import filterfalse, chain @@ -527,7 +528,7 @@ def read_release_file(suite_dir): return result -def read_sources_file(filename, sources=None, intern=sys.intern, component=None): +def read_sources_file(filename, sources=None, intern=sys.intern): """Parse a single Sources file into a hash Parse a single Sources file into a dict mapping a source package @@ -575,6 +576,7 @@ def read_sources_file(filename, sources=None, intern=sys.intern, component=None) build_deps_indep = get_field('Build-Depends-Indep') if build_deps_indep is not None: build_deps_indep = sys.intern(build_deps_indep) + # XXX: Do the get_component thing in a much nicer way that can be upstreamed sources[intern(pkg)] = SourcePackage(intern(pkg), intern(ver), section, @@ -585,7 +587,7 @@ def read_sources_file(filename, sources=None, intern=sys.intern, component=None) build_deps_indep, get_field('Testsuite', '').split(), get_field('Testsuite-Triggers', '').replace(',', '').split(), - component, + UbuntuComponent.get_component(section), ) return sources @@ -955,3 +957,52 @@ def parse_builtusing(builtusing_raw, pkg_id=None, logger=None): part = (bu, bu_version) nbu.append(part) return nbu + + +@unique +class UbuntuComponent(Enum): + MAIN = 0 + RESTRICTED = 1 + UNIVERSE = 2 + MULTIVERSE = 3 + + @classmethod + def get_component(cls, section): + """Parse section and return component + + Given a section, return component. Packages in MAIN have no + prefix, all others have / prefix. + """ + name2component = { + "restricted": cls.RESTRICTED, + "universe": cls.UNIVERSE, + "multiverse": cls.MULTIVERSE, + } + + if "/" in section: + return name2component[section.split("/", 1)[0]] + + return cls.MAIN + + def allowed_component(self, dep): + """Check if I can depend on the other component""" + + component_dependencies = { + UbuntuComponent.MAIN: [UbuntuComponent.MAIN], + UbuntuComponent.RESTRICTED: [ + UbuntuComponent.MAIN, + UbuntuComponent.RESTRICTED, + ], + UbuntuComponent.UNIVERSE: [ + UbuntuComponent.MAIN, + UbuntuComponent.UNIVERSE, + ], + UbuntuComponent.MULTIVERSE: [ + UbuntuComponent.MAIN, + UbuntuComponent.RESTRICTED, + UbuntuComponent.UNIVERSE, + UbuntuComponent.MULTIVERSE, + ], + } + + return dep in component_dependencies[self] diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index ee83506..c9416f7 100644 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -2191,12 +2191,12 @@ class AT(TestAutopkgtestBase): self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'], {'lightgreen': { 'amd64': ['RUNNING-ALWAYSFAIL', - 'https://autopkgtest.ubuntu.com/status/pending', + 'https://autopkgtest.ubuntu.com/running', None, None, None], 'i386': ['RUNNING-ALWAYSFAIL', - 'https://autopkgtest.ubuntu.com/status/pending', + 'https://autopkgtest.ubuntu.com/running', None, None, None]}, diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100755 index 0000000..6806dd3 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,115 @@ +#!/usr/bin/python3 +# (C) 2014 - 2016 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import sys +import unittest + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PROJECT_DIR) + +from britney2.utils import UbuntuComponent # noqa: E402 + + +class UtilTests(unittest.TestCase): + def test_get_component(self): + self.assertEqual( + UbuntuComponent.get_component("utils"), UbuntuComponent.MAIN + ) + self.assertEqual( + UbuntuComponent.get_component("utils"), UbuntuComponent.MAIN + ) + self.assertEqual( + UbuntuComponent.get_component("restricted/admin"), + UbuntuComponent.RESTRICTED, + ) + self.assertEqual( + UbuntuComponent.get_component("universe/web"), + UbuntuComponent.UNIVERSE, + ) + self.assertEqual( + UbuntuComponent.get_component("multiverse/libs"), + UbuntuComponent.MULTIVERSE, + ) + + def test_allowed_component(self): + allowed_component = UbuntuComponent.allowed_component + + self.assertTrue( + allowed_component(UbuntuComponent.MAIN, UbuntuComponent.MAIN) + ) + self.assertFalse( + allowed_component(UbuntuComponent.MAIN, UbuntuComponent.UNIVERSE) + ) + self.assertFalse( + allowed_component(UbuntuComponent.MAIN, UbuntuComponent.MULTIVERSE) + ) + self.assertFalse( + allowed_component(UbuntuComponent.MAIN, UbuntuComponent.RESTRICTED) + ) + + self.assertTrue( + allowed_component(UbuntuComponent.RESTRICTED, UbuntuComponent.MAIN) + ) + self.assertFalse( + allowed_component( + UbuntuComponent.RESTRICTED, UbuntuComponent.UNIVERSE + ) + ) + self.assertFalse( + allowed_component( + UbuntuComponent.RESTRICTED, UbuntuComponent.MULTIVERSE + ) + ) + self.assertTrue( + allowed_component( + UbuntuComponent.RESTRICTED, UbuntuComponent.RESTRICTED + ) + ) + + self.assertTrue( + allowed_component(UbuntuComponent.UNIVERSE, UbuntuComponent.MAIN) + ) + self.assertTrue( + allowed_component( + UbuntuComponent.UNIVERSE, UbuntuComponent.UNIVERSE + ) + ) + self.assertFalse( + allowed_component( + UbuntuComponent.UNIVERSE, UbuntuComponent.MULTIVERSE + ) + ) + self.assertFalse( + allowed_component( + UbuntuComponent.UNIVERSE, UbuntuComponent.RESTRICTED + ) + ) + + self.assertTrue( + allowed_component(UbuntuComponent.MULTIVERSE, UbuntuComponent.MAIN) + ) + self.assertTrue( + allowed_component( + UbuntuComponent.MULTIVERSE, UbuntuComponent.UNIVERSE + ) + ) + self.assertTrue( + allowed_component( + UbuntuComponent.MULTIVERSE, UbuntuComponent.MULTIVERSE + ) + ) + self.assertTrue( + allowed_component( + UbuntuComponent.MULTIVERSE, UbuntuComponent.RESTRICTED + ) + ) + + +if __name__ == "__main__": + unittest.main()