Implement Ubuntu component relationship constraints (ogre model)

britney2/inputs/suiteloader.py

@@ -8,7 +8,8 @@ import sys
 
 from britney2 import SuiteClass, Suite, TargetSuite, Suites, BinaryPackage, BinaryPackageId, SourcePackage
 from britney2.utils import (
-    read_release_file, possibly_compressed, read_sources_file, create_provides_map, parse_provides, parse_builtusing
+    read_release_file, possibly_compressed, read_sources_file, create_provides_map, parse_provides, parse_builtusing,
+    UbuntuComponent
 )
 
 
@@ -216,7 +217,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
                 filename = os.path.join(basedir, component, "source", "Sources")
                 filename = possibly_compressed(filename)
                 self.logger.info("Loading source packages from %s", filename)
-                read_sources_file(filename, sources, component=component)
+                read_sources_file(filename, sources)
         else:
             filename = os.path.join(basedir, "Sources")
             self.logger.info("Loading source packages from %s", filename)
@@ -297,7 +298,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
         """
         return separator.join(filter(None, (get_field(x) for x in field_names))) or None
 
-    def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern, component=None):
+    def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern):
         self.logger.info("Loading binary packages from %s", filename)
 
         if packages is None:
@@ -312,6 +313,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
         while step():
             pkg = get_field('Package')
             version = get_field('Version')
+            section = get_field('Section')
 
             # There may be multiple versions of any arch:all packages
             # (in unstable) if some architectures have out-of-date
@@ -373,6 +375,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
             else:
                 builtusing = []
 
+            # XXX: Do the get_component thing in a much nicer way that can be upstreamed
             dpkg = BinaryPackage(version,
                                  intern(get_field('Section')),
                                  source,
@@ -385,7 +388,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
                                  ess,
                                  pkg_id,
                                  builtusing,
-                                 component,
+                                 UbuntuComponent.get_component(section),
                                  )
 
             # if the source package is available in the distribution, then register this binary package
@@ -400,6 +403,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
                     srcdist[source].binaries.add(pkg_id)
             # if the source package doesn't exist, create a fake one
             else:
+                # XXX: Do the get_component thing in a much nicer way that can be upstreamed
                 srcdist[source] = SourcePackage(source,
                                                 source_version,
                                                 'faux',
@@ -410,7 +414,7 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
                                                 None,
                                                 [],
                                                 [],
-                                                component)
+                                                UbuntuComponent.get_component(section))
 
             # add the resulting dictionary to the package list
             packages[pkg] = dpkg
@@ -480,13 +484,11 @@ class DebMirrorLikeSuiteContentLoader(SuiteContentLoader):
                     self._read_packages_file(filename,
                                              arch,
                                              suite.sources,
-                                             packages,
+                                             packages)
-                                             component=component)
                     self._read_packages_file(udeb_filename,
                                              arch,
                                              suite.sources,
-                                             packages,
+                                             packages)
-                                             component=component)
                 # create provides
                 provides = create_provides_map(packages)
                 binaries[arch] = packages

britney2/policies/policy.py

@@ -896,9 +896,34 @@ class DependsPolicy(BasePolicy):
                     continue
                 is_ok = False
                 needed_for_dep = set()
+                any_relationship_allowed = False
+                relationship_not_allowed_reasons = []
 
                 for alternative in dep:
-                    if target_suite.is_pkg_in_the_suite(alternative):
+                    alt_bin = self._britney.all_binaries[alternative]
+
+                    component = binary_u.component
+                    alt_component = alt_bin.component
+
+                    # Don't check components when testing PPAs, as they do not have this concept
+                    if self.options.adt_ppas:
+                        component = None
+
+                    # This relationship is good wrt. components if either the binary being
+                    # considered doesn't have a component, or if the ogre model
+                    # permits it (see UbuntuComponent)
+                    relationship_is_allowed = component is None or component.allowed_component(alt_component)
+                    any_relationship_allowed = any_relationship_allowed or relationship_is_allowed
+
+                    if not relationship_is_allowed:
+                        relationship_not_allowed_reasons.append('%s/%s in %s cannot depend on %s in %s' %
+                                                                (pkg_name,
+                                                                 arch,
+                                                                 component.value,
+                                                                 alternative.package_name,
+                                                                 alt_component.value))
+
+                    if target_suite.is_pkg_in_the_suite(alternative) and relationship_is_allowed:
                         # dep can be satisfied in testing - ok
                         is_ok = True
                     elif alternative in my_bins:
@@ -912,6 +937,11 @@ class DependsPolicy(BasePolicy):
                     spec = DependencySpec(DependencyType.DEPENDS, arch)
                     excuse.add_package_depends(spec, needed_for_dep)
 
+                if not any_relationship_allowed:
+                    verdict = PolicyVerdict.REJECTED_PERMANENTLY
+                    for reason in relationship_not_allowed_reasons:
+                        excuse.add_verdict_info(verdict, reason)
+
         return verdict
 
 
@@ -1745,6 +1775,7 @@ class LPBlockBugPolicy(BasePolicy):
     The dates are expressed as the number of seconds from the Unix epoch
     (1970-01-01 00:00:00 UTC).
     """
+
     def __init__(self, options, suite_info):
         super().__init__('block-bugs', options, suite_info, {SuiteClass.PRIMARY_SOURCE_SUITE})
 
@@ -1753,7 +1784,7 @@ class LPBlockBugPolicy(BasePolicy):
         self.blocks = {}  # srcpkg -> [(bug, date), ...]
 
         filename = os.path.join(self.options.unstable, "Blocks")
-        self.log("Loading user-supplied block data from %s" % filename)
+        self.logger.info("Loading user-supplied block data from %s" % filename)
         for line in open(filename):
             ln = line.split()
             if len(ln) != 3:

britney2/utils.py

@@ -29,6 +29,7 @@ import sys
 import time
 from collections import defaultdict
 from datetime import datetime
+from enum import Enum, unique
 from functools import partial
 from itertools import filterfalse, chain
 
@@ -527,7 +528,7 @@ def read_release_file(suite_dir):
     return result
 
 
-def read_sources_file(filename, sources=None, intern=sys.intern, component=None):
+def read_sources_file(filename, sources=None, intern=sys.intern):
     """Parse a single Sources file into a hash
 
     Parse a single Sources file into a dict mapping a source package
@@ -575,6 +576,7 @@ def read_sources_file(filename, sources=None, intern=sys.intern, component=None)
         build_deps_indep = get_field('Build-Depends-Indep')
         if build_deps_indep is not None:
             build_deps_indep = sys.intern(build_deps_indep)
+        # XXX: Do the get_component thing in a much nicer way that can be upstreamed
         sources[intern(pkg)] = SourcePackage(intern(pkg),
                                              intern(ver),
                                              section,
@@ -585,7 +587,7 @@ def read_sources_file(filename, sources=None, intern=sys.intern, component=None)
                                              build_deps_indep,
                                              get_field('Testsuite', '').split(),
                                              get_field('Testsuite-Triggers', '').replace(',', '').split(),
-                                             component,
+                                             UbuntuComponent.get_component(section),
                                              )
     return sources
 
@@ -955,3 +957,52 @@ def parse_builtusing(builtusing_raw, pkg_id=None, logger=None):
             part = (bu, bu_version)
             nbu.append(part)
     return nbu
+
+
+@unique
+class UbuntuComponent(Enum):
+    MAIN = 'main'
+    RESTRICTED = 'restricted'
+    UNIVERSE = 'universe'
+    MULTIVERSE = 'multiverse'
+
+    @classmethod
+    def get_component(cls, section):
+        """Parse section and return component
+
+        Given a section, return component. Packages in MAIN have no
+        prefix, all others have <component>/ prefix.
+        """
+        name2component = {
+            "restricted": cls.RESTRICTED,
+            "universe": cls.UNIVERSE,
+            "multiverse": cls.MULTIVERSE,
+        }
+
+        if "/" in section:
+            return name2component[section.split("/", 1)[0]]
+
+        return cls.MAIN
+
+    def allowed_component(self, dep):
+        """Check if I can depend on the other component"""
+
+        component_dependencies = {
+            UbuntuComponent.MAIN: [UbuntuComponent.MAIN],
+            UbuntuComponent.RESTRICTED: [
+                UbuntuComponent.MAIN,
+                UbuntuComponent.RESTRICTED,
+            ],
+            UbuntuComponent.UNIVERSE: [
+                UbuntuComponent.MAIN,
+                UbuntuComponent.UNIVERSE,
+            ],
+            UbuntuComponent.MULTIVERSE: [
+                UbuntuComponent.MAIN,
+                UbuntuComponent.RESTRICTED,
+                UbuntuComponent.UNIVERSE,
+                UbuntuComponent.MULTIVERSE,
+            ],
+        }
+
+        return dep in component_dependencies[self]

tests/test_autopkgtest.py

@@ -2191,12 +2191,12 @@ class AT(TestAutopkgtestBase):
         self.assertEqual(exc['lightgreen']['policy_info']['autopkgtest'],
                          {'lightgreen': {
                              'amd64': ['RUNNING-ALWAYSFAIL',
-                                       'https://autopkgtest.ubuntu.com/status/pending',
+                                       'https://autopkgtest.ubuntu.com/running',
                                        None,
                                        None,
                                        None],
                              'i386': ['RUNNING-ALWAYSFAIL',
-                                      'https://autopkgtest.ubuntu.com/status/pending',
+                                      'https://autopkgtest.ubuntu.com/running',
                                       None,
                                       None,
                                       None]},

tests/test_util.py

@@ -0,0 +1,115 @@
+#!/usr/bin/python3
+# (C) 2014 - 2016 Canonical Ltd.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+import os
+import sys
+import unittest
+
+PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_DIR)
+
+from britney2.utils import UbuntuComponent  # noqa: E402
+
+
+class UtilTests(unittest.TestCase):
+    def test_get_component(self):
+        self.assertEqual(
+            UbuntuComponent.get_component("utils"), UbuntuComponent.MAIN
+        )
+        self.assertEqual(
+            UbuntuComponent.get_component("utils"), UbuntuComponent.MAIN
+        )
+        self.assertEqual(
+            UbuntuComponent.get_component("restricted/admin"),
+            UbuntuComponent.RESTRICTED,
+        )
+        self.assertEqual(
+            UbuntuComponent.get_component("universe/web"),
+            UbuntuComponent.UNIVERSE,
+        )
+        self.assertEqual(
+            UbuntuComponent.get_component("multiverse/libs"),
+            UbuntuComponent.MULTIVERSE,
+        )
+
+    def test_allowed_component(self):
+        allowed_component = UbuntuComponent.allowed_component
+
+        self.assertTrue(
+            allowed_component(UbuntuComponent.MAIN, UbuntuComponent.MAIN)
+        )
+        self.assertFalse(
+            allowed_component(UbuntuComponent.MAIN, UbuntuComponent.UNIVERSE)
+        )
+        self.assertFalse(
+            allowed_component(UbuntuComponent.MAIN, UbuntuComponent.MULTIVERSE)
+        )
+        self.assertFalse(
+            allowed_component(UbuntuComponent.MAIN, UbuntuComponent.RESTRICTED)
+        )
+
+        self.assertTrue(
+            allowed_component(UbuntuComponent.RESTRICTED, UbuntuComponent.MAIN)
+        )
+        self.assertFalse(
+            allowed_component(
+                UbuntuComponent.RESTRICTED, UbuntuComponent.UNIVERSE
+            )
+        )
+        self.assertFalse(
+            allowed_component(
+                UbuntuComponent.RESTRICTED, UbuntuComponent.MULTIVERSE
+            )
+        )
+        self.assertTrue(
+            allowed_component(
+                UbuntuComponent.RESTRICTED, UbuntuComponent.RESTRICTED
+            )
+        )
+
+        self.assertTrue(
+            allowed_component(UbuntuComponent.UNIVERSE, UbuntuComponent.MAIN)
+        )
+        self.assertTrue(
+            allowed_component(
+                UbuntuComponent.UNIVERSE, UbuntuComponent.UNIVERSE
+            )
+        )
+        self.assertFalse(
+            allowed_component(
+                UbuntuComponent.UNIVERSE, UbuntuComponent.MULTIVERSE
+            )
+        )
+        self.assertFalse(
+            allowed_component(
+                UbuntuComponent.UNIVERSE, UbuntuComponent.RESTRICTED
+            )
+        )
+
+        self.assertTrue(
+            allowed_component(UbuntuComponent.MULTIVERSE, UbuntuComponent.MAIN)
+        )
+        self.assertTrue(
+            allowed_component(
+                UbuntuComponent.MULTIVERSE, UbuntuComponent.UNIVERSE
+            )
+        )
+        self.assertTrue(
+            allowed_component(
+                UbuntuComponent.MULTIVERSE, UbuntuComponent.MULTIVERSE
+            )
+        )
+        self.assertTrue(
+            allowed_component(
+                UbuntuComponent.MULTIVERSE, UbuntuComponent.RESTRICTED
+            )
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()