britney.py: Add support for multiple components

Adds a --components command line argument (and corresponding config file
option). If specified, package info is expected to be in the usual Debian
mirror layout, ie:

   testing/source/Sources
   testing/binary-${ARCH}/Packages

(nthykier: Squashed, rebased and did some porting to Python3)
Signed-off-by: Niels Thykier <niels@thykier.net>

britney.py

@@ -387,6 +387,8 @@ class Britney(object):
                                help="do not build the non-installability status, use the cache from file")
         parser.add_option("", "--print-uninst", action="store_true", dest="print_uninst", default=False,
                                help="just print a summary of uninstallable packages")
+        parser.add_option("", "--components", action="store", dest="components",
+                               help="Sources/Packages are laid out by components listed (, sep)")
         (self.options, self.args) = parser.parse_args()
         
         # integrity checks
@@ -417,6 +419,11 @@ class Britney(object):
                          not getattr(self.options, k.lower()):
                         setattr(self.options, k.lower(), v)
 
+        if getattr(self.options, "components", None):
+            self.options.components = [s.strip() for s in self.options.components.split(",")]
+        else:
+            self.options.components = None
+
         if not hasattr(self.options, "heidi_delta_output"):
             self.options.heidi_delta_output = self.options.heidi_output + "Delta"
 
@@ -543,19 +550,10 @@ class Britney(object):
     # Data reading/writing methods
     # ----------------------------
 
-    def read_sources(self, basedir, intern=sys.intern):
+    def _read_sources_file(self, filename, sources=None, intern=sys.intern):
-        """Read the list of source packages from the specified directory
+        if sources is None:
-        
+            sources = {}
-        The source packages are read from the `Sources' file within the
-        directory specified as `basedir' parameter. Considering the
-        large amount of memory needed, not all the fields are loaded
-        in memory. The available fields are Version, Maintainer and Section.
 
-        The method returns a list where every item represents a source
-        package as a dictionary.
-        """
-        sources = {}
-        filename = os.path.join(basedir, "Sources")
         self.__log("Loading source packages from %s" % filename)
 
         with open(filename, encoding='utf-8') as f:
@@ -583,38 +581,37 @@ class Britney(object):
                            ]
         return sources
 
-    def read_binaries(self, basedir, distribution, arch, intern=sys.intern):
+    def read_sources(self, basedir):
-        """Read the list of binary packages from the specified directory
+        """Read the list of source packages from the specified directory
 
-        The binary packages are read from the `Packages_${arch}' files
+        The source packages are read from the `Sources' file within the
-        within the directory specified as `basedir' parameter, replacing
+        directory specified as `basedir' parameter. Considering the
-        ${arch} with the value of the arch parameter. Considering the
         large amount of memory needed, not all the fields are loaded
-        in memory. The available fields are Version, Source, Multi-Arch,
+        in memory. The available fields are Version, Maintainer and Section.
-        Depends, Conflicts, Provides and Architecture.
-        
-        After reading the packages, reverse dependencies are computed
-        and saved in the `rdepends' keys, and the `Provides' field is
-        used to populate the virtual packages list.
-
-        The dependencies are parsed with the apt_pkg.parse_depends method,
-        and they are stored both as the format of its return value and
-        text.
 
-        The method returns a tuple. The first element is a list where
+        The method returns a list where every item represents a source
-        every item represents a binary package as a dictionary; the second
+        package as a dictionary.
-        element is a dictionary which maps virtual packages to real
-        packages that provide them.
         """
 
-        packages = {}
+        if self.options.components:
-        provides = defaultdict(set)
+            sources = {}
-        sources = self.sources
+            for component in self.options.components:
-        all_binaries = self.all_binaries
+                filename = os.path.join(basedir, component, "source", "Sources")
+                self._read_sources_file(filename, sources)
+        else:
+            filename = os.path.join(basedir, "Sources")
+            sources = self._read_sources_file(filename)
+
+        return sources
 
-        filename = os.path.join(basedir, "Packages_%s" % arch)
+    def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern):
         self.__log("Loading binary packages from %s" % filename)
 
+        if packages is None:
+            packages = {}
+
+        all_binaries = self.all_binaries
+
         with open(filename, encoding='utf-8') as f:
             Packages = apt_pkg.TagFile(f)
         get_field = Packages.section.get
@@ -675,20 +672,19 @@ class Britney(object):
                     dpkg[SOURCEVER] = intern(source[source.find("(")+1:source.find(")")])
 
             # if the source package is available in the distribution, then register this binary package
-            if dpkg[SOURCE] in sources[distribution]:
+            if dpkg[SOURCE] in srcdist:
                 # There may be multiple versions of any arch:all packages
                 # (in unstable) if some architectures have out-of-date
                 # binaries.  We only want to include the package in the
                 # source -> binary mapping once. It doesn't matter which
                 # of the versions we include as only the package name and
                 # architecture are recorded.
-                if pkg_id not in sources[distribution][dpkg[SOURCE]][BINARIES]:
+                if pkg_id not in srcdist[dpkg[SOURCE]][BINARIES]:
-                    sources[distribution][dpkg[SOURCE]][BINARIES].append(pkg_id)
+                    srcdist[dpkg[SOURCE]][BINARIES].append(pkg_id)
             # if the source package doesn't exist, create a fake one
             else:
-                sources[distribution][dpkg[SOURCE]] = [dpkg[SOURCEVER], 'faux', [pkg_id], None, True]
+                srcdist[dpkg[SOURCE]] = [dpkg[SOURCEVER], 'faux', [pkg_id], None, True]
 
-            # register virtual packages and real packages that provide them
             if dpkg[PROVIDES]:
                 parts = apt_pkg.parse_depends(dpkg[PROVIDES], False)
                 nprov = []
@@ -706,7 +702,6 @@ class Britney(object):
                         provided = intern(provided)
                         provided_version = intern(provided_version)
                         part = (provided, provided_version, intern(op))
-                        provides[provided].add((pkg, provided_version))
                         nprov.append(part)
                 dpkg[PROVIDES] = nprov
             else:
@@ -719,6 +714,66 @@ class Britney(object):
             else:
                 all_binaries[pkg_id] = dpkg
 
+            # add the resulting dictionary to the package list
+            packages[pkg] = dpkg
+
+        return packages
+
+    def read_binaries(self, basedir, distribution, arch):
+        """Read the list of binary packages from the specified directory
+
+        The binary packages are read from the `Packages' files for `arch'.
+
+        If components are specified, the files
+        for each component are loaded according to the usual Debian mirror
+        layout.
+
+        If no components are specified, a single file named
+        `Packages_${arch}' is expected to be within the directory
+        specified as `basedir' parameter, replacing ${arch} with the
+        value of the arch parameter.
+
+        Considering the
+        large amount of memory needed, not all the fields are loaded
+        in memory. The available fields are Version, Source, Multi-Arch,
+        Depends, Conflicts, Provides and Architecture.
+
+        After reading the packages, reverse dependencies are computed
+        and saved in the `rdepends' keys, and the `Provides' field is
+        used to populate the virtual packages list.
+
+        The dependencies are parsed with the apt_pkg.parse_depends method,
+        and they are stored both as the format of its return value and
+        text.
+
+        The method returns a tuple. The first element is a list where
+        every item represents a binary package as a dictionary; the second
+        element is a dictionary which maps virtual packages to real
+        packages that provide them.
+        """
+
+        if self.options.components:
+            packages = {}
+            for component in self.options.components:
+                filename = os.path.join(basedir,
+                             component, "binary-%s" % arch, "Packages")
+                self._read_packages_file(filename, arch,
+                      self.sources[distribution], packages)
+        else:
+            filename = os.path.join(basedir, "Packages_%s" % arch)
+            packages = self._read_packages_file(filename, arch,
+                             self.sources[distribution])
+
+        # create provides
+        provides = defaultdict(set)
+
+        for pkg, dpkg in packages.items():
+            # register virtual packages and real packages that provide
+            # them
+            for provided_pkg, provided_version, _ in dpkg[PROVIDES]:
+                provides[provided_pkg].add((pkg, provided_version))
+
+
         # return a tuple with the list of real and virtual packages
         return (packages, provides)