From 496e48c9a77f5adac6cfa6db59ca61d3f906e2bc Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Mon, 27 Apr 2015 02:13:27 +1000 Subject: [PATCH] britney.py: Add support for multiple components Adds a --components command line argument (and corresponding config file option). If specified, package info is expected to be in the usual Debian mirror layout, ie: testing/source/Sources testing/binary-${ARCH}/Packages (nthykier: Squashed, rebased and did some porting to Python3) Signed-off-by: Niels Thykier --- britney.py | 141 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 98 insertions(+), 43 deletions(-) diff --git a/britney.py b/britney.py index 670877a..c6c32b1 100755 --- a/britney.py +++ b/britney.py @@ -387,6 +387,8 @@ class Britney(object): help="do not build the non-installability status, use the cache from file") parser.add_option("", "--print-uninst", action="store_true", dest="print_uninst", default=False, help="just print a summary of uninstallable packages") + parser.add_option("", "--components", action="store", dest="components", + help="Sources/Packages are laid out by components listed (, sep)") (self.options, self.args) = parser.parse_args() # integrity checks @@ -417,6 +419,11 @@ class Britney(object): not getattr(self.options, k.lower()): setattr(self.options, k.lower(), v) + if getattr(self.options, "components", None): + self.options.components = [s.strip() for s in self.options.components.split(",")] + else: + self.options.components = None + if not hasattr(self.options, "heidi_delta_output"): self.options.heidi_delta_output = self.options.heidi_output + "Delta" @@ -543,19 +550,10 @@ class Britney(object): # Data reading/writing methods # ---------------------------- - def read_sources(self, basedir, intern=sys.intern): - """Read the list of source packages from the specified directory - - The source packages are read from the `Sources' file within the - directory specified as `basedir' parameter. Considering the - large amount of memory needed, not all the fields are loaded - in memory. The available fields are Version, Maintainer and Section. + def _read_sources_file(self, filename, sources=None, intern=sys.intern): + if sources is None: + sources = {} - The method returns a list where every item represents a source - package as a dictionary. - """ - sources = {} - filename = os.path.join(basedir, "Sources") self.__log("Loading source packages from %s" % filename) with open(filename, encoding='utf-8') as f: @@ -583,38 +581,37 @@ class Britney(object): ] return sources - def read_binaries(self, basedir, distribution, arch, intern=sys.intern): - """Read the list of binary packages from the specified directory - - The binary packages are read from the `Packages_${arch}' files - within the directory specified as `basedir' parameter, replacing - ${arch} with the value of the arch parameter. Considering the + def read_sources(self, basedir): + """Read the list of source packages from the specified directory + + The source packages are read from the `Sources' file within the + directory specified as `basedir' parameter. Considering the large amount of memory needed, not all the fields are loaded - in memory. The available fields are Version, Source, Multi-Arch, - Depends, Conflicts, Provides and Architecture. - - After reading the packages, reverse dependencies are computed - and saved in the `rdepends' keys, and the `Provides' field is - used to populate the virtual packages list. + in memory. The available fields are Version, Maintainer and Section. - The dependencies are parsed with the apt_pkg.parse_depends method, - and they are stored both as the format of its return value and - text. - - The method returns a tuple. The first element is a list where - every item represents a binary package as a dictionary; the second - element is a dictionary which maps virtual packages to real - packages that provide them. + The method returns a list where every item represents a source + package as a dictionary. """ - packages = {} - provides = defaultdict(set) - sources = self.sources - all_binaries = self.all_binaries + if self.options.components: + sources = {} + for component in self.options.components: + filename = os.path.join(basedir, component, "source", "Sources") + self._read_sources_file(filename, sources) + else: + filename = os.path.join(basedir, "Sources") + sources = self._read_sources_file(filename) - filename = os.path.join(basedir, "Packages_%s" % arch) + return sources + + def _read_packages_file(self, filename, arch, srcdist, packages=None, intern=sys.intern): self.__log("Loading binary packages from %s" % filename) + if packages is None: + packages = {} + + all_binaries = self.all_binaries + with open(filename, encoding='utf-8') as f: Packages = apt_pkg.TagFile(f) get_field = Packages.section.get @@ -675,20 +672,19 @@ class Britney(object): dpkg[SOURCEVER] = intern(source[source.find("(")+1:source.find(")")]) # if the source package is available in the distribution, then register this binary package - if dpkg[SOURCE] in sources[distribution]: + if dpkg[SOURCE] in srcdist: # There may be multiple versions of any arch:all packages # (in unstable) if some architectures have out-of-date # binaries. We only want to include the package in the # source -> binary mapping once. It doesn't matter which # of the versions we include as only the package name and # architecture are recorded. - if pkg_id not in sources[distribution][dpkg[SOURCE]][BINARIES]: - sources[distribution][dpkg[SOURCE]][BINARIES].append(pkg_id) + if pkg_id not in srcdist[dpkg[SOURCE]][BINARIES]: + srcdist[dpkg[SOURCE]][BINARIES].append(pkg_id) # if the source package doesn't exist, create a fake one else: - sources[distribution][dpkg[SOURCE]] = [dpkg[SOURCEVER], 'faux', [pkg_id], None, True] + srcdist[dpkg[SOURCE]] = [dpkg[SOURCEVER], 'faux', [pkg_id], None, True] - # register virtual packages and real packages that provide them if dpkg[PROVIDES]: parts = apt_pkg.parse_depends(dpkg[PROVIDES], False) nprov = [] @@ -706,7 +702,6 @@ class Britney(object): provided = intern(provided) provided_version = intern(provided_version) part = (provided, provided_version, intern(op)) - provides[provided].add((pkg, provided_version)) nprov.append(part) dpkg[PROVIDES] = nprov else: @@ -719,6 +714,66 @@ class Britney(object): else: all_binaries[pkg_id] = dpkg + # add the resulting dictionary to the package list + packages[pkg] = dpkg + + return packages + + def read_binaries(self, basedir, distribution, arch): + """Read the list of binary packages from the specified directory + + The binary packages are read from the `Packages' files for `arch'. + + If components are specified, the files + for each component are loaded according to the usual Debian mirror + layout. + + If no components are specified, a single file named + `Packages_${arch}' is expected to be within the directory + specified as `basedir' parameter, replacing ${arch} with the + value of the arch parameter. + + Considering the + large amount of memory needed, not all the fields are loaded + in memory. The available fields are Version, Source, Multi-Arch, + Depends, Conflicts, Provides and Architecture. + + After reading the packages, reverse dependencies are computed + and saved in the `rdepends' keys, and the `Provides' field is + used to populate the virtual packages list. + + The dependencies are parsed with the apt_pkg.parse_depends method, + and they are stored both as the format of its return value and + text. + + The method returns a tuple. The first element is a list where + every item represents a binary package as a dictionary; the second + element is a dictionary which maps virtual packages to real + packages that provide them. + """ + + if self.options.components: + packages = {} + for component in self.options.components: + filename = os.path.join(basedir, + component, "binary-%s" % arch, "Packages") + self._read_packages_file(filename, arch, + self.sources[distribution], packages) + else: + filename = os.path.join(basedir, "Packages_%s" % arch) + packages = self._read_packages_file(filename, arch, + self.sources[distribution]) + + # create provides + provides = defaultdict(set) + + for pkg, dpkg in packages.items(): + # register virtual packages and real packages that provide + # them + for provided_pkg, provided_version, _ in dpkg[PROVIDES]: + provides[provided_pkg].add((pkg, provided_version)) + + # return a tuple with the list of real and virtual packages return (packages, provides)