diff --git a/britney.py b/britney.py index 7a13b19..fed9e1d 100755 --- a/britney.py +++ b/britney.py @@ -192,6 +192,7 @@ import urllib import apt_pkg +from collections import defaultdict from functools import reduce, partial from itertools import chain, ifilter, product from operator import attrgetter @@ -220,7 +221,7 @@ from britney_util import (old_libraries_format, same_source, undo_changes, read_nuninst, write_nuninst, write_heidi, eval_uninst, newly_uninst, make_migrationitem, write_excuses, write_heidi_delta, write_controlfiles, - old_libraries, ensuredir) + old_libraries, is_nuninst_asgood_generous, ensuredir) from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) @@ -454,10 +455,12 @@ class Britney(object): depends = [] conflicts = [] + possible_dep_ranges = {} # We do not differ between depends and pre-depends if pkgdata[DEPENDS]: depends.extend(apt_pkg.parse_depends(pkgdata[DEPENDS], False)) + if pkgdata[CONFLICTS]: conflicts = apt_pkg.parse_depends(pkgdata[CONFLICTS], False) @@ -465,8 +468,10 @@ class Britney(object): for (al, dep) in [(depends, True), \ (conflicts, False)]: + for block in al: sat = set() + for dep_dist in binaries: (_, pkgs) = solvers(block, arch, dep_dist) for p in pkgs: @@ -483,7 +488,37 @@ class Britney(object): # is using ยง7.6.2 relations.add_breaks(pt) if dep: - relations.add_dependency_clause(sat) + if len(block) != 1: + relations.add_dependency_clause(sat) + else: + # This dependency might be a part + # of a version-range a la: + # + # Depends: pkg-a (>= 1), + # pkg-a (<< 2~) + # + # In such a case we want to reduce + # that to a single clause for + # efficiency. + # + # In theory, it could also happen + # with "non-minimal" dependencies + # a la: + # + # Depends: pkg-a, pkg-a (>= 1) + # + # But dpkg is known to fix that up + # at build time, so we will + # probably only see "ranges" here. + key = block[0][0] + if key in possible_dep_ranges: + possible_dep_ranges[key] &= sat + else: + possible_dep_ranges[key] = sat + + if dep: + for clause in possible_dep_ranges.itervalues(): + relations.add_dependency_clause(clause) self._inst_tester = builder.build() @@ -731,7 +766,7 @@ class Britney(object): The method returns a dictionary where the key is the binary package name and the value is the list of open RC bugs for it. """ - bugs = {} + bugs = defaultdict(list) filename = os.path.join(basedir, "BugsV") self.__log("Loading RC bugs data from %s" % filename) try: @@ -742,7 +777,6 @@ class Britney(object): type='W') continue pkg = l[0] - bugs.setdefault(pkg, []) bugs[pkg] += l[1].split(",") except IOError: self.__log("%s missing; skipping bug-based processing" % filename) @@ -1196,7 +1230,7 @@ class Britney(object): binary_u = self.binaries[suite][arch][0][pkg_name] # this is the source version for the new binary package - pkgsv = self.binaries[suite][arch][0][pkg_name][SOURCEVER] + pkgsv = binary_u[SOURCEVER] # if the new binary package is architecture-independent, then skip it if binary_u[ARCHITECTURE] == 'all': @@ -1972,14 +2006,6 @@ class Britney(object): return "%d+%d: %s" % (total, totalbreak, ":".join(res)) - def is_nuninst_asgood_generous(self, old, new): - diff = 0 - for arch in self.options.architectures: - if arch in self.options.break_arches.split(): continue - diff = diff + (len(new[arch]) - len(old[arch])) - return diff <= 0 - - def _compute_groups(self, source_name, suite, migration_architecture, is_removal, include_hijacked=False, allow_smooth_updates=True, @@ -2159,9 +2185,9 @@ class Britney(object): This method applies the changes required by the action `item` tracking them so it will be possible to revert them. - The method returns a list of the package name, the suite where the - package comes from, the set of packages affected by the change and - the dictionary undo which can be used to rollback the changes. + The method returns a tuple containing a set of packages + affected by the change (as (name, arch)-tuples) and the + dictionary undo which can be used to rollback the changes. """ undo = {'binaries': {}, 'sources': {}, 'virtual': {}, 'nvirtual': []} @@ -2169,39 +2195,63 @@ class Britney(object): # local copies for better performances sources = self.sources - binaries = self.binaries['testing'] - get_reverse_tree = partial(compute_reverse_tree, self.binaries["testing"]) + packages_t = self.binaries['testing'] + get_reverse_tree = partial(compute_reverse_tree, packages_t) + inst_tester = self._inst_tester + eqv_set = set() + # remove all binary packages (if the source already exists) if item.architecture == 'source' or not item.is_removal: if item.package in sources['testing']: source = sources['testing'][item.package] - _, bins, _ = self._compute_groups(item.package, - item.suite, - item.architecture, - item.is_removal, - removals=removals) + updates, rms, _ = self._compute_groups(item.package, + item.suite, + item.architecture, + item.is_removal, + removals=removals) + + eqv_table = {} + + for binary, version, parch in rms: + key = (binary, parch) + eqv_table[key] = version + + for p1 in updates: + binary, _, parch = p1 + key = (binary, parch) + old_version = eqv_table.get(key) + if old_version is not None: + p2 = (binary, old_version, parch) + if inst_tester.are_equivalent(p1, p2): + eqv_set.add(key) # remove all the binaries which aren't being smooth updated - for bin_data in bins: - binary, _, parch = bin_data + for rm_tuple in rms: + binary, version, parch = rm_tuple p = binary + "/" + parch + binaries_t_a, provides_t_a = packages_t[parch] + pkey = (binary, parch) + + pkg_data = binaries_t_a[binary] # save the old binary for undo - undo['binaries'][p] = binaries[parch][0][binary] - # all the reverse dependencies are affected by the change - affected.update(get_reverse_tree(binary, parch)) + undo['binaries'][p] = pkg_data + if pkey not in eqv_set: + # all the reverse dependencies are affected by + # the change + affected.update(get_reverse_tree(binary, parch)) + # remove the provided virtual packages - for j in binaries[parch][0][binary][PROVIDES]: + for j in pkg_data[PROVIDES]: key = j + "/" + parch if key not in undo['virtual']: - undo['virtual'][key] = binaries[parch][1][j][:] - binaries[parch][1][j].remove(binary) - if len(binaries[parch][1][j]) == 0: - del binaries[parch][1][j] + undo['virtual'][key] = provides_t_a[j][:] + provides_t_a[j].remove(binary) + if not provides_t_a[j]: + del provides_t_a[j] # finally, remove the binary package - version = binaries[parch][0][binary][VERSION] - del binaries[parch][0][binary] - self._inst_tester.remove_testing_binary(binary, version, parch) + del binaries_t_a[binary] + inst_tester.remove_testing_binary(binary, version, parch) # remove the source package if item.architecture == 'source': undo['sources'][item.package] = source @@ -2212,37 +2262,47 @@ class Britney(object): # single binary removal; used for clearing up after smooth # updates but not supported as a manual hint - elif item.package in binaries[item.architecture][0]: - undo['binaries'][item.package + "/" + item.architecture] = binaries[item.architecture][0][item.package] + elif item.package in packages_t[item.architecture][0]: + binaries_t_a = packages_t[item.architecture][0] + undo['binaries'][item.package + "/" + item.architecture] = binaries_t_a[item.package] affected.update(get_reverse_tree(item.package, item.architecture)) - version = binaries[item.architecture][0][item.package][VERSION] - del binaries[item.architecture][0][item.package] - self._inst_tester.remove_testing_binary(item.package, version, item.architecture) + version = binaries_t_a[item.package][VERSION] + del binaries_t_a[item.package] + inst_tester.remove_testing_binary(item.package, version, item.architecture) # add the new binary packages (if we are not removing) if not item.is_removal: source = sources[item.suite][item.package] + packages_s = self.binaries[item.suite] for p in source[BINARIES]: binary, parch = p.split("/") if item.architecture not in ['source', parch]: continue key = (binary, parch) + binaries_t_a, provides_t_a = packages_t[parch] + equivalent_replacement = key in eqv_set + # obviously, added/modified packages are affected - if key not in affected: affected.add(key) + if not equivalent_replacement and key not in affected: + affected.add(key) # if the binary already exists in testing, it is currently # built by another source package. we therefore remove the # version built by the other source package, after marking # all of its reverse dependencies as affected - if binary in binaries[parch][0]: + if binary in binaries_t_a: + old_pkg_data = binaries_t_a[binary] # save the old binary package - undo['binaries'][p] = binaries[parch][0][binary] - # all the reverse dependencies are affected by the change - affected.update(get_reverse_tree(binary, parch)) - # all the reverse conflicts and their dependency tree are affected by the change - for j in binaries[parch][0][binary][RCONFLICTS]: - affected.update(get_reverse_tree(j, parch)) - version = binaries[parch][0][binary][VERSION] - self._inst_tester.remove_testing_binary(binary, version, parch) + undo['binaries'][p] = old_pkg_data + if not equivalent_replacement: + # all the reverse dependencies are affected by + # the change + affected.update(get_reverse_tree(binary, parch)) + # all the reverse conflicts and their + # dependency tree are affected by the change + for j in old_pkg_data[RCONFLICTS]: + affected.update(get_reverse_tree(j, parch)) + old_version = old_pkg_data[VERSION] + inst_tester.remove_testing_binary(binary, old_version, parch) else: # the binary isn't in testing, but it may have been at # the start of the current hint and have been removed @@ -2258,23 +2318,26 @@ class Britney(object): for (tundo, tpkg) in hint_undo: if p in tundo['binaries']: for rdep in tundo['binaries'][p][RDEPENDS]: - if rdep in binaries[parch][0] and rdep not in source[BINARIES]: + if rdep in binaries_t_a and rdep not in source[BINARIES]: affected.update(get_reverse_tree(rdep, parch)) - # add/update the binary package - binaries[parch][0][binary] = self.binaries[item.suite][parch][0][binary] - version = binaries[parch][0][binary][VERSION] - self._inst_tester.add_testing_binary(binary, version, parch) + + # add/update the binary package from the source suite + new_pkg_data = packages_s[parch][0][binary] + new_version = new_pkg_data[VERSION] + binaries_t_a[binary] = new_pkg_data + inst_tester.add_testing_binary(binary, new_version, parch) # register new provided packages - for j in binaries[parch][0][binary][PROVIDES]: + for j in new_pkg_data[PROVIDES]: key = j + "/" + parch - if j not in binaries[parch][1]: + if j not in provides_t_a: undo['nvirtual'].append(key) - binaries[parch][1][j] = [] + provides_t_a[j] = [] elif key not in undo['virtual']: - undo['virtual'][key] = binaries[parch][1][j][:] - binaries[parch][1][j].append(binary) - # all the reverse dependencies are affected by the change - affected.update(get_reverse_tree(binary, parch)) + undo['virtual'][key] = provides_t_a[j][:] + provides_t_a[j].append(binary) + if not equivalent_replacement: + # all the reverse dependencies are affected by the change + affected.update(get_reverse_tree(binary, parch)) # register reverse dependencies and conflicts for the new binary packages if item.architecture == 'source': @@ -2282,14 +2345,14 @@ class Britney(object): else: ext = "/" + item.architecture pkg_iter = (p.split("/")[0] for p in source[BINARIES] if p.endswith(ext)) - register_reverses(binaries[parch][0], binaries[parch][1], iterator=pkg_iter) + register_reverses(binaries_t_a, provides_t_a, iterator=pkg_iter) # add/update the source package if item.architecture == 'source': sources['testing'][item.package] = sources[item.suite][item.package] # return the package name, the suite, the list of affected packages and the undo dictionary - return (item, affected, undo) + return (affected, undo) def _check_packages(self, binaries, arch, affected, skip_archall, nuninst): @@ -2324,7 +2387,51 @@ class Britney(object): self._installability_test(p, version, arch, broken, to_check, nuninst_arch) - def iter_packages(self, packages, selected, hint=False, nuninst=None, lundo=None): + def iter_packages_hint(self, hinted_packages, lundo=None): + """Iter on hinted list of actions and apply them in one go + + This method applies the changes from "hinted_packages" to + testing and computes the uninstallability counters after te + actions are performed. + + The method returns the new uninstallability counters. + """ + + removals = set() + all_affected = set() + nobreakall_arches = self.options.nobreakall_arches.split() + binaries_t = self.binaries['testing'] + check_packages = partial(self._check_packages, binaries_t) + # Deep copy nuninst (in case the hint is undone) + nuninst = {k:v.copy() for k,v in self.nuninst_orig.iteritems()} + + + for item in hinted_packages: + _, rms, _ = self._compute_groups(item.package, item.suite, + item.architecture, + item.is_removal, + allow_smooth_updates=False) + removals.update(rms) + + for item in hinted_packages: + affected, undo = self.doop_source(item, + removals=removals) + all_affected.update(affected) + if lundo is not None: + lundo.append((undo,item)) + + for arch in self.options.architectures: + if arch not in nobreakall_arches: + skip_archall = True + else: + skip_archall = False + + check_packages(arch, all_affected, skip_archall, nuninst) + + return nuninst + + + def iter_packages(self, packages, selected, nuninst=None, lundo=None): """Iter on the list of actions and apply them one-by-one This method applies the changes from `packages` to testing, checking the uninstallability @@ -2353,29 +2460,14 @@ class Britney(object): dependencies = self.dependencies check_packages = partial(self._check_packages, binaries) - # pre-process a hint batch - pre_process = {} - if selected and hint: - removals = set() - for item in selected: - _, rms, _ = self._compute_groups(item.package, item.suite, - item.architecture, - item.is_removal, - allow_smooth_updates=False) - removals.update(rms) - for package in selected: - pkg, affected, undo = self.doop_source(package, - removals=removals) - pre_process[package] = (pkg, affected, undo) - if lundo is None: lundo = [] - if not hint: - self.output_write("recur: [%s] %s %d/%d\n" % ("", ",".join(x.uvname for x in selected), len(packages), len(extra))) + + self.output_write("recur: [%s] %s %d/%d\n" % ("", ",".join(x.uvname for x in selected), len(packages), len(extra))) # loop on the packages (or better, actions) while packages: - pkg = packages.pop(0) + item = packages.pop(0) # this is the marker for the first loop if not mark_passed and position < 0: @@ -2387,61 +2479,48 @@ class Britney(object): # defer packages if their dependency has been already skipped if not mark_passed: defer = False - for p in dependencies.get(pkg, []): + for p in dependencies.get(item, []): if p in skipped: - deferred.append(make_migrationitem(pkg, self.sources)) - skipped.append(make_migrationitem(pkg, self.sources)) + deferred.append(item) + skipped.append(item) defer = True break if defer: continue - if not hint: - self.output_write("trying: %s\n" % (pkg.uvname)) + self.output_write("trying: %s\n" % (item.uvname)) better = True nuninst = {} # apply the changes - if pkg in pre_process: - item, affected, undo = pre_process[pkg] - else: - item, affected, undo = self.doop_source(pkg, lundo) - if hint: - lundo.append((undo, item)) + affected, undo = self.doop_source(item, lundo) # check the affected packages on all the architectures for arch in (item.architecture == 'source' and architectures or (item.architecture,)): if arch not in nobreakall_arches: skip_archall = True - else: skip_archall = False + else: + skip_archall = False nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) check_packages(arch, affected, skip_archall, nuninst) - # if we are processing hints, go ahead - if hint: - nuninst_comp[arch] = nuninst[arch] - nuninst_comp[arch + "+all"] = nuninst[arch + "+all"] - continue - # if the uninstallability counter is worse than before, break the loop if ((item.architecture != 'source' and arch not in new_arches) or \ (arch not in break_arches)) and len(nuninst[arch]) > len(nuninst_comp[arch]): better = False break - # if we are processing hints or the package is already accepted, go ahead - if hint or item in selected: continue # check if the action improved the uninstallability counters if better: lundo.append((undo, item)) - selected.append(pkg) + selected.append(item) packages.extend(extra) extra = [] - self.output_write("accepted: %s\n" % (pkg.uvname)) + self.output_write("accepted: %s\n" % (item.uvname)) self.output_write(" ori: %s\n" % (self.eval_nuninst(self.nuninst_orig))) self.output_write(" pre: %s\n" % (self.eval_nuninst(nuninst_comp))) self.output_write(" now: %s\n" % (self.eval_nuninst(nuninst, nuninst_comp))) @@ -2452,8 +2531,8 @@ class Britney(object): for k in nuninst: nuninst_comp[k] = nuninst[k] else: - self.output_write("skipped: %s (%d <- %d)\n" % (pkg.uvname, len(extra), len(packages))) - self.output_write(" got: %s\n" % (self.eval_nuninst(nuninst, pkg.architecture != 'source' and nuninst_comp or None))) + self.output_write("skipped: %s (%d <- %d)\n" % (item.uvname, len(extra), len(packages))) + self.output_write(" got: %s\n" % (self.eval_nuninst(nuninst, item.architecture != 'source' and nuninst_comp or None))) self.output_write(" * %s: %s\n" % (arch, ", ".join(sorted(b for b in nuninst[arch] if b not in nuninst_comp[arch])))) extra.append(item) @@ -2463,9 +2542,6 @@ class Britney(object): # (local-scope) binaries is actually self.binaries["testing"] so we cannot use it here. undo_changes(single_undo, self._inst_tester, sources, self.binaries) - # if we are processing hints, return now - if hint: - return (nuninst_comp, []) self.output_write(" finish: [%s]\n" % ",".join( x.uvname for x in selected )) self.output_write("endloop: %s\n" % (self.eval_nuninst(self.nuninst_orig))) @@ -2476,6 +2552,7 @@ class Britney(object): return (nuninst_comp, extra) + def do_all(self, hinttype=None, init=None, actions=None): """Testing update runner @@ -2495,6 +2572,8 @@ class Britney(object): recurse = True lundo = None nuninst_end = None + better = True + extra = () # empty tuple if hinttype == "easy" or hinttype == "force-hint": force = hinttype == "force-hint" @@ -2519,7 +2598,11 @@ class Britney(object): if init: # init => a hint (e.g. "easy") - so do the hint run - (nuninst_end, extra) = self.iter_packages(init, selected, hint=True, lundo=lundo) + nuninst_end = self.iter_packages_hint(selected, lundo=lundo) + if recurse: + # Ensure upgrade_me and selected do not overlap, if we + # follow-up with a recurse ("hint"-hint). + upgrade_me = [x for x in upgrade_me if x not in set(selected)] if recurse: # Either the main run or the recursive run of a "hint"-hint. @@ -2537,7 +2620,14 @@ class Britney(object): self.output_write(eval_uninst(self.options.architectures, newly_uninst(nuninst_start, nuninst_end))) - if force or self.is_nuninst_asgood_generous(self.nuninst_orig, nuninst_end): + if not force: + break_arches = self.options.break_arches.split() + better = is_nuninst_asgood_generous(self.options.architectures, + self.nuninst_orig, + nuninst_end, + break_arches) + + if better: # Result accepted either by force or by being better than the original result. if recurse: self.output_write("Apparently successful\n") @@ -2559,7 +2649,7 @@ class Britney(object): if recurse: self.upgrade_me = sorted(extra) else: - self.upgrade_me = [x for x in self.upgrade_me if x not in selected] + self.upgrade_me = [x for x in self.upgrade_me if x not in set(selected)] self.sort_actions() else: self.output_write("FAILED\n") @@ -2952,10 +3042,11 @@ class Britney(object): def nuninst_arch_report(self, nuninst, arch): """Print a report of uninstallable packages for one architecture.""" - all = {} + all = defaultdict(set) for p in nuninst[arch]: pkg = self.binaries['testing'][arch][0][p] - all.setdefault((pkg[SOURCE], pkg[SOURCEVER]), set()).add(p) + all[(pkg[SOURCE], pkg[SOURCEVER])].add(p) + print '* %s' % (arch,) diff --git a/britney_util.py b/britney_util.py index c74cdb8..f08d4ee 100644 --- a/britney_util.py +++ b/britney_util.py @@ -583,3 +583,24 @@ def old_libraries(sources, packages, same_source=same_source): migration = "-" + "/".join((pkg_name, arch, pkg[SOURCEVER])) removals.append(MigrationItem(migration)) return removals + + +def is_nuninst_asgood_generous(architectures, old, new, break_arches=frozenset()): + """Compares the nuninst counters to see if they improved + + Given a list of architecters, the previous and the current nuninst + counters, this function determines if the current nuninst counter + is better than the previous one. Optionally it also accepts a set + of "break_arches", the nuninst counter for any architecture listed + in this set are completely ignored. + + Returns True if the new nuninst counter is better than the + previous. Returns False otherwise. + + """ + diff = 0 + for arch in architectures: + if arch in break_arches: + continue + diff = diff + (len(new[arch]) - len(old[arch])) + return diff <= 0 diff --git a/installability/builder.py b/installability/builder.py index 10b78eb..23ff716 100644 --- a/installability/builder.py +++ b/installability/builder.py @@ -12,6 +12,7 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +from collections import defaultdict from contextlib import contextmanager from britney_util import ifilter_except, iter_except @@ -28,7 +29,7 @@ class _RelationBuilder(object): self._new_breaks = set(binary_data[1]) - def add_dependency_clause(self, or_clause): + def add_dependency_clause(self, or_clause, frozenset=frozenset): """Add a dependency clause The clause must be a sequence of (name, version, architecture) @@ -48,12 +49,12 @@ class _RelationBuilder(object): binary = self._binary itbuilder = self._itbuilder package_table = itbuilder._package_table - reverse_package_table = itbuilder._reverse_package_table okay = False for dep_tuple in clause: okay = True - reverse_relations = itbuilder._reverse_relations(dep_tuple) - reverse_relations[0].add(binary) + rdeps, _, rdep_relations = itbuilder._reverse_relations(dep_tuple) + rdeps.add(binary) + rdep_relations.add(clause) self._new_deps.add(clause) if not okay: @@ -193,15 +194,17 @@ class InstallabilityTesterBuilder(object): if binary in self._reverse_package_table: return self._reverse_package_table[binary] - rel = [set(), set()] + rel = [set(), set(), set()] self._reverse_package_table[binary] = rel return rel def build(self): - # Merge reverse conflicts with conflicts - this saves some - # operations in _check_loop since we only have to check one - # set (instead of two) and we remove a few duplicates here - # and there. + """Compile the installability tester + + This method will compile an installability tester from the + information given and (where possible) try to optimise a + few things. + """ package_table = self._package_table reverse_package_table = self._reverse_package_table intern_set = self._intern_set @@ -220,18 +223,26 @@ class InstallabilityTesterBuilder(object): return False return True + + # Merge reverse conflicts with conflicts - this saves some + # operations in _check_loop since we only have to check one + # set (instead of two) and we remove a few duplicates here + # and there. + # + # At the same time, intern the rdep sets for pkg in reverse_package_table: if pkg not in package_table: raise RuntimeError("%s/%s/%s referenced but not added!" % pkg) - if not reverse_package_table[pkg][1]: - # no rconflicts - ignore - continue deps, con = package_table[pkg] - if not con: - con = intern_set(reverse_package_table[pkg][1]) - else: - con = intern_set(con | reverse_package_table[pkg][1]) - package_table[pkg] = (deps, con) + rdeps, rcon, rdep_relations = reverse_package_table[pkg] + if rcon: + if not con: + con = intern_set(rcon) + else: + con = intern_set(con | rcon) + package_table[pkg] = (deps, con) + reverse_package_table[pkg] = (intern_set(rdeps), con, + intern_set(rdep_relations)) # Check if we can expand broken. for t in not_broken(iter_except(check.pop, KeyError)): @@ -301,8 +312,88 @@ class InstallabilityTesterBuilder(object): # add all rdeps (except those already in the safe_set) check.update(reverse_package_table[pkg][0] - safe_set) + eqv_table = self._build_eqv_packages_table(package_table, + reverse_package_table) return InstallabilitySolver(package_table, reverse_package_table, self._testing, self._broken, - self._essentials, safe_set) + self._essentials, safe_set, + eqv_table) + + + def _build_eqv_packages_table(self, package_table, + reverse_package_table, + frozenset=frozenset): + """Attempt to build a table of equivalent packages + + This method attempts to create a table of packages that are + equivalent (in terms of installability). If two packages (A + and B) are equivalent then testing the installability of A is + the same as testing the installability of B. This equivalency + also applies to co-installability. + + The example cases: + * aspell-* + * ispell-* + + Cases that do *not* apply: + * MTA's + + The theory: + + The packages A and B are equivalent iff: + + reverse_depends(A) == reverse_depends(B) AND + conflicts(A) == conflicts(B) AND + depends(A) == depends(B) + + Where "reverse_depends(X)" is the set of reverse dependencies + of X, "conflicts(X)" is the set of negative dependencies of X + (Breaks and Conflicts plus the reverse ones of those combined) + and "depends(X)" is the set of strong dependencies of X + (Depends and Pre-Depends combined). + + To be honest, we are actually equally interested another + property as well, namely substitutability. The package A can + always used instead of B, iff: + + reverse_depends(A) >= reverse_depends(B) AND + conflicts(A) <= conflicts(B) AND + depends(A) == depends(B) + + (With the same definitions as above). Note that equivalency + is just a special-case of substitutability, where A and B can + substitute each other (i.e. a two-way substituation). + + Finally, note that the "depends(A) == depends(B)" for + substitutability is actually not a strict requirement. There + are cases where those sets are different without affecting the + property. + """ + # Despite talking about substitutability, the method currently + # only finds the equivalence cases. Lets leave + # substitutability for a future version. + + find_eqv_table = defaultdict(list) + eqv_table = {} + + for pkg in reverse_package_table: + rdeps = reverse_package_table[pkg][2] + if not rdeps: + # we don't care for things without rdeps (because + # it is not worth it) + continue + deps, con = package_table[pkg] + ekey = (deps, con, rdeps) + find_eqv_table[ekey].append(pkg) + + for pkg_list in find_eqv_table.itervalues(): + if len(pkg_list) < 2: + continue + + eqv_set = frozenset(pkg_list) + for pkg in pkg_list: + eqv_table[pkg] = eqv_set + + return eqv_table diff --git a/installability/solver.py b/installability/solver.py index cc5acee..318d5f9 100644 --- a/installability/solver.py +++ b/installability/solver.py @@ -24,7 +24,7 @@ from britney_util import (ifilter_only, iter_except) class InstallabilitySolver(InstallabilityTester): def __init__(self, universe, revuniverse, testing, broken, essentials, - safe_set): + safe_set, eqv_table): """Create a new installability solver universe is a dict mapping package tuples to their @@ -44,7 +44,7 @@ class InstallabilitySolver(InstallabilityTester): (simplifies caches and dependency checking) """ InstallabilityTester.__init__(self, universe, revuniverse, testing, - broken, essentials, safe_set) + broken, essentials, safe_set, eqv_table) def solve_groups(self, groups): diff --git a/installability/tester.py b/installability/tester.py index 2b98b1b..4be7dba 100644 --- a/installability/tester.py +++ b/installability/tester.py @@ -20,7 +20,7 @@ from britney_util import iter_except class InstallabilityTester(object): def __init__(self, universe, revuniverse, testing, broken, essentials, - safe_set): + safe_set, eqv_table): """Create a new installability tester universe is a dict mapping package tuples to their @@ -51,6 +51,7 @@ class InstallabilityTester(object): self._essentials = essentials self._revuniverse = revuniverse self._safe_set = safe_set + self._eqv_table = eqv_table # Cache of packages known to be broken - we deliberately do not # include "broken" in it. See _optimize for more info. @@ -80,11 +81,33 @@ class InstallabilityTester(object): check_inst = self._check_inst cbroken = self._cache_broken cache_inst = self._cache_inst - tcopy = [x for x in self._testing] + eqv_table = self._eqv_table + testing = self._testing + tcopy = [x for x in testing] for t in ifilterfalse(cache_inst.__contains__, tcopy): if t in cbroken: continue - check_inst(t) + res = check_inst(t) + if t in eqv_table: + eqv = (x for x in eqv_table[t] if x in testing) + if res: + cache_inst.update(eqv) + else: + eqv_set = frozenset(eqv) + testing -= eqv_set + cbroken |= eqv_set + + + def are_equivalent(self, p1, p2): + """Test if p1 and p2 are equivalent + + Returns True if p1 and p2 have the same "signature" in + the package dependency graph (i.e. relations can not tell + them appart sematically except for their name) + """ + eqv_table = self._eqv_table + return p1 in eqv_table and p2 in eqv_table[p1] + def add_testing_binary(self, pkg_name, pkg_version, pkg_arch): """Add a binary package to "testing" @@ -195,6 +218,7 @@ class InstallabilityTester(object): testing = self._testing cbroken = self._cache_broken safe_set = self._safe_set + eqv_table = self._eqv_table # Our installability verdict - start with "yes" and change if # prove otherwise. @@ -235,8 +259,9 @@ class InstallabilityTester(object): never.update(ess_never) # curry check_loop - check_loop = partial(self._check_loop, universe, testing, musts, - never, choices, cbroken) + check_loop = partial(self._check_loop, universe, testing, + eqv_table, musts, never, choices, + cbroken) # Useful things to remember: @@ -258,7 +283,7 @@ class InstallabilityTester(object): # of t via recursion (calls _check_inst). In this case # check and choices are not (always) empty. - def _pick_choice(rebuild): + def _pick_choice(rebuild, set=set, len=len): """Picks a choice from choices and updates rebuild. Prunes the choices and updates "rebuild" to reflect the @@ -317,18 +342,55 @@ class InstallabilityTester(object): last = next(choice) # pick one to go last for p in choice: musts_copy = musts.copy() - never_copy = never.copy() - choices_copy = choices.copy() - if self._check_inst(p, musts_copy, never_copy, choices_copy): + never_tmp = set() + choices_tmp = set() + check_tmp = set([p]) + if not self._check_loop(universe, testing, eqv_table, + musts_copy, never_tmp, + choices_tmp, cbroken, + check_tmp): + # p cannot be chosen/is broken (unlikely, but ...) + continue + + # Test if we can pick p without any consequences. + # - when we can, we avoid a backtrack point. + if never_tmp <= never and choices_tmp <= rebuild: + # we can pick p without picking up new conflicts + # or unresolved choices. Therefore we commit to + # using p. + # + # NB: Optimally, we would go to the start of this + # routine, but to conserve stack-space, we return + # and expect to be called again later. + musts.update(musts_copy) + return False + + if not musts.isdisjoint(never_tmp): + # If we pick p, we will definitely end up making + # t uninstallable, so p is a no-go. + continue + + # We are not sure that p is safe, setup a backtrack + # point and recurse. + never_tmp |= never + choices_tmp |= rebuild + if self._check_inst(p, musts_copy, never_tmp, + choices_tmp): + # Success, p was a valid choice and made it all + # installable return True - # If we get here, we failed to find something that would satisfy choice (without breaking - # the installability of t). This means p cannot be used to satisfy the dependencies, so - # pretend to conflict with it - hopefully it will reduce future choices. + + # If we get here, we failed to find something that + # would satisfy choice (without breaking the + # installability of t). This means p cannot be used + # to satisfy the dependencies, so pretend to conflict + # with it - hopefully it will reduce future choices. never.add(p) - # Optimization for the last case; avoid the recursive call and just - # assume the last will lead to a solution. If it doesn't there is - # no solution and if it does, we don't have to back-track anyway. + # Optimization for the last case; avoid the recursive call + # and just assume the last will lead to a solution. If it + # doesn't there is no solution and if it does, we don't + # have to back-track anyway. check.add(last) musts.add(last) return False @@ -359,8 +421,9 @@ class InstallabilityTester(object): return verdict - def _check_loop(self, universe, testing, musts, never, - choices, cbroken, check): + def _check_loop(self, universe, testing, eqv_table, musts, never, + choices, cbroken, check, len=len, + frozenset=frozenset): """Finds all guaranteed dependencies via "check". If it returns False, t is not installable. If it returns True @@ -368,8 +431,6 @@ class InstallabilityTester(object): returns True, then t is installable. """ # Local variables for faster access... - l = len - fset = frozenset not_satisfied = partial(ifilter, musts.isdisjoint) # While we have guaranteed dependencies (in check), examine all @@ -401,9 +462,9 @@ class InstallabilityTester(object): # - not in testing # - known to be broken (by cache) # - in never - candidates = fset((depgroup & testing) - never) + candidates = frozenset((depgroup & testing) - never) - if l(candidates) == 0: + if len(candidates) == 0: # We got no candidates to satisfy it - this # package cannot be installed with the current # testing @@ -413,21 +474,43 @@ class InstallabilityTester(object): cbroken.add(cur) testing.remove(cur) return False - if l(candidates) == 1: + if len(candidates) == 1: # only one possible solution to this choice and we # haven't seen it before check.update(candidates) musts.update(candidates) else: + possible_eqv = set(x for x in candidates if x in eqv_table) + if len(possible_eqv) > 1: + # Exploit equivalency to reduce the number of + # candidates if possible. Basically, this + # code maps "similar" candidates into a single + # candidate that will give a identical result + # to any other candidate it eliminates. + # + # See InstallabilityTesterBuilder's + # _build_eqv_packages_table method for more + # information on how this works. + new_cand = set(x for x in candidates if x not in possible_eqv) + for chosen in iter_except(possible_eqv.pop, KeyError): + new_cand.add(chosen) + possible_eqv -= eqv_table[chosen] + if len(new_cand) == 1: + check.update(new_cand) + musts.update(new_cand) + continue + candidates = frozenset(new_cand) # defer this choice till later choices.add(candidates) return True + def _get_min_pseudo_ess_set(self, arch): if arch not in self._cache_ess: # The minimal essential set cache is not present - # compute it now. testing = self._testing + eqv_table = self._eqv_table cbroken = self._cache_broken universe = self._universe safe_set = self._safe_set @@ -439,8 +522,9 @@ class InstallabilityTester(object): not_satisified = partial(ifilter, start.isdisjoint) while ess_base: - self._check_loop(universe, testing, start, ess_never,\ - ess_choices, cbroken, ess_base) + self._check_loop(universe, testing, eqv_table, + start, ess_never, ess_choices, + cbroken, ess_base) if ess_choices: # Try to break choices where possible nchoice = set()