From 8d97761decada04bb4296aec0a836ed8b7511b3f Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Fri, 24 Apr 2015 05:11:42 +0000 Subject: [PATCH 01/59] britney{,_nobreakall}.conf: remove kfreebsd Signed-off-by: Adam D. Barratt --- britney.conf | 4 ++-- britney_nobreakall.conf | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/britney.conf b/britney.conf index 163e191..4639bc5 100644 --- a/britney.conf +++ b/britney.conf @@ -14,13 +14,13 @@ UPGRADE_OUTPUT = /srv/release.debian.org/britney/var/data-b2/output/output.tx HEIDI_OUTPUT = /srv/release.debian.org/britney/var/data-b2/output/HeidiResult # List of release architectures -ARCHITECTURES = i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mipsel powerpc ppc64el s390x +ARCHITECTURES = i386 amd64 arm64 armel armhf mips mipsel powerpc ppc64el s390x # if you're not in this list, arch: all packages are allowed to break on you NOBREAKALL_ARCHES = i386 # if you're in this list, your packages may not stay in sync with the source -FUCKED_ARCHES = kfreebsd-i386 kfreebsd-amd64 +FUCKED_ARCHES = # if you're in this list, your uninstallability count may increase BREAK_ARCHES = diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index f4e8c1a..99e9d21 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -14,13 +14,13 @@ UPGRADE_OUTPUT = /srv/release.debian.org/britney/var/data-b2/output/output.tx HEIDI_OUTPUT = /srv/release.debian.org/britney/var/data-b2/output/HeidiResult # List of release architectures -ARCHITECTURES = i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mipsel powerpc ppc64el s390x +ARCHITECTURES = i386 amd64 arm64 armel armhf mips mipsel powerpc ppc64el s390x # if you're not in this list, arch: all packages are allowed to break on you -NOBREAKALL_ARCHES = i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mipsel powerpc ppc64el s390x +NOBREAKALL_ARCHES = i386 amd64 arm64 armel armhf mips mipsel powerpc ppc64el s390x # if you're in this list, your packages may not stay in sync with the source -FUCKED_ARCHES = kfreebsd-i386 kfreebsd-amd64 +FUCKED_ARCHES = # if you're in this list, your uninstallability count may increase BREAK_ARCHES = From d0d17dac5b38befc44022a11415c5a83d86dc1ae Mon Sep 17 00:00:00 2001 From: Jonathan Wiltshire Date: Sat, 25 Apr 2015 20:03:48 +0000 Subject: [PATCH 02/59] britney.conf: NOBREAKALL amd64 in addition to i386 Signed-off-by: Jonathan Wiltshire --- britney.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.conf b/britney.conf index 4639bc5..fc60879 100644 --- a/britney.conf +++ b/britney.conf @@ -17,7 +17,7 @@ HEIDI_OUTPUT = /srv/release.debian.org/britney/var/data-b2/output/HeidiResu ARCHITECTURES = i386 amd64 arm64 armel armhf mips mipsel powerpc ppc64el s390x # if you're not in this list, arch: all packages are allowed to break on you -NOBREAKALL_ARCHES = i386 +NOBREAKALL_ARCHES = i386 amd64 # if you're in this list, your packages may not stay in sync with the source FUCKED_ARCHES = From bf1d91d587af26fd6150fa9664c4486ad7615d6c Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 03/59] Fix typo Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 3bd172b..f325213 100755 --- a/britney.py +++ b/britney.py @@ -17,7 +17,7 @@ # GNU General Public License for more details. """ -= Introdution = += Introduction = This is the Debian testing updater script, also known as "Britney". From 726bbde3a3bb580306e95cd40ae9675d5b269bf9 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 04/59] Remove sys.path frobbing I don't think this has been necessary since we stopped loading a C extension. Signed-off-by: Julien Cristau --- britney.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/britney.py b/britney.py index f325213..6918cea 100755 --- a/britney.py +++ b/britney.py @@ -194,20 +194,6 @@ from functools import reduce, partial from itertools import chain, ifilter, product from operator import attrgetter -if __name__ == '__main__': - # Check if there is a python-search dir for this version of - # python. If so, use the britney module for that. - mdir = os.path.dirname(sys.argv[0]) - if sys.version_info[0] == 3: - python_dir = "python3" - else: - python_dir = "python2.%d" % (sys.version_info[1]) - idir = os.path.join(mdir, python_dir) - if os.path.isdir(idir): - print "N: Loading from %s" % python_dir - # Insert in front (else current dir is before it, which makes - # it useless). - sys.path.insert(0, idir) from installability.builder import InstallabilityTesterBuilder from excuse import Excuse From 5d7393cf86831658cbbbdcefd5bda4ca756ebe53 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 05/59] Switch to print_function Signed-off-by: Julien Cristau --- britney.py | 21 +++++++++++---------- hints.py | 4 +++- installability/solver.py | 32 +++++++++++++++++--------------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/britney.py b/britney.py index 6918cea..6e361ff 100755 --- a/britney.py +++ b/britney.py @@ -179,6 +179,7 @@ does for the generation of the update excuses. * The excuses are written in an HTML file. """ +from __future__ import print_function import os import sys @@ -255,8 +256,8 @@ class Britney(object): if self.options.nuninst_cache: self.__log("Not building the list of non-installable packages, as requested", type="I") if self.options.print_uninst: - print '* summary' - print '\n'.join(map(lambda x: '%4d %s' % (len(nuninst[x]), x), self.options.architectures)) + print('* summary') + print('\n'.join('%4d %s' % (len(nuninst[x]), x) for x in self.options.architectures)) return # read the source and binary packages for the involved distributions @@ -303,8 +304,8 @@ class Britney(object): self.nuninst_arch_report(nuninst, arch) if self.options.print_uninst: - print '* summary' - print '\n'.join(map(lambda x: '%4d %s' % (len(nuninst[x]), x), self.options.architectures)) + print('* summary') + print('\n'.join(map(lambda x: '%4d %s' % (len(nuninst[x]), x), self.options.architectures))) return else: write_nuninst(self.options.noninst_status, nuninst) @@ -396,7 +397,7 @@ class Britney(object): printed only if verbose logging is enabled. """ if self.options.verbose or type in ("E", "W"): - print "%s: [%s] - %s" % (type, time.asctime(), msg) + print("%s: [%s] - %s" % (type, time.asctime(), msg)) def _build_installability_tester(self, archs): """Create the installability tester""" @@ -2600,10 +2601,10 @@ class Britney(object): try: input = raw_input('britney> ').lower().split() except EOFError: - print "" + print("") break except KeyboardInterrupt: - print "" + print("") continue # quit the hint tester if input and input[0] in ('quit', 'exit'): @@ -2822,16 +2823,16 @@ class Britney(object): all[(pkg[SOURCE], pkg[SOURCEVER])].add(p) - print '* %s' % (arch,) + print('* %s' % (arch,)) for (src, ver), pkgs in sorted(all.iteritems()): - print ' %s (%s): %s' % (src, ver, ' '.join(sorted(pkgs))) + print(' %s (%s): %s' % (src, ver, ' '.join(sorted(pkgs)))) print def output_write(self, msg): """Simple wrapper for output writing""" - print msg, + print(msg, end='') self.__output.write(msg) def main(self): diff --git a/hints.py b/hints.py index 7b43cdb..ab47675 100644 --- a/hints.py +++ b/hints.py @@ -12,6 +12,8 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +from __future__ import print_function + from migrationitem import MigrationItem class HintCollection(object): @@ -36,7 +38,7 @@ class HintCollection(object): try: self._hints.append(Hint(hint, user)) except AssertionError: - print "Ignoring broken hint %r from %s" % (hint, user) + print("Ignoring broken hint %r from %s" % (hint, user)) class Hint(object): NO_VERSION = [ 'block', 'block-all', 'block-udeb' ] diff --git a/installability/solver.py b/installability/solver.py index 318d5f9..8ca402e 100644 --- a/installability/solver.py +++ b/installability/solver.py @@ -14,6 +14,8 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +from __future__ import print_function + from functools import partial import os @@ -106,7 +108,7 @@ class InstallabilitySolver(InstallabilityTester): # "Self-conflicts" => ignore continue if debug_solver and other not in order[key]['before']: - print "N: Conflict induced order: %s before %s" % (key, other) + print("N: Conflict induced order: %s before %s" % (key, other)) order[key]['before'].add(other) order[other]['after'].add(key) @@ -125,7 +127,7 @@ class InstallabilitySolver(InstallabilityTester): # "Self-dependency" => ignore continue if debug_solver and other not in order[key]['after']: - print "N: Removal induced order: %s before %s" % (key, other) + print("N: Removal induced order: %s before %s" % (key, other)) order[key]['after'].add(other) order[other]['before'].add(key) @@ -162,13 +164,13 @@ class InstallabilitySolver(InstallabilityTester): for other in (other_adds - other_rms): if debug_solver and other != key and other not in order[key]['after']: - print "N: Dependency induced order (add): %s before %s" % (key, other) + print("N: Dependency induced order (add): %s before %s" % (key, other)) order[key]['after'].add(other) order[other]['before'].add(key) for other in (other_rms - other_adds): if debug_solver and other != key and other not in order[key]['before']: - print "N: Dependency induced order (remove): %s before %s" % (key, other) + print("N: Dependency induced order (remove): %s before %s" % (key, other)) order[key]['before'].add(other) order[other]['after'].add(key) @@ -208,7 +210,7 @@ class InstallabilitySolver(InstallabilityTester): merged[n] = scc_id del order[n] if debug_solver: - print "N: SCC: %s -- %s" % (scc_id, str(sorted(com))) + print("N: SCC: %s -- %s" % (scc_id, str(sorted(com)))) for com in comps: node = com[0] @@ -223,27 +225,27 @@ class InstallabilitySolver(InstallabilityTester): if debug_solver: - print "N: -- PARTIAL ORDER --" + print("N: -- PARTIAL ORDER --") for com in sorted(order): if debug_solver and order[com]['before']: - print "N: %s <= %s" % (com, str(sorted(order[com]['before']))) + print("N: %s <= %s" % (com, str(sorted(order[com]['before'])))) if not order[com]['after']: # This component can be scheduled immediately, add it # to "check" check.add(com) elif debug_solver: - print "N: %s >= %s" % (com, str(sorted(order[com]['after']))) + print("N: %s >= %s" % (com, str(sorted(order[com]['after'])))) if debug_solver: - print "N: -- END PARTIAL ORDER --" - print "N: -- LINEARIZED ORDER --" + print("N: -- END PARTIAL ORDER --") + print("N: -- LINEARIZED ORDER --") for cur in iter_except(check.pop, KeyError): if order[cur]['after'] <= emitted: # This item is ready to be emitted right now if debug_solver: - print "N: %s -- %s" % (cur, sorted(scc[cur])) + print("N: %s -- %s" % (cur, sorted(scc[cur]))) emitted.add(cur) result.append([key2item[x] for x in scc[cur]]) if order[cur]['before']: @@ -254,7 +256,7 @@ class InstallabilitySolver(InstallabilityTester): check.update(order[cur]['before'] - emitted) if debug_solver: - print "N: -- END LINEARIZED ORDER --" + print("N: -- END LINEARIZED ORDER --") return result @@ -301,8 +303,8 @@ class InstallabilitySolver(InstallabilityTester): return result def _dump_groups(self, groups): - print "N: === Groups ===" + print("N: === Groups ===") for (item, adds, rms) in groups: - print "N: %s => A: %s, R: %s" % (str(item), str(adds), str(rms)) - print "N: === END Groups ===" + print("N: %s => A: %s, R: %s" % (str(item), str(adds), str(rms))) + print("N: === END Groups ===") From b64afb639fa5113134dafe679f36a8545697409a Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 06/59] Don't use the file builtin Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 6e361ff..cc68ab9 100755 --- a/britney.py +++ b/britney.py @@ -364,7 +364,7 @@ class Britney(object): # are handled as an ad-hoc case self.MINDAYS = {} self.HINTS = {'command-line': self.HINTS_ALL} - for k, v in [map(string.strip,r.split('=', 1)) for r in file(self.options.config) if '=' in r and not r.strip().startswith('#')]: + for k, v in [map(string.strip,r.split('=', 1)) for r in open(self.options.config) if '=' in r and not r.strip().startswith('#')]: if k.startswith("MINDAYS_"): self.MINDAYS[k.split("_")[1].lower()] = int(v) elif k.startswith("HINTS_"): From 49475f79bafdd07ba5106336eb89e18df879af1d Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 07/59] Simplify a bit the loop to read our config file - split the one-liner into a for and an if - use open() as a context manager - don't use string.strip which is gone in python3 Signed-off-by: Julien Cristau --- britney.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/britney.py b/britney.py index cc68ab9..62af5bd 100755 --- a/britney.py +++ b/britney.py @@ -364,15 +364,20 @@ class Britney(object): # are handled as an ad-hoc case self.MINDAYS = {} self.HINTS = {'command-line': self.HINTS_ALL} - for k, v in [map(string.strip,r.split('=', 1)) for r in open(self.options.config) if '=' in r and not r.strip().startswith('#')]: - if k.startswith("MINDAYS_"): - self.MINDAYS[k.split("_")[1].lower()] = int(v) - elif k.startswith("HINTS_"): - self.HINTS[k.split("_")[1].lower()] = \ - reduce(lambda x,y: x+y, [hasattr(self, "HINTS_" + i) and getattr(self, "HINTS_" + i) or (i,) for i in v.split()]) - elif not hasattr(self.options, k.lower()) or \ - not getattr(self.options, k.lower()): - setattr(self.options, k.lower(), v) + with open(self.options.config) as config: + for line in config: + if '=' in line and not line.strip().startswith('#'): + k, v = line.split('=', 1) + k = k.strip() + v = v.strip() + if k.startswith("MINDAYS_"): + self.MINDAYS[k.split("_")[1].lower()] = int(v) + elif k.startswith("HINTS_"): + self.HINTS[k.split("_")[1].lower()] = \ + reduce(lambda x,y: x+y, [hasattr(self, "HINTS_" + i) and getattr(self, "HINTS_" + i) or (i,) for i in v.split()]) + elif not hasattr(self.options, k.lower()) or \ + not getattr(self.options, k.lower()): + setattr(self.options, k.lower(), v) if not hasattr(self.options, "heidi_delta_output"): self.options.heidi_delta_output = self.options.heidi_output + "Delta" From 24e8e9337cf674fc99d2398e6c2e72a52fa0c60a Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 08/59] Replace map() with list comprehensions As a bonus this removes a use of string.strip (not in python3). Signed-off-by: Julien Cristau --- britney.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/britney.py b/britney.py index 62af5bd..8468846 100755 --- a/britney.py +++ b/britney.py @@ -183,7 +183,6 @@ from __future__ import print_function import os import sys -import string import time import optparse import urllib @@ -389,7 +388,7 @@ class Britney(object): arches += [x for x in allarches if x not in arches and x not in self.options.break_arches.split()] arches += [x for x in allarches if x not in arches and x not in self.options.new_arches.split()] arches += [x for x in allarches if x not in arches] - self.options.architectures = map(intern, arches) + self.options.architectures = [intern(arch) for arch in arches] self.options.smooth_updates = self.options.smooth_updates.split() def __log(self, msg, type="I"): @@ -641,7 +640,7 @@ class Britney(object): # register virtual packages and real packages that provide them if dpkg[PROVIDES]: - parts = map(string.strip, dpkg[PROVIDES].split(",")) + parts = [p.strip() for p in dpkg[PROVIDES].split(",")] for p in parts: if p not in provides: provides[p] = [] From c580fb76839c2eb0f9837a3940ec4a9b54b22b71 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 09/59] Use python3-compatible form for except clause Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 8468846..2f752e2 100755 --- a/britney.py +++ b/britney.py @@ -2628,7 +2628,7 @@ class Britney(object): continue try: readline.write_history_file(histfile) - except IOError, e: + except IOError as e: self.__log("Could not write %s: %s" % (histfile, e), type="W") def do_hint(self, hinttype, who, pkgvers): From b3aef7fe6de8c159b2c9f017d630b18eb83f2028 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 10/59] Stop using dict.iter* methods Signed-off-by: Julien Cristau --- britney.py | 8 ++++---- britney_util.py | 2 +- installability/builder.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/britney.py b/britney.py index 2f752e2..241bc54 100755 --- a/britney.py +++ b/britney.py @@ -485,7 +485,7 @@ class Britney(object): possible_dep_ranges[key] = sat if dep: - for clause in possible_dep_ranges.itervalues(): + for clause in possible_dep_ranges.values(): relations.add_dependency_clause(clause) self._inst_tester = builder.build() @@ -1901,7 +1901,7 @@ class Britney(object): ptuple = check[p] binary, _, parch = ptuple rdeps = [ bin for bin in binaries_t[parch][0][binary][RDEPENDS] \ - if bin in [y[0] for y in smoothbins.itervalues()] ] + if bin in [y[0] for y in smoothbins.values()] ] if rdeps: smoothbins[p] = ptuple @@ -1937,7 +1937,7 @@ class Britney(object): version = self.binaries[suite][parch][0][binary][VERSION] adds.add((binary, version, parch)) - return (adds, rms, set(smoothbins.itervalues())) + return (adds, rms, set(smoothbins.values())) def doop_source(self, item, hint_undo=None, removals=frozenset()): @@ -2829,7 +2829,7 @@ class Britney(object): print('* %s' % (arch,)) - for (src, ver), pkgs in sorted(all.iteritems()): + for (src, ver), pkgs in sorted(all.items()): print(' %s (%s): %s' % (src, ver, ' '.join(sorted(pkgs)))) print diff --git a/britney_util.py b/britney_util.py index 2e14870..3612a36 100644 --- a/britney_util.py +++ b/britney_util.py @@ -220,7 +220,7 @@ def register_reverses(packages, provides, check_doubles=True, iterator=None, the loops. """ if iterator is None: - iterator = packages.iterkeys() + iterator = packages.keys() else: iterator = ifilter_only(packages, iterator) diff --git a/installability/builder.py b/installability/builder.py index 23ff716..becc6b3 100644 --- a/installability/builder.py +++ b/installability/builder.py @@ -388,7 +388,7 @@ class InstallabilityTesterBuilder(object): ekey = (deps, con, rdeps) find_eqv_table[ekey].append(pkg) - for pkg_list in find_eqv_table.itervalues(): + for pkg_list in find_eqv_table.values(): if len(pkg_list) < 2: continue From 63ccd53759d3ebb719a2d091f97c25e11cce6c08 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 11/59] Stop using sys.maxint It doesn't exist in python3, but 1000 days should be safe enough as a fallback for a package without urgency. Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 241bc54..3ff371d 100755 --- a/britney.py +++ b/britney.py @@ -789,7 +789,7 @@ class Britney(object): # read the minimum days associated with the urgencies urgency_old = urgencies.get(l[0], None) - mindays_old = self.MINDAYS.get(urgency_old, sys.maxint) + mindays_old = self.MINDAYS.get(urgency_old, 1000) mindays_new = self.MINDAYS.get(l[2], self.MINDAYS[self.options.default_urgency]) # if the new urgency is lower (so the min days are higher), do nothing From 1e1f574f8a46cab376d184708b725dcef3c03c5b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 12/59] Use the key= argument to sorted() cmp is gone in python3. Also add a sorting method to Excuse that is compatible with its __eq__/__hash__ methods. Signed-off-by: Julien Cristau --- britney.py | 2 +- excuse.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/britney.py b/britney.py index 3ff371d..f29a103 100755 --- a/britney.py +++ b/britney.py @@ -1623,7 +1623,7 @@ class Britney(object): self.excuses.append(excuse) # sort the excuses by daysold and name - self.excuses.sort(key=attrgetter('daysold', 'name')) + self.excuses.sort(key=lambda x: x.sortkey()) # extract the not considered packages, which are in the excuses but not in upgrade_me unconsidered = [e.name for e in self.excuses if e.name not in upgrade_me] diff --git a/excuse.py b/excuse.py index 02b000f..2281dfa 100644 --- a/excuse.py +++ b/excuse.py @@ -62,6 +62,11 @@ class Excuse(object): self.reason = {} self.htmlline = [] + def sortkey(self): + if self.daysold == None: + return (-1, self.name) + return (self.daysold, self.name) + @property def is_valid(self): return self._is_valid @@ -144,7 +149,7 @@ class Excuse(object): for x in self.htmlline: res = res + "
  • " + x + "\n" lastdep = "" - for x in sorted(self.deps, lambda x,y: cmp(x.split('/')[0], y.split('/')[0])): + for x in sorted(self.deps, key=lambda x: x.split('/')[0]): dep = x.split('/')[0] if dep == lastdep: continue lastdep = dep @@ -196,7 +201,7 @@ class Excuse(object): for x in self.htmlline: res.append("" + x + "") lastdep = "" - for x in sorted(self.deps, lambda x,y: cmp(x.split('/')[0], y.split('/')[0])): + for x in sorted(self.deps, key=lambda x: x.split('/')[0]): dep = x.split('/')[0] if dep == lastdep: continue lastdep = dep From d75813eb073476ac6acd7f8074de750f5e24108b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 13/59] Stop using string.find It's gone in python3 Signed-off-by: Julien Cristau --- excuse.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/excuse.py b/excuse.py index 2281dfa..7f87ae8 100644 --- a/excuse.py +++ b/excuse.py @@ -15,7 +15,6 @@ # GNU General Public License for more details. import re -import string class Excuse(object): @@ -137,7 +136,7 @@ class Excuse(object): (self.name, self.name, self.name, self.ver[0], self.ver[1]) if self.maint: res = res + "
  • Maintainer: %s\n" % (self.maint) - if self.section and string.find(self.section, "/") > -1: + if self.section and self.section.find("/") > -1: res = res + "
  • Section: %s\n" % (self.section) if self.daysold != None: if self.daysold < self.mindays: @@ -189,7 +188,7 @@ class Excuse(object): except UnicodeDecodeError: maint = unicode(self.maint,'utf-8') res.append("Maintainer: %s" % maint) - if self.section and string.find(self.section, "/") > -1: + if self.section and self.section.find("/") > -1: res.append("Section: %s" % (self.section)) if self.daysold != None: if self.daysold < self.mindays: From ba981aabc2f26df769c84db85922ddc34d92d6d9 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:09 +0200 Subject: [PATCH 14/59] Add sort method to MigrationItem write_excuses wants them sorted, and python3 doesn't allow sorting arbitrary objects. Signed-off-by: Julien Cristau --- migrationitem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/migrationitem.py b/migrationitem.py index f5b49b2..fe934bc 100644 --- a/migrationitem.py +++ b/migrationitem.py @@ -54,6 +54,9 @@ class MigrationItem(object): def __hash__(self): return hash((self.uvname, self.version)) + def __lt__(self, other): + return (self.uvname, self.version) < (other.uvname, other.version) + @property def name(self): return self._name From b354afc39de26374628720a83703e3a1fa724854 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 15/59] Use six.moves for itertools, urllib and intern They're renamed in python3. Signed-off-by: Julien Cristau --- britney.py | 4 ++-- britney_util.py | 6 +++++- installability/tester.py | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/britney.py b/britney.py index f29a103..1dbfdc4 100755 --- a/britney.py +++ b/britney.py @@ -185,15 +185,15 @@ import os import sys import time import optparse -import urllib import apt_pkg from collections import defaultdict from functools import reduce, partial -from itertools import chain, ifilter, product +from itertools import chain, product from operator import attrgetter +from six.moves import filter as ifilter, intern, urllib_parse as urllib from installability.builder import InstallabilityTesterBuilder from excuse import Excuse diff --git a/britney_util.py b/britney_util.py index 3612a36..e32bb32 100644 --- a/britney_util.py +++ b/britney_util.py @@ -24,12 +24,16 @@ import apt_pkg from functools import partial from datetime import datetime -from itertools import chain, ifilter, ifilterfalse, izip, repeat +from itertools import chain, repeat import os import re import time import yaml +from six.moves import (filter as ifilter, + filterfalse as ifilterfalse, + zip as izip) + from migrationitem import MigrationItem, UnversionnedMigrationItem from consts import (VERSION, BINARIES, PROVIDES, DEPENDS, CONFLICTS, diff --git a/installability/tester.py b/installability/tester.py index 4be7dba..ca0372e 100644 --- a/installability/tester.py +++ b/installability/tester.py @@ -13,7 +13,8 @@ # GNU General Public License for more details. from functools import partial -from itertools import ifilter, ifilterfalse + +from six.moves import filter as ifilter, filterfalse as ifilterfalse from britney_util import iter_except From 49f37a89658a4aa3f23f0ac2035d4e2a340d528b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 16/59] Disable a code path for encoded yaml in python3 The comment says we should no longer need that. Signed-off-by: Julien Cristau --- excuse.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/excuse.py b/excuse.py index 7f87ae8..e0e1a3f 100644 --- a/excuse.py +++ b/excuse.py @@ -15,7 +15,7 @@ # GNU General Public License for more details. import re - +import six class Excuse(object): """Excuse class @@ -183,10 +183,11 @@ class Excuse(object): maint = self.maint # ugly hack to work around strange encoding in pyyaml # should go away with pyyaml in python 3 - try: - maint.decode('ascii') - except UnicodeDecodeError: - maint = unicode(self.maint,'utf-8') + if isinstance(maint, six.binary_type): + try: + maint.decode('ascii') + except UnicodeDecodeError: + maint = six.string_type(self.maint,'utf-8') res.append("Maintainer: %s" % maint) if self.section and self.section.find("/") > -1: res.append("Section: %s" % (self.section)) From 71b21083b7f700a2b0b67b3976d3ee5837690d01 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 17/59] Use super() instead of explicitly calling our superclass Signed-off-by: Julien Cristau --- installability/solver.py | 4 ++-- migrationitem.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/installability/solver.py b/installability/solver.py index 8ca402e..5841a10 100644 --- a/installability/solver.py +++ b/installability/solver.py @@ -45,8 +45,8 @@ class InstallabilitySolver(InstallabilityTester): - NB: arch:all packages are "re-mapped" to given architecture. (simplifies caches and dependency checking) """ - InstallabilityTester.__init__(self, universe, revuniverse, testing, - broken, essentials, safe_set, eqv_table) + super(InstallabilitySolver, self).__init__(universe, revuniverse, testing, + broken, essentials, safe_set, eqv_table) def solve_groups(self, groups): diff --git a/migrationitem.py b/migrationitem.py index fe934bc..1b94403 100644 --- a/migrationitem.py +++ b/migrationitem.py @@ -145,5 +145,5 @@ class MigrationItem(object): return self._uvname class UnversionnedMigrationItem(MigrationItem): - def __init__(self, name = None): - MigrationItem.__init__(self, name = name, versionned = False) + def __init__(self, name=None): + super(UnversionnedMigrationItem, self).__init__(name=name, versionned=False) From 8044667ed6c2cf4354426d6ca54f1b31c58fb560 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 18/59] Use python3 Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 1dbfdc4..fe29615 100755 --- a/britney.py +++ b/britney.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.7 -u +#!/usr/bin/python3 -u # -*- coding: utf-8 -*- # Copyright (C) 2001-2008 Anthony Towns From c42fbcc5d3362ba0e9490c9178d4514f3d757be1 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 19/59] Remove dependency on six Signed-off-by: Julien Cristau --- britney.py | 26 +++++++++++++------------- britney_util.py | 16 ++++++---------- excuse.py | 8 -------- installability/tester.py | 13 ++++++------- 4 files changed, 25 insertions(+), 38 deletions(-) diff --git a/britney.py b/britney.py index fe29615..c76f16e 100755 --- a/britney.py +++ b/britney.py @@ -193,7 +193,7 @@ from functools import reduce, partial from itertools import chain, product from operator import attrgetter -from six.moves import filter as ifilter, intern, urllib_parse as urllib +from urllib.parse import quote from installability.builder import InstallabilityTesterBuilder from excuse import Excuse @@ -388,7 +388,7 @@ class Britney(object): arches += [x for x in allarches if x not in arches and x not in self.options.break_arches.split()] arches += [x for x in allarches if x not in arches and x not in self.options.new_arches.split()] arches += [x for x in allarches if x not in arches] - self.options.architectures = [intern(arch) for arch in arches] + self.options.architectures = [sys.intern(arch) for arch in arches] self.options.smooth_updates = self.options.smooth_updates.split() def __log(self, msg, type="I"): @@ -447,7 +447,7 @@ class Britney(object): # the package name extracted from the field and is therefore # not interned. pdata = binaries[dep_dist][arch][0][p] - pt = (intern(p), pdata[VERSION], arch) + pt = (sys.intern(p), pdata[VERSION], arch) if dep: sat.add(pt) elif t != pt: @@ -494,7 +494,7 @@ class Britney(object): # Data reading/writing methods # ---------------------------- - def read_sources(self, basedir, intern=intern): + def read_sources(self, basedir, intern=sys.intern): """Read the list of source packages from the specified directory The source packages are read from the `Sources' file within the @@ -533,7 +533,7 @@ class Britney(object): ] return sources - def read_binaries(self, basedir, distribution, arch, intern=intern): + def read_binaries(self, basedir, distribution, arch, intern=sys.intern): """Read the list of binary packages from the specified directory The binary packages are read from the `Packages_${arch}' files @@ -1081,7 +1081,7 @@ class Britney(object): anyworthdoing = False # for every binary package produced by this source in unstable for this architecture - for pkg in sorted(ifilter(lambda x: x.endswith("/" + arch), source_u[BINARIES]), key=lambda x: x.split("/")[0]): + for pkg in sorted(filter(lambda x: x.endswith("/" + arch), source_u[BINARIES]), key=lambda x: x.split("/")[0]): pkg_name = pkg.split("/")[0] # retrieve the testing (if present) and unstable corresponding binary packages @@ -1343,7 +1343,7 @@ class Britney(object): base = 'testing' else: base = 'stable' - text = "Not yet built on %s (relative to testing)" % (urllib.quote(arch), urllib.quote(src), urllib.quote(source_u[VERSION]), base, arch) + text = "Not yet built on %s (relative to testing)" % (quote(arch), quote(src), quote(source_u[VERSION]), base, arch) if arch in self.options.fucked_arches.split(): text = text + " (but %s isn't keeping up, so never mind)" % (arch) @@ -1396,15 +1396,15 @@ class Britney(object): if oodtxt: oodtxt = oodtxt + "; " oodtxt = oodtxt + "%s (from %s)" % \ - (", ".join(sorted(oodbins[v])), urllib.quote(arch), urllib.quote(src), urllib.quote(v), v) + (", ".join(sorted(oodbins[v])), quote(arch), quote(src), quote(v), v) if uptodatebins: text = "old binaries left on %s: %s" % \ - (urllib.quote(arch), urllib.quote(src), urllib.quote(source_u[VERSION]), arch, oodtxt) + (quote(arch), quote(src), quote(source_u[VERSION]), arch, oodtxt) else: text = "missing build on %s: %s" % \ - (urllib.quote(arch), urllib.quote(src), urllib.quote(source_u[VERSION]), arch, oodtxt) + (quote(arch), quote(src), quote(source_u[VERSION]), arch, oodtxt) if arch in self.options.fucked_arches.split(): text = text + " (but %s isn't keeping up, so nevermind)" % (arch) @@ -1454,15 +1454,15 @@ class Britney(object): if len(new_bugs) > 0: excuse.addhtml("%s (%s) has new bugs!" % (pkg, ", ".join(pkgs[pkg]), urllib.quote(pkg))) + "target=\"_blank\">has new bugs!" % (pkg, ", ".join(pkgs[pkg]), quote(pkg))) excuse.addhtml("Updating %s introduces new bugs: %s" % (pkg, ", ".join( - ["#%s" % (urllib.quote(a), a) for a in new_bugs]))) + ["#%s" % (quote(a), a) for a in new_bugs]))) update_candidate = False excuse.addreason("buggy") if len(old_bugs) > 0: excuse.addhtml("Updating %s fixes old bugs: %s" % (pkg, ", ".join( - ["#%s" % (urllib.quote(a), a) for a in old_bugs]))) + ["#%s" % (quote(a), a) for a in old_bugs]))) if len(old_bugs) > len(new_bugs) and len(new_bugs) > 0: excuse.addhtml("%s introduces new bugs, so still ignored (even " "though it fixes more than it introduces, whine at debian-release)" % pkg) diff --git a/britney_util.py b/britney_util.py index e32bb32..16d5298 100644 --- a/britney_util.py +++ b/britney_util.py @@ -24,16 +24,12 @@ import apt_pkg from functools import partial from datetime import datetime -from itertools import chain, repeat +from itertools import chain, repeat, filterfalse import os import re import time import yaml -from six.moves import (filter as ifilter, - filterfalse as ifilterfalse, - zip as izip) - from migrationitem import MigrationItem, UnversionnedMigrationItem from consts import (VERSION, BINARIES, PROVIDES, DEPENDS, CONFLICTS, @@ -75,8 +71,8 @@ def ifilter_except(container, iterable=None): iterators that are not known on beforehand. """ if iterable is not None: - return ifilterfalse(container.__contains__, iterable) - return partial(ifilterfalse, container.__contains__) + return filterfalse(container.__contains__, iterable) + return partial(filterfalse, container.__contains__) def ifilter_only(container, iterable=None): @@ -88,8 +84,8 @@ def ifilter_only(container, iterable=None): iterators that are not known on beforehand. """ if iterable is not None: - return ifilter(container.__contains__, iterable) - return partial(ifilter, container.__contains__) + return filter(container.__contains__, iterable) + return partial(filter, container.__contains__) # iter_except is from the "itertools" recipe @@ -301,7 +297,7 @@ def compute_reverse_tree(packages_s, pkg, arch, # generate the next iteration, which is the reverse-dependencies of # the current iteration rev_deps = set(revfilt(flatten( binaries[x][RDEPENDS] for x in binfilt(rev_deps) ))) - return izip(seen, repeat(arch)) + return zip(seen, repeat(arch)) def write_nuninst(filename, nuninst): diff --git a/excuse.py b/excuse.py index e0e1a3f..66192fb 100644 --- a/excuse.py +++ b/excuse.py @@ -15,7 +15,6 @@ # GNU General Public License for more details. import re -import six class Excuse(object): """Excuse class @@ -181,13 +180,6 @@ class Excuse(object): (self.name, self.ver[0], self.ver[1])) if self.maint: maint = self.maint - # ugly hack to work around strange encoding in pyyaml - # should go away with pyyaml in python 3 - if isinstance(maint, six.binary_type): - try: - maint.decode('ascii') - except UnicodeDecodeError: - maint = six.string_type(self.maint,'utf-8') res.append("Maintainer: %s" % maint) if self.section and self.section.find("/") > -1: res.append("Section: %s" % (self.section)) diff --git a/installability/tester.py b/installability/tester.py index ca0372e..3dd5118 100644 --- a/installability/tester.py +++ b/installability/tester.py @@ -13,8 +13,7 @@ # GNU General Public License for more details. from functools import partial - -from six.moves import filter as ifilter, filterfalse as ifilterfalse +from itertools import filterfalse from britney_util import iter_except @@ -85,7 +84,7 @@ class InstallabilityTester(object): eqv_table = self._eqv_table testing = self._testing tcopy = [x for x in testing] - for t in ifilterfalse(cache_inst.__contains__, tcopy): + for t in filterfalse(cache_inst.__contains__, tcopy): if t in cbroken: continue res = check_inst(t) @@ -300,7 +299,7 @@ class InstallabilityTester(object): # We already satisfied/chosen at least one of the litterals # in the choice, so the choice is gone - for choice in ifilter(musts.isdisjoint, choices): + for choice in filter(musts.isdisjoint, choices): # cbroken is needed here because (in theory) it could # have changed since the choice was discovered and it # is smaller than testing (so presumably faster) @@ -308,7 +307,7 @@ class InstallabilityTester(object): if len(remain) > 1 and not remain.isdisjoint(safe_set): first = None - for r in ifilter(safe_set.__contains__, remain): + for r in filter(safe_set.__contains__, remain): # don't bother giving extra arguments to _check_inst. "safe" packages are # usually trivial to satisfy on their own and will not involve conflicts # (so never will not help) @@ -432,7 +431,7 @@ class InstallabilityTester(object): returns True, then t is installable. """ # Local variables for faster access... - not_satisfied = partial(ifilter, musts.isdisjoint) + not_satisfied = partial(filter, musts.isdisjoint) # While we have guaranteed dependencies (in check), examine all # of them. @@ -520,7 +519,7 @@ class InstallabilityTester(object): start = set(ess_base) ess_never = set() ess_choices = set() - not_satisified = partial(ifilter, start.isdisjoint) + not_satisified = partial(filter, start.isdisjoint) while ess_base: self._check_loop(universe, testing, eqv_table, From e8c84e8cc7a98ac6065a8265bc9a04cf99c40c5a Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 20/59] Use python3-style super() --- installability/solver.py | 4 ++-- migrationitem.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/installability/solver.py b/installability/solver.py index 5841a10..274ed53 100644 --- a/installability/solver.py +++ b/installability/solver.py @@ -45,8 +45,8 @@ class InstallabilitySolver(InstallabilityTester): - NB: arch:all packages are "re-mapped" to given architecture. (simplifies caches and dependency checking) """ - super(InstallabilitySolver, self).__init__(universe, revuniverse, testing, - broken, essentials, safe_set, eqv_table) + super().__init__(universe, revuniverse, testing, + broken, essentials, safe_set, eqv_table) def solve_groups(self, groups): diff --git a/migrationitem.py b/migrationitem.py index 1b94403..0757626 100644 --- a/migrationitem.py +++ b/migrationitem.py @@ -146,4 +146,4 @@ class MigrationItem(object): class UnversionnedMigrationItem(MigrationItem): def __init__(self, name=None): - super(UnversionnedMigrationItem, self).__init__(name=name, versionned=False) + super().__init__(name=name, versionned=False) From 7b0138ecb7b7cff5f8ab05c5e41499817bbe6e6c Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 21/59] Don't crash if the urgencies file contains non-ascii The live-2011-12-13 test set has random garbage in the middle. Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index c76f16e..791ee4a 100755 --- a/britney.py +++ b/britney.py @@ -783,7 +783,7 @@ class Britney(object): urgencies = {} filename = os.path.join(basedir, "Urgency") self.__log("Loading upload urgencies from %s" % filename) - for line in open(filename): + for line in open(filename, errors='surrogateescape'): l = line.split() if len(l) != 3: continue From f64f7072c0e6ebef33222feb4adb2779dd7bcb75 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 22/59] Sources and Packages are utf-8 Signed-off-by: Julien Cristau --- britney.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/britney.py b/britney.py index 791ee4a..8945e0c 100755 --- a/britney.py +++ b/britney.py @@ -509,7 +509,8 @@ class Britney(object): filename = os.path.join(basedir, "Sources") self.__log("Loading source packages from %s" % filename) - Packages = apt_pkg.TagFile(open(filename)) + with open(filename, encoding='utf-8') as f: + Packages = apt_pkg.TagFile(f) get_field = Packages.section.get step = Packages.step @@ -564,7 +565,8 @@ class Britney(object): filename = os.path.join(basedir, "Packages_%s" % arch) self.__log("Loading binary packages from %s" % filename) - Packages = apt_pkg.TagFile(open(filename)) + with open(filename, encoding='utf-8') as f: + Packages = apt_pkg.TagFile(f) get_field = Packages.section.get step = Packages.step From 62c1c9ec3c2da8c051e58f5d72c74999fe75f85b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 23/59] Add explicit encoding to all files Signed-off-by: Julien Cristau --- britney.py | 35 +++++++++++++++++------------------ britney_util.py | 16 ++++++++-------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/britney.py b/britney.py index 8945e0c..2bc0d4b 100755 --- a/britney.py +++ b/britney.py @@ -363,7 +363,7 @@ class Britney(object): # are handled as an ad-hoc case self.MINDAYS = {} self.HINTS = {'command-line': self.HINTS_ALL} - with open(self.options.config) as config: + with open(self.options.config, encoding='utf-8') as config: for line in config: if '=' in line and not line.strip().startswith('#'): k, v = line.split('=', 1) @@ -675,7 +675,7 @@ class Britney(object): bugs = defaultdict(list) filename = os.path.join(basedir, "BugsV") self.__log("Loading RC bugs data from %s" % filename) - for line in open(filename): + for line in open(filename, encoding='ascii'): l = line.split() if len(l) != 2: self.__log("Malformed line found in line %s" % (line), type='W') @@ -745,7 +745,7 @@ class Britney(object): dates = {} filename = os.path.join(basedir, "Dates") self.__log("Loading upload data from %s" % filename) - for line in open(filename): + for line in open(filename, encoding='ascii'): l = line.split() if len(l) != 3: continue try: @@ -762,10 +762,9 @@ class Britney(object): """ filename = os.path.join(basedir, "Dates") self.__log("Writing upload data to %s" % filename) - f = open(filename, 'w') - for pkg in sorted(dates): - f.write("%s %s %d\n" % ((pkg,) + dates[pkg])) - f.close() + with open(filename, 'w', encoding='utf-8'): + for pkg in sorted(dates): + f.write("%s %s %d\n" % ((pkg,) + dates[pkg])) def read_urgencies(self, basedir): @@ -785,7 +784,7 @@ class Britney(object): urgencies = {} filename = os.path.join(basedir, "Urgency") self.__log("Loading upload urgencies from %s" % filename) - for line in open(filename, errors='surrogateescape'): + for line in open(filename, errors='surrogateescape', encoding='ascii'): l = line.split() if len(l) != 3: continue @@ -839,7 +838,8 @@ class Britney(object): self.__log("Cannot read hints list from %s, no such file!" % filename, type="E") continue self.__log("Loading hints list from %s" % filename) - lines = open(filename) + with open(filename, encoding='utf-8') as f: + lines = f.readlines() for line in lines: line = line.strip() if line == "": continue @@ -2858,16 +2858,15 @@ class Britney(object): else: self.upgrade_me = self.options.actions.split() - self.__output = open(self.options.upgrade_output, 'w') + with open(self.options.upgrade_output, 'w', encoding='utf-8') as f: + self.__output = f - # run the hint tester - if self.options.hint_tester: - self.hint_tester() - # run the upgrade test - else: - self.upgrade_testing() - - self.__output.close() + # run the hint tester + if self.options.hint_tester: + self.hint_tester() + # run the upgrade test + else: + self.upgrade_testing() def _installability_test(self, pkg_name, pkg_version, pkg_arch, broken, to_check, nuninst_arch): """Test for installability of a package on an architecture diff --git a/britney_util.py b/britney_util.py index 16d5298..3766170 100644 --- a/britney_util.py +++ b/britney_util.py @@ -306,7 +306,7 @@ def write_nuninst(filename, nuninst): Write the non-installable report derived from "nuninst" to the file denoted by "filename". """ - with open(filename, 'w') as f: + with open(filename, 'w', encoding='utf-8') as f: # Having two fields with (almost) identical dates seems a bit # redundant. f.write("Built on: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "\n") @@ -323,7 +323,7 @@ def read_nuninst(filename, architectures): will be included in the report. """ nuninst = {} - with open(filename) as f: + with open(filename, encoding='ascii') as f: for r in f: if ":" not in r: continue arch, packages = r.strip().split(":", 1) @@ -381,7 +381,7 @@ def write_heidi(filename, sources_t, packages_t, The "X=X" parameters are optimizations to avoid "load global" in the loops. """ - with open(filename, 'w') as f: + with open(filename, 'w', encoding='ascii') as f: # write binary packages for arch in sorted(packages_t): @@ -420,7 +420,7 @@ def write_heidi_delta(filename, all_selected): The order corresponds to that shown in update_output. """ - with open(filename, "w") as fd: + with open(filename, "w", encoding='ascii') as fd: fd.write("#HeidiDelta\n") @@ -456,7 +456,7 @@ def write_excuses(excuses, dest_file, output_format="yaml"): or "legacy-html". """ if output_format == "yaml": - with open(dest_file, 'w') as f: + with open(dest_file, 'w', encoding='utf-8') as f: excuselist = [] for e in excuses: excuselist.append(e.excusedata()) @@ -465,7 +465,7 @@ def write_excuses(excuses, dest_file, output_format="yaml"): excusesdata["generated-date"] = datetime.utcnow() f.write(yaml.dump(excusesdata, default_flow_style=False, allow_unicode=True)) elif output_format == "legacy-html": - with open(dest_file, 'w') as f: + with open(dest_file, 'w', encoding='utf-8') as f: f.write("\n") f.write("excuses...") f.write("\n") @@ -488,7 +488,7 @@ def write_sources(sources_s, filename): key_pairs = ((VERSION, 'Version'), (SECTION, 'Section'), (MAINTAINER, 'Maintainer')) - with open(filename, 'w') as f: + with open(filename, 'w', encoding='utf-8') as f: for src in sources_s: src_data = sources_s[src] output = "Package: %s\n" % src @@ -518,7 +518,7 @@ def write_controlfiles(sources, packages, suite, basedir): for arch in packages_s: filename = os.path.join(basedir, 'Packages_%s' % arch) binaries = packages_s[arch][0] - with open(filename, 'w') as f: + with open(filename, 'w', encoding='utf-8') as f: for pkg in binaries: output = "Package: %s\n" % pkg bin_data = binaries[pkg] From 14107520d7c21226419c89b93cc1c5ab78639725 Mon Sep 17 00:00:00 2001 From: Ivo De Decker Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 24/59] Make sure reason is a list, even with python 3 Signed-off-by: Ivo De Decker --- excuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/excuse.py b/excuse.py index 66192fb..0171926 100644 --- a/excuse.py +++ b/excuse.py @@ -223,7 +223,7 @@ class Excuse(object): excusedata["forced-reason"] = self.reason.keys() excusedata["reason"] = [] else: - excusedata["reason"] = self.reason.keys() + excusedata["reason"] = list(self.reason.keys()) excusedata["is-candidate"] = self.is_valid return excusedata From d127ac65ae99031b90596a03a600efbd5754d4a9 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 25/59] Fix silly NameError Signed-off-by: Julien Cristau --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 2bc0d4b..0b63524 100755 --- a/britney.py +++ b/britney.py @@ -762,7 +762,7 @@ class Britney(object): """ filename = os.path.join(basedir, "Dates") self.__log("Writing upload data to %s" % filename) - with open(filename, 'w', encoding='utf-8'): + with open(filename, 'w', encoding='utf-8') as f: for pkg in sorted(dates): f.write("%s %s %d\n" % ((pkg,) + dates[pkg])) From 3a05c6ba918a70db5c2f8f90e8511a942260d468 Mon Sep 17 00:00:00 2001 From: Ivo De Decker Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 26/59] Also make sure forced-reason is a list Signed-off-by: Ivo De Decker --- excuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/excuse.py b/excuse.py index 0171926..87f5538 100644 --- a/excuse.py +++ b/excuse.py @@ -220,7 +220,7 @@ class Excuse(object): excusedata["new-bugs"] = sorted(self.newbugs) excusedata["old-bugs"] = sorted(self.oldbugs) if self.forced: - excusedata["forced-reason"] = self.reason.keys() + excusedata["forced-reason"] = list(self.reason.keys()) excusedata["reason"] = [] else: excusedata["reason"] = list(self.reason.keys()) From 3710980b102458ed113997413391f2f0d451baef Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 27/59] get_dependency_solvers: Avoid unnecessary boolean ret value The get_dependency_solvers method returns a (boolean, list)-tuple, but the boolean can always be implied from the list (in boolean context). Signed-off-by: Niels Thykier --- britney.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/britney.py b/britney.py index 0b63524..e2a9266 100755 --- a/britney.py +++ b/britney.py @@ -441,7 +441,7 @@ class Britney(object): sat = set() for dep_dist in binaries: - (_, pkgs) = solvers(block, arch, dep_dist) + pkgs = solvers(block, arch, dep_dist) for p in pkgs: # version and arch is already interned, but solvers use # the package name extracted from the field and is therefore @@ -946,7 +946,8 @@ class Britney(object): if op == '' and version == '' and archqual is None: packages.append(prov) - return (len(packages) > 0, packages) + return packages + def excuse_unsat_deps(self, pkg, src, arch, suite, excuse): """Find unsatisfied dependencies for a binary package @@ -972,15 +973,15 @@ class Britney(object): # for every dependency block (formed as conjunction of disjunction) for block, block_txt in zip(parse_depends(deps, False), deps.split(',')): # if the block is satisfied in testing, then skip the block - solved, packages = get_dependency_solvers(block, arch, 'testing') - if solved: + packages = get_dependency_solvers(block, arch, 'testing') + if packages: for p in packages: if p not in self.binaries[suite][arch][0]: continue excuse.add_sane_dep(self.binaries[suite][arch][0][p][SOURCE]) continue # check if the block can be satisfied in unstable, and list the solving packages - solved, packages = get_dependency_solvers(block, arch, suite) + packages = get_dependency_solvers(block, arch, suite) packages = [self.binaries[suite][arch][0][p][SOURCE] for p in packages] # if the dependency can be satisfied by the same source package, skip the block: @@ -988,7 +989,7 @@ class Britney(object): if src in packages: continue # if no package can satisfy the dependency, add this information to the excuse - if len(packages) == 0: + if not packages: excuse.addhtml("%s/%s unsatisfiable Depends: %s" % (pkg, arch, block_txt.strip())) excuse.addreason("depends"); continue From 11e84d01a370e24919aafedc1ed7a3a1a09b144e Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 28/59] britney: Optimise the original auto-hinter a bit Notably: * Avoid repeated calls frozenset(X), where we can trivially do without. * Skip the inner loop, when "i" is in "to_skip". * Use a set rather than a list for "to_skip" as we do more membership tests. Signed-off-by: Niels Thykier --- britney.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/britney.py b/britney.py index e2a9266..688d147 100755 --- a/britney.py +++ b/britney.py @@ -2801,22 +2801,33 @@ class Britney(object): if not looped and len(items) > 1: mincands.append(items[:]) looped = True - if len(items) > 1 and frozenset(items) != frozenset(mincands[-1]): + if (len(items) > 1 and len(items) != len(mincands[-1]) and + frozenset(items) != frozenset(mincands[-1])): candidates.append(items) for l in [ candidates, mincands ]: - to_skip = [] + to_skip = set() for i in range(len(l)): + if i in to_skip: + continue + l_i = None for j in range(i+1, len(l)): - if i in to_skip or j in to_skip: + if j in to_skip: # we already know this list isn't interesting continue - elif frozenset(l[i]) >= frozenset(l[j]): + if l_i is None: + l_i = frozenset(l[i]) + l_j = frozenset(l[j]) + if l_i >= l_j: # j is a subset of i; ignore it - to_skip.append(j) - elif frozenset(l[i]) <= frozenset(l[j]): - # i is a subset of j; ignore it - to_skip.append(i) + to_skip.add(j) + elif l_i < l_j: + # i is a subset of j; ignore it and the rest of the + # "i" series. + # NB: We use < and not <= because the "==" case is + # already covered above + to_skip.add(i) + break for i in range(len(l)): if i not in to_skip: self.do_hint("easy", "autohinter", [ MigrationItem("%s/%s" % (x[0], x[1])) for x in l[i] ]) From a13386ae343ee7390f177cbcc97c95954ab10acf Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 29/59] britney.py: Avoid some redundancy in auto_hinter() Signed-off-by: Niels Thykier --- britney.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/britney.py b/britney.py index 688d147..8b9f7c0 100755 --- a/britney.py +++ b/britney.py @@ -2767,11 +2767,11 @@ class Britney(object): if e not in excuses: return False excuse = excuses[e] - if e in self.sources['testing'] and self.sources['testing'][e][VERSION] == excuse.ver[1]: + if e in sources_t and sources_t[e][VERSION] == excuse.ver[1]: return True if not circular_first: hint[e] = excuse.ver[1] - if len(excuse.deps) == 0: + if not excuse.deps: return hint for p in excuse.deps: if p in hint: continue @@ -2784,9 +2784,9 @@ class Britney(object): mincands = [] for e in excuses: excuse = excuses[e] - if e in self.sources['testing'] and self.sources['testing'][e][VERSION] == excuse.ver[1]: + if e in sources_t and sources_t[e][VERSION] == excuse.ver[1]: continue - if len(excuse.deps) > 0: + if excuse.deps: hint = find_related(e, {}, True) if isinstance(hint, dict) and e in hint and hint not in candidates: candidates.append(hint.items()) From 3230d1dc75bee7b62f9eea5f7e618b8476ff320b Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 30/59] britney.py: Fix use before assignment Signed-off-by: Niels Thykier --- britney.py | 1 + 1 file changed, 1 insertion(+) diff --git a/britney.py b/britney.py index 8b9f7c0..b18b5e1 100755 --- a/britney.py +++ b/britney.py @@ -832,6 +832,7 @@ class Britney(object): for who in self.HINTS.keys(): if who == 'command-line': lines = self.options.hints and self.options.hints.split(';') or () + filename = '' else: filename = os.path.join(basedir, "Hints", who) if not os.path.isfile(filename): From a5aad846991c357337e3ad62484856d39a0cbf97 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:10 +0200 Subject: [PATCH 31/59] britney.py: Remove trailing semi-colons Signed-off-by: Niels Thykier --- britney.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/britney.py b/britney.py index b18b5e1..4a9caa1 100755 --- a/britney.py +++ b/britney.py @@ -992,7 +992,7 @@ class Britney(object): # if no package can satisfy the dependency, add this information to the excuse if not packages: excuse.addhtml("%s/%s unsatisfiable Depends: %s" % (pkg, arch, block_txt.strip())) - excuse.addreason("depends"); + excuse.addreason("depends") continue # for the solving packages, update the excuse to add the dependencies @@ -1225,7 +1225,7 @@ class Britney(object): if source_t and apt_pkg.version_compare(source_u[VERSION], source_t[VERSION]) < 0: excuse.addhtml("ALERT: %s is newer in testing (%s %s)" % (src, source_t[VERSION], source_u[VERSION])) self.excuses.append(excuse) - excuse.addreason("newerintesting"); + excuse.addreason("newerintesting") return False # check if the source package really exists or if it is a fake one From bd5b3ac4ecc0709215c26dd412c1370c58349e3e Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 32/59] Remove unused assignments/parameters Signed-off-by: Niels Thykier --- britney.py | 1 - britney_util.py | 2 +- completer.py | 1 - hints.py | 2 +- installability/builder.py | 3 +-- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/britney.py b/britney.py index 4a9caa1..d530f44 100755 --- a/britney.py +++ b/britney.py @@ -2514,7 +2514,6 @@ class Britney(object): # a package is obsolete if none of the binary packages in testing # are built by it self.__log("> Removing obsolete source packages from testing", type="I") - removals = [] # local copies for performance sources = self.sources['testing'] binaries = self.binaries['testing'] diff --git a/britney_util.py b/britney_util.py index 3766170..fbe39f5 100644 --- a/britney_util.py +++ b/britney_util.py @@ -115,7 +115,7 @@ def iter_except(func, exception, first=None): def undo_changes(lundo, inst_tester, sources, binaries, - BINARIES=BINARIES, PROVIDES=PROVIDES): + BINARIES=BINARIES): """Undoes one or more changes to testing * lundo is a list of (undo, item)-tuples diff --git a/completer.py b/completer.py index e5b0ca0..27437c1 100644 --- a/completer.py +++ b/completer.py @@ -39,7 +39,6 @@ class Completer(object): complete = [] tpu = [] for e in britney.excuses: - ver = None pkg = e.name suite = 'unstable' if pkg[0] == '-': diff --git a/hints.py b/hints.py index ab47675..6643257 100644 --- a/hints.py +++ b/hints.py @@ -24,7 +24,7 @@ class HintCollection(object): return self.search(type) def search(self, type=None, onlyactive=True, package=None, \ - version=None, days=None, removal=None): + version=None, removal=None): return [ hint for hint in self._hints if (type is None or type == hint.type) and diff --git a/installability/builder.py b/installability/builder.py index becc6b3..2998d16 100644 --- a/installability/builder.py +++ b/installability/builder.py @@ -29,7 +29,7 @@ class _RelationBuilder(object): self._new_breaks = set(binary_data[1]) - def add_dependency_clause(self, or_clause, frozenset=frozenset): + def add_dependency_clause(self, or_clause): """Add a dependency clause The clause must be a sequence of (name, version, architecture) @@ -48,7 +48,6 @@ class _RelationBuilder(object): clause = self._itbuilder._intern_set(or_clause) binary = self._binary itbuilder = self._itbuilder - package_table = itbuilder._package_table okay = False for dep_tuple in clause: okay = True From 9ace17b38a50a19112c84e92a4b8d0236191fdf8 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 33/59] solver.py: Remove unused import Signed-off-by: Niels Thykier --- installability/solver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/installability/solver.py b/installability/solver.py index 274ed53..9e1f1d8 100644 --- a/installability/solver.py +++ b/installability/solver.py @@ -16,7 +16,6 @@ from __future__ import print_function -from functools import partial import os from installability.tester import InstallabilityTester From 222f7114fcdde3d7643b3a1c5b30e7fa0c6dd3fb Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 34/59] Compute simple stats installability tester graph Signed-off-by: Niels Thykier --- britney.py | 8 ++++ installability/tester.py | 85 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/britney.py b/britney.py index d530f44..a396ba5 100755 --- a/britney.py +++ b/britney.py @@ -309,6 +309,14 @@ class Britney(object): else: write_nuninst(self.options.noninst_status, nuninst) + stats = self._inst_tester.compute_stats() + self.__log("> Installability tester statistics (per architecture)", type="I") + for arch in self.options.architectures: + arch_stat = stats[arch] + self.__log("> %s" % arch, type="I") + for stat in arch_stat.stat_summary(): + self.__log("> - %s" % stat, type="I") + # read the release-critical bug summaries for testing and unstable self.bugs = {'unstable': self.read_bugs(self.options.unstable), 'testing': self.read_bugs(self.options.testing),} diff --git a/installability/tester.py b/installability/tester.py index 3dd5118..f38034c 100644 --- a/installability/tester.py +++ b/installability/tester.py @@ -12,8 +12,9 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +from collections import defaultdict from functools import partial -from itertools import filterfalse +from itertools import chain, filterfalse from britney_util import iter_except @@ -548,3 +549,85 @@ class InstallabilityTester(object): return self._cache_ess[arch] + def compute_stats(self): + universe = self._universe + eqv_table = self._eqv_table + graph_stats = defaultdict(ArchStats) + seen_eqv = defaultdict(set) + + for pkg in universe: + (pkg_name, pkg_version, pkg_arch) = pkg + deps, con = universe[pkg] + arch_stats = graph_stats[pkg_arch] + + arch_stats.nodes += 1 + + if pkg in eqv_table and pkg not in seen_eqv[pkg_arch]: + eqv = [e for e in eqv_table[pkg] if e[2] == pkg_arch] + arch_stats.eqv_nodes += len(eqv) + + arch_stats.add_dep_edges(deps) + arch_stats.add_con_edges(con) + + for stat in graph_stats.values(): + stat.compute_all() + + return graph_stats + + +class ArchStats(object): + + def __init__(self): + self.nodes = 0 + self.eqv_nodes = 0 + self.dep_edges = [] + self.con_edges = [] + self.stats = defaultdict(lambda: defaultdict(int)) + + def stat(self, statname): + return self.stats[statname] + + def stat_summary(self): + text = [] + for statname in ['nodes', 'dependency-clauses', 'dependency-clause-alternatives', 'negative-dependency-clauses']: + stat = self.stats[statname] + if statname != 'nodes': + format_str = "%s, max: %d, min: %d, median: %d, average: %f (%d/%d)" + values = [statname, stat['max'], stat['min'], stat['median'], stat['average'], stat['sum'], stat['size']] + if 'average-per-node' in stat: + format_str += ", average-per-node: %f" + values.append(stat['average-per-node']) + else: + format_str = "nodes: %d, eqv-nodes: %d" + values = (self.nodes, self.eqv_nodes) + text.append(format_str % tuple(values)) + return text + + def add_dep_edges(self, edges): + self.dep_edges.append(edges) + + def add_con_edges(self, edges): + self.con_edges.append(edges) + + def _list_stats(self, stat_name, sorted_list, average_per_node=False): + if sorted_list: + stats = self.stats[stat_name] + stats['max'] = sorted_list[-1] + stats['min'] = sorted_list[0] + stats['sum'] = sum(sorted_list) + stats['size'] = len(sorted_list) + stats['average'] = float(stats['sum'])/len(sorted_list) + stats['median'] = sorted_list[len(sorted_list)//2] + if average_per_node: + stats['average-per-node'] = float(stats['sum'])/self.nodes + + def compute_all(self): + dep_edges = self.dep_edges + con_edges = self.con_edges + sorted_no_dep_edges = sorted(len(x) for x in dep_edges) + sorted_size_dep_edges = sorted(len(x) for x in chain.from_iterable(dep_edges)) + sorted_no_con_edges = sorted(len(x) for x in con_edges) + self._list_stats('dependency-clauses', sorted_no_dep_edges) + self._list_stats('dependency-clause-alternatives', sorted_size_dep_edges, average_per_node=True) + self._list_stats('negative-dependency-clauses', sorted_no_con_edges) + From 082139f4e3edac9ea5f4b9e92662011d9ecd3747 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 35/59] Collect more statistics from the installability tester Signed-off-by: Niels Thykier --- britney.py | 4 ++ installability/tester.py | 83 ++++++++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/britney.py b/britney.py index a396ba5..63ded46 100755 --- a/britney.py +++ b/britney.py @@ -2888,6 +2888,10 @@ class Britney(object): else: self.upgrade_testing() + self.__log('> Stats from the installability tester', type="I") + for stat in self._inst_tester.stats.stats(): + self.__log('> %s' % stat, type="I") + def _installability_test(self, pkg_name, pkg_version, pkg_arch, broken, to_check, nuninst_arch): """Test for installability of a package on an architecture diff --git a/installability/tester.py b/installability/tester.py index f38034c..37c200b 100644 --- a/installability/tester.py +++ b/installability/tester.py @@ -18,6 +18,7 @@ from itertools import chain, filterfalse from britney_util import iter_except + class InstallabilityTester(object): def __init__(self, universe, revuniverse, testing, broken, essentials, @@ -53,6 +54,7 @@ class InstallabilityTester(object): self._revuniverse = revuniverse self._safe_set = safe_set self._eqv_table = eqv_table + self._stats = InstallabilityStats() # Cache of packages known to be broken - we deliberately do not # include "broken" in it. See _optimize for more info. @@ -98,13 +100,16 @@ class InstallabilityTester(object): testing -= eqv_set cbroken |= eqv_set + @property + def stats(self): + return self._stats def are_equivalent(self, p1, p2): """Test if p1 and p2 are equivalent Returns True if p1 and p2 have the same "signature" in the package dependency graph (i.e. relations can not tell - them appart sematically except for their name) + them apart semantically except for their name) """ eqv_table = self._eqv_table return p1 in eqv_table and p2 in eqv_table[p1] @@ -114,7 +119,7 @@ class InstallabilityTester(object): """Add a binary package to "testing" If the package is not known, this method will throw an - Keyrror. + KeyError. """ t = (pkg_name, pkg_version, pkg_arch) @@ -126,6 +131,8 @@ class InstallabilityTester(object): self._testing.add(t) elif t not in self._testing: self._testing.add(t) + if self._cache_inst: + self._stats.cache_drops += 1 self._cache_inst = set() if self._cache_broken: # Re-add broken packages as some of them may now be installable @@ -164,6 +171,7 @@ class InstallabilityTester(object): if t not in self._broken and t in self._cache_inst: # It is in our cache (and not guaranteed to be broken) - throw out the cache self._cache_inst = set() + self._stats.cache_drops += 1 return True @@ -177,17 +185,21 @@ class InstallabilityTester(object): Returns False otherwise. """ + self._stats.is_installable_calls += 1 t = (pkg_name, pkg_version, pkg_arch) if t not in self._universe: raise KeyError(str(t)) if t not in self._testing or t in self._broken: + self._stats.cache_hits += 1 return False if t in self._cache_inst: + self._stats.cache_hits += 1 return True + self._stats.cache_misses += 1 return self._check_inst(t) @@ -195,8 +207,9 @@ class InstallabilityTester(object): # See the explanation of musts, never and choices below. cache_inst = self._cache_inst + stats = self._stats - if t in cache_inst and not never: + if musts and t in cache_inst and not never: # use the inst cache only for direct queries/simple queries. cache = True if choices: @@ -214,7 +227,6 @@ class InstallabilityTester(object): if cache: return True - universe = self._universe testing = self._testing cbroken = self._cache_broken @@ -255,16 +267,16 @@ class InstallabilityTester(object): # set conflicts with t - either way, t is f***ed cbroken.add(t) testing.remove(t) + stats.conflicts_essential += 1 return False musts.update(start) never.update(ess_never) # curry check_loop check_loop = partial(self._check_loop, universe, testing, - eqv_table, musts, never, choices, + eqv_table, stats, musts, never, choices, cbroken) - # Useful things to remember: # # * musts and never are disjointed at all times @@ -278,7 +290,7 @@ class InstallabilityTester(object): # # * check never includes choices (these are always in choices) # - # * A package is installable if never and musts are disjoined + # * A package is installable if never and musts are disjointed # and both check and choices are empty. # - exception: _pick_choice may determine the installability # of t via recursion (calls _check_inst). In this case @@ -318,6 +330,7 @@ class InstallabilityTester(object): if first: musts.add(first) check.add(first) + stats.choice_resolved_using_safe_set += 1 continue # None of the safe set choices are installable, so drop them remain -= safe_set @@ -326,11 +339,13 @@ class InstallabilityTester(object): # the choice was reduced to one package we haven't checked - check that check.update(remain) musts.update(remain) + stats.choice_presolved += 1 continue if not remain: # all alternatives would violate the conflicts or are uninstallable # => package is not installable + stats.choice_presolved += 1 return None # The choice is still deferred @@ -347,7 +362,7 @@ class InstallabilityTester(object): choices_tmp = set() check_tmp = set([p]) if not self._check_loop(universe, testing, eqv_table, - musts_copy, never_tmp, + stats, musts_copy, never_tmp, choices_tmp, cbroken, check_tmp): # p cannot be chosen/is broken (unlikely, but ...) @@ -364,6 +379,7 @@ class InstallabilityTester(object): # routine, but to conserve stack-space, we return # and expect to be called again later. musts.update(musts_copy) + stats.choice_resolved_without_restore_point += 1 return False if not musts.isdisjoint(never_tmp): @@ -371,6 +387,7 @@ class InstallabilityTester(object): # t uninstallable, so p is a no-go. continue + stats.backtrace_restore_point_created += 1 # We are not sure that p is safe, setup a backtrack # point and recurse. never_tmp |= never @@ -387,6 +404,7 @@ class InstallabilityTester(object): # to satisfy the dependencies, so pretend to conflict # with it - hopefully it will reduce future choices. never.add(p) + stats.backtrace_restore_point_used += 1 # Optimization for the last case; avoid the recursive call # and just assume the last will lead to a solution. If it @@ -394,6 +412,7 @@ class InstallabilityTester(object): # have to back-track anyway. check.add(last) musts.add(last) + stats.backtrace_last_option += 1 return False # END _pick_choice @@ -418,11 +437,13 @@ class InstallabilityTester(object): if verdict: # if t is installable, then so are all packages in musts self._cache_inst.update(musts) + stats.solved_installable += 1 + else: + stats.solved_uninstallable += 1 return verdict - - def _check_loop(self, universe, testing, eqv_table, musts, never, + def _check_loop(self, universe, testing, eqv_table, stats, musts, never, choices, cbroken, check, len=len, frozenset=frozenset): """Finds all guaranteed dependencies via "check". @@ -493,13 +514,19 @@ class InstallabilityTester(object): # _build_eqv_packages_table method for more # information on how this works. new_cand = set(x for x in candidates if x not in possible_eqv) + stats.eqv_table_times_used += 1 + for chosen in iter_except(possible_eqv.pop, KeyError): new_cand.add(chosen) possible_eqv -= eqv_table[chosen] + stats.eqv_table_total_number_of_alternatives_eliminated += len(candidates) - len(new_cand) if len(new_cand) == 1: check.update(new_cand) musts.update(new_cand) + stats.eqv_table_reduced_to_one += 1 continue + elif len(candidates) == len(new_cand): + stats.eqv_table_reduced_by_zero += 1 candidates = frozenset(new_cand) # defer this choice till later choices.add(candidates) @@ -514,6 +541,7 @@ class InstallabilityTester(object): eqv_table = self._eqv_table cbroken = self._cache_broken universe = self._universe + stats = self._stats safe_set = self._safe_set ess_base = set(x for x in self._essentials if x[2] == arch and x in testing) @@ -523,7 +551,7 @@ class InstallabilityTester(object): not_satisified = partial(filter, start.isdisjoint) while ess_base: - self._check_loop(universe, testing, eqv_table, + self._check_loop(universe, testing, eqv_table, stats, start, ess_never, ess_choices, cbroken, ess_base) if ess_choices: @@ -575,6 +603,39 @@ class InstallabilityTester(object): return graph_stats +class InstallabilityStats(object): + + def __init__(self): + self.cache_hits = 0 + self.cache_misses = 0 + self.cache_drops = 0 + self.backtrace_restore_point_created = 0 + self.backtrace_restore_point_used = 0 + self.backtrace_last_option = 0 + self.choice_presolved = 0 + self.choice_resolved_using_safe_set = 0 + self.choice_resolved_without_restore_point = 0 + self.is_installable_calls = 0 + self.solved_installable = 0 + self.solved_uninstallable = 0 + self.conflicts_essential = 0 + self.eqv_table_times_used = 0 + self.eqv_table_reduced_to_one = 0 + self.eqv_table_reduced_by_zero = 0 + self.eqv_table_total_number_of_alternatives_eliminated = 0 + + def stats(self): + formats = [ + "Requests - is_installable: {is_installable_calls}", + "Cache - hits: {cache_hits}, misses: {cache_misses}, drops: {cache_drops}", + "Choices - pre-solved: {choice_presolved}, safe-set: {choice_resolved_using_safe_set}, No RP: {choice_resolved_without_restore_point}", + "Backtrace - RP created: {backtrace_restore_point_created}, RP used: {backtrace_restore_point_used}, reached last option: {backtrace_last_option}", + "Solved - installable: {solved_installable}, uninstallable: {solved_uninstallable}, conflicts essential: {conflicts_essential}", + "Eqv - times used: {eqv_table_times_used}, perfect reductions: {eqv_table_reduced_to_one}, failed reductions: {eqv_table_reduced_by_zero}, total no. of alternatives pruned: {eqv_table_total_number_of_alternatives_eliminated}", + ] + return [x.format(**self.__dict__) for x in formats] + + class ArchStats(object): def __init__(self): From f6c653b3f5324285584e908b14bb7a7c01622ef0 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 36/59] inst-tester: Fix bug "choices" would not be updated In some scenarios, it was possible to trigger a bug in the installability tester, where it would fail to update the "choices" set. The requirements for triggering this seems to be something like: * Obtain a choice that is possible to solve. * Resolve the choice without recursing (with a backtrack point) * Obtain a second choice that is impossible to solve. After the two first steps, the installability tester would fail to update the "choices" set (or, rather, changes would be invisible to the "_pick_choice" function). Fortunately, most packages are either trivially installable or trivially uninstallable, so the bug seems to be rather rare if triggred at all. Signed-off-by: Niels Thykier --- installability/tester.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/installability/tester.py b/installability/tester.py index 37c200b..e148c62 100644 --- a/installability/tester.py +++ b/installability/tester.py @@ -274,8 +274,7 @@ class InstallabilityTester(object): # curry check_loop check_loop = partial(self._check_loop, universe, testing, - eqv_table, stats, musts, never, choices, - cbroken) + eqv_table, stats, musts, never, cbroken) # Useful things to remember: # @@ -310,7 +309,7 @@ class InstallabilityTester(object): rebuild. """ - # We already satisfied/chosen at least one of the litterals + # We already satisfied/chosen at least one of the literals # in the choice, so the choice is gone for choice in filter(musts.isdisjoint, choices): # cbroken is needed here because (in theory) it could @@ -363,7 +362,7 @@ class InstallabilityTester(object): check_tmp = set([p]) if not self._check_loop(universe, testing, eqv_table, stats, musts_copy, never_tmp, - choices_tmp, cbroken, + cbroken, choices_tmp, check_tmp): # p cannot be chosen/is broken (unlikely, but ...) continue @@ -417,7 +416,7 @@ class InstallabilityTester(object): # END _pick_choice while check: - if not check_loop(check): + if not check_loop(choices, check): verdict = False break @@ -444,7 +443,7 @@ class InstallabilityTester(object): return verdict def _check_loop(self, universe, testing, eqv_table, stats, musts, never, - choices, cbroken, check, len=len, + cbroken, choices, check, len=len, frozenset=frozenset): """Finds all guaranteed dependencies via "check". @@ -475,7 +474,7 @@ class InstallabilityTester(object): # so "obviously" we can never choose any of its conflicts never.update(cons & testing) - # depgroup can be satisifed by picking something that is + # depgroup can be satisfied by picking something that is # already in musts - lets pick that (again). :) for depgroup in not_satisfied(deps): @@ -552,8 +551,8 @@ class InstallabilityTester(object): while ess_base: self._check_loop(universe, testing, eqv_table, stats, - start, ess_never, ess_choices, - cbroken, ess_base) + start, ess_never, cbroken, + ess_choices, ess_base) if ess_choices: # Try to break choices where possible nchoice = set() From 06df61849ecc914b7e5bb56842d7f7fdef4efc8e Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 37/59] britney: Check for mismatched packages between suites. britney assumes that a package build is uniquely described by its name, version and architecture. Particularly when constructing Packages files by hand for testing purposes this assumption can be violated, leading to confusing behaviour. This change makes britney look for such mismatches, and report if any are found. --- britney.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/britney.py b/britney.py index 63ded46..3a11e53 100755 --- a/britney.py +++ b/britney.py @@ -288,6 +288,9 @@ class Britney(object): # properly initialised, so insert two empty dicts # here. self.binaries['pu'][arch] = ({}, {}) + + self._check_mismatches(arch) + self._build_installability_tester(self.options.architectures) if not self.options.nuninst_cache: @@ -328,6 +331,40 @@ class Britney(object): self.excuses = [] self.dependencies = {} + def _check_mismatches(self, arch): + suites = [s for s in self.binaries if arch in self.binaries[s]] + + check_field_name = dict( (globals()[fn], fn) for fn in + ("SOURCE SOURCEVER ARCHITECTURE MULTIARCH" + + " DEPENDS CONFLICTS PROVIDES ESSENTIAL").split() ) + check_fields = check_field_name.keys() + + any_mismatch = False + for s1, s2 in product(suites, suites): + if s1 >= s2: continue + s1_pkgs = self.binaries[s1][arch][0] + s2_pkgs = self.binaries[s2][arch][0] + pkgs = set(s1_pkgs) & set(s2_pkgs) + for p in pkgs: + if s1_pkgs[p][VERSION] != s2_pkgs[p][VERSION]: continue + bad = [] + for f in check_fields: + if s1_pkgs[p][f] != s2_pkgs[p][f]: + bad.append((f, s1_pkgs[p][f], s2_pkgs[p][f])) + + if bad: + any_mismatch = True + self.__log("Mismatch found %s %s %s differs in %s vs %s" % ( + p, s1_pkgs[p][VERSION], arch, s1, s2), type="E") + for f, v1, v2 in bad: + self.__log(" ... %s %s != %s" % (check_field_name[f], v1, v2)) + + # test suite doesn't appreciate aborts of this nature + #if any_mismatch: + # self.__log("Mismatches found, exiting.", type="I") + # sys.exit(1) + return + def __parse_arguments(self): """Parse the command line arguments From ed91345c061ed926cc1d04ce94c20f1c2689d370 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 38/59] britney.py: Skip ESSENTIAL consistency check The live-data tests rely on an inconsistency, since they were before Britney started to record the Essential field. Signed-off-by: Niels Thykier --- britney.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 3a11e53..170c194 100755 --- a/britney.py +++ b/britney.py @@ -334,9 +334,13 @@ class Britney(object): def _check_mismatches(self, arch): suites = [s for s in self.binaries if arch in self.binaries[s]] + # NB: ESSENTIAL deliberately skipped as the 2011 and 2012 + # parts of the live-data tests requires it (britney merges + # this field correctly from the unstable version where + # available) check_field_name = dict( (globals()[fn], fn) for fn in ("SOURCE SOURCEVER ARCHITECTURE MULTIARCH" - + " DEPENDS CONFLICTS PROVIDES ESSENTIAL").split() ) + + " DEPENDS CONFLICTS PROVIDES").split() ) check_fields = check_field_name.keys() any_mismatch = False From 98070388e77db014b031cc293fc098f555dbb253 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 39/59] britney.py: Enable the new consistency checks by default Signed-off-by: Niels Thykier --- britney.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/britney.py b/britney.py index 170c194..1a8e7a9 100755 --- a/britney.py +++ b/britney.py @@ -364,9 +364,9 @@ class Britney(object): self.__log(" ... %s %s != %s" % (check_field_name[f], v1, v2)) # test suite doesn't appreciate aborts of this nature - #if any_mismatch: - # self.__log("Mismatches found, exiting.", type="I") - # sys.exit(1) + if any_mismatch: + self.__log("Mismatches found, exiting.", type="I") + sys.exit(1) return def __parse_arguments(self): From efe5c82f2289e280d7ebe95152548f819039424a Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Sun, 26 Apr 2015 18:20:11 +0200 Subject: [PATCH 40/59] britney: Take more care with hijacked binaries This updates the doop_source and _compute_groups functions so that binary packages that are built from a different source aren't included as part of an update to the original source. In the event that it's a binary-only update, also don't remove the hijacked packages from testing. This change also removes an obsolete comment regardarding pre-conditions for the _compute_groups function. --- britney.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/britney.py b/britney.py index 1a8e7a9..d45288c 100755 --- a/britney.py +++ b/britney.py @@ -1866,11 +1866,6 @@ class Britney(object): InstallabilityTester. - Pre-Conditions: The source package must be in testing and this - should only be used when considering to do an upgrade - migration from the input suite. (e.g. do not use this for - removals). - Unlike doop_source, this will not modify any data structure. """ # local copies for better performances @@ -1971,7 +1966,7 @@ class Britney(object): suite != 'unstable' and \ binaries_t[parch][0][binary][ARCHITECTURE] == 'all': continue - else: + else: rms.add((binary, version, parch)) # single binary removal; used for clearing up after smooth @@ -1988,11 +1983,24 @@ class Britney(object): if migration_architecture not in ['source', parch]: continue version = self.binaries[suite][parch][0][binary][VERSION] + + if (not include_hijacked + and self.binaries[suite][parch][0][binary][SOURCE] != source_name): + # This binary package has been hijacked by some other source. + # So don't add it as part of this update. + # + # Also, if this isn't a source update, don't remove + # the package that's been hijacked if it's present. + if migration_architecture != 'source': + for rm_b, rm_v, rm_p in list(rms): + if (rm_b, rm_p) == (binary, parch): + rms.remove((rm_b, rm_v, rm_p)) + continue + adds.add((binary, version, parch)) return (adds, rms, set(smoothbins.values())) - def doop_source(self, item, hint_undo=None, removals=frozenset()): """Apply a change to the testing distribution as requested by `pkg` @@ -2023,16 +2031,19 @@ class Britney(object): inst_tester = self._inst_tester eqv_set = set() + updates, rms, _ = self._compute_groups(item.package, + item.suite, + item.architecture, + item.is_removal, + removals=removals) + #print("+++ %s" % (sorted(updates))) + #print("--- %s" % (sorted(rms))) + # remove all binary packages (if the source already exists) if item.architecture == 'source' or not item.is_removal: if item.package in sources['testing']: source = sources['testing'][item.package] - updates, rms, _ = self._compute_groups(item.package, - item.suite, - item.architecture, - item.is_removal, - removals=removals) eqv_table = {} @@ -2098,9 +2109,9 @@ class Britney(object): if not item.is_removal: source = sources[item.suite][item.package] packages_s = self.binaries[item.suite] - for p in source[BINARIES]: - binary, parch = p.split("/") - if item.architecture not in ['source', parch]: continue + + for binary, version, parch in updates: + p = "%s/%s" % (binary, parch) key = (binary, parch) binaries_t_a, provides_t_a = packages_t[parch] equivalent_replacement = key in eqv_set From 7a53010adc4ebda79e155eaa3c24aa106852e58b Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 26 Apr 2015 19:45:28 +0200 Subject: [PATCH 41/59] britney.conf: Renable smooth updates for libs+oldlibs --- britney.conf | 2 +- britney_nobreakall.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/britney.conf b/britney.conf index fc60879..39e2ca8 100644 --- a/britney.conf +++ b/britney.conf @@ -58,4 +58,4 @@ HINTS_AUTO-REMOVALS = remove # # naming a non-existent section will effectively disable new smooth # updates but still allow removals to occur -SMOOTH_UPDATES = badgers +SMOOTH_UPDATES = libs oldlibs diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index 99e9d21..6939d44 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -58,4 +58,4 @@ HINTS_AUTO-REMOVALS = remove # # naming a non-existent section will effectively disable new smooth # updates but still allow removals to occur -SMOOTH_UPDATES = badgers +SMOOTH_UPDATES = libs oldlibs From d8fa67cee89098de4b8a54be59418428868a4103 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Mon, 27 Apr 2015 18:50:48 +0000 Subject: [PATCH 42/59] britney.py: drop inaccurate comment Signed-off-by: Adam D. Barratt --- britney.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/britney.py b/britney.py index d45288c..5535241 100755 --- a/britney.py +++ b/britney.py @@ -1768,8 +1768,7 @@ class Britney(object): skip_archall = True else: skip_archall = False - # check all the packages for this architecture, calling add_nuninst if a new - # uninstallable package is found + # check all the packages for this architecture nuninst[arch] = set() for pkg_name in binaries[arch][0]: pkgdata = binaries[arch][0][pkg_name] From 18885d3f3be083fd1d95ff1d40057bfe2a1f3a83 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Mon, 27 Apr 2015 19:14:08 +0000 Subject: [PATCH 43/59] britney.py: typo fixes Signed-off-by: Adam D. Barratt --- britney.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/britney.py b/britney.py index 5535241..9bd4a60 100755 --- a/britney.py +++ b/britney.py @@ -284,7 +284,7 @@ class Britney(object): if hasattr(self.options, 'pu'): self.binaries['pu'][arch] = self.read_binaries(self.options.pu, "pu", arch) else: - # _build_installability_tester relies it being + # _build_installability_tester relies on it being # properly initialised, so insert two empty dicts # here. self.binaries['pu'][arch] = ({}, {}) @@ -335,7 +335,7 @@ class Britney(object): suites = [s for s in self.binaries if arch in self.binaries[s]] # NB: ESSENTIAL deliberately skipped as the 2011 and 2012 - # parts of the live-data tests requires it (britney merges + # parts of the live-data tests require it (britney merges # this field correctly from the unstable version where # available) check_field_name = dict( (globals()[fn], fn) for fn in @@ -403,7 +403,7 @@ class Britney(object): if self.options.nuninst_cache and self.options.print_uninst: self.__log("nuninst_cache and print_uninst are mutually exclusive!", type="E") sys.exit(1) - # if the configuration file exists, than read it and set the additional options + # if the configuration file exists, then read it and set the additional options elif not os.path.isfile(self.options.config): self.__log("Unable to read the configuration file (%s), exiting!" % self.options.config, type="E") sys.exit(1) @@ -474,7 +474,7 @@ class Britney(object): conflicts = [] possible_dep_ranges = {} - # We do not differ between depends and pre-depends + # We do not differentiate between depends and pre-depends if pkgdata[DEPENDS]: depends.extend(apt_pkg.parse_depends(pkgdata[DEPENDS], False)) @@ -493,7 +493,7 @@ class Britney(object): pkgs = solvers(block, arch, dep_dist) for p in pkgs: # version and arch is already interned, but solvers use - # the package name extracted from the field and is therefore + # the package name extracted from the field and it is therefore # not interned. pdata = binaries[dep_dist][arch][0][p] pt = (sys.intern(p), pdata[VERSION], arch) @@ -634,7 +634,7 @@ class Britney(object): # Merge Pre-Depends with Depends and Conflicts with # Breaks. Britney is not interested in the "finer - # semantical differences" of these fields anyway. + # semantic differences" of these fields anyway. pdeps = get_field('Pre-Depends') deps = get_field('Depends') if deps and pdeps: @@ -897,7 +897,7 @@ class Britney(object): if l[0] == 'finished': break if l[0] == 'remark': - # Ignore "no-op" hint, which sole purpose is to be + # Ignore "no-op" hint, the sole purpose of which is to be # found by hint grep (and show up in "d"'s # output). continue @@ -966,7 +966,7 @@ class Britney(object): packages = [] - # local copies for better performances + # local copies for better performance binaries = self.binaries[distribution][arch] # for every package, version and operation in the block @@ -1071,7 +1071,7 @@ class Britney(object): # if the source package is available in unstable, then do nothing if pkg in self.sources['unstable']: return False - # otherwise, add a new excuse for its removal and return True + # otherwise, add a new excuse for its removal src = self.sources['testing'][pkg] excuse = Excuse("-" + pkg) excuse.addhtml("Package not in unstable, will try to remove") @@ -1425,7 +1425,7 @@ class Britney(object): pkgsv = binary_u[SOURCEVER] # if it wasn't built by the same source, it is out-of-date - # it there is at least one binary which is up-to-date, there + # if there is at least one binary which is up-to-date, there # is a build on this arch if not same_source(source_u[VERSION], pkgsv): if pkgsv not in oodbins: @@ -2023,7 +2023,7 @@ class Britney(object): affected = set() - # local copies for better performances + # local copies for better performance sources = self.sources packages_t = self.binaries['testing'] get_reverse_tree = partial(compute_reverse_tree, packages_t) From 1d98bf293d256072b22be9f0d2ab2f59b596a86f Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Mon, 27 Apr 2015 19:43:54 +0000 Subject: [PATCH 44/59] britney.py: reverse sense of tests for architecture-indep packages It's more natural to say "check this package if the current arch is in this list" than "do not check this package if the current arch is not in this list" Signed-off-by: Adam D. Barratt --- britney.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/britney.py b/britney.py index 9bd4a60..5c2171e 100755 --- a/britney.py +++ b/britney.py @@ -1764,9 +1764,7 @@ class Britney(object): for arch in self.options.architectures: if requested_arch and arch != requested_arch: continue # if it is in the nobreakall ones, check arch-independent packages too - if arch not in self.options.nobreakall_arches.split(): - skip_archall = True - else: skip_archall = False + check_archall = arch in self.options.nobreakall_arches.split() # check all the packages for this architecture nuninst[arch] = set() @@ -1778,7 +1776,7 @@ class Britney(object): # if they are not required, remove architecture-independent packages nuninst[arch + "+all"] = nuninst[arch].copy() - if skip_archall: + if not check_archall: for pkg in nuninst[arch + "+all"]: bpkg = binaries[arch][0][pkg] if bpkg[ARCHITECTURE] == 'all': @@ -2188,7 +2186,7 @@ class Britney(object): return (affected, undo) - def _check_packages(self, binaries, arch, affected, skip_archall, nuninst): + def _check_packages(self, binaries, arch, affected, check_archall, nuninst): broken = nuninst[arch + "+all"] to_check = [] @@ -2200,7 +2198,8 @@ class Britney(object): version = pkgdata[VERSION] parch = pkgdata[ARCHITECTURE] nuninst_arch = None - if not (skip_archall and parch == 'all'): + # only check arch:all packages if requested + if check_archall or parch != 'all': nuninst_arch = nuninst[arch] self._installability_test(p, version, arch, broken, to_check, nuninst_arch) @@ -2215,7 +2214,8 @@ class Britney(object): version = pkgdata[VERSION] parch = pkgdata[ARCHITECTURE] nuninst_arch = None - if not (skip_archall and parch == 'all'): + # only check arch:all packages if requested + if check_archall or parch != 'all': nuninst_arch = nuninst[arch] self._installability_test(p, version, arch, broken, to_check, nuninst_arch) @@ -2264,12 +2264,9 @@ class Britney(object): nuninst[arch + '+all'] = set(x for x in nuninst_arch_all if x in binaries_t_a) for arch in self.options.architectures: - if arch not in nobreakall_arches: - skip_archall = True - else: - skip_archall = False + check_archall = arch in nobreakall_arches - check_packages(arch, all_affected, skip_archall, nuninst) + check_packages(arch, all_affected, check_archall, nuninst) return nuninst @@ -2337,15 +2334,12 @@ class Britney(object): # check the affected packages on all the architectures for arch in (item.architecture == 'source' and architectures or (item.architecture,)): - if arch not in nobreakall_arches: - skip_archall = True - else: - skip_archall = False + check_archall = arch in nobreakall_arches nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) - check_packages(arch, affected, skip_archall, nuninst) + check_packages(arch, affected, check_archall, nuninst) # if the uninstallability counter is worse than before, break the loop if ((item.architecture != 'source' and arch not in new_arches) or \ From 06a45f2ed4155797049ef58ed1b2a9a31a5d96ae Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:29:47 +0200 Subject: [PATCH 45/59] britney_util: Typo fix Signed-off-by: Niels Thykier --- britney_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney_util.py b/britney_util.py index fbe39f5..1043ad3 100644 --- a/britney_util.py +++ b/britney_util.py @@ -482,7 +482,7 @@ def write_sources(sources_s, filename): """Write a sources file from Britney's state for a given suite Britney discards fields she does not care about, so the resulting - file omitts a lot of regular fields. + file omits a lot of regular fields. """ key_pairs = ((VERSION, 'Version'), (SECTION, 'Section'), From 375b468512c2ddbcb25f5338ee8e6a74dafe5b74 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:29:47 +0200 Subject: [PATCH 46/59] Remove unused local variable Signed-off-by: Niels Thykier --- britney.py | 1 - 1 file changed, 1 deletion(-) diff --git a/britney.py b/britney.py index 5c2171e..58e8eef 100755 --- a/britney.py +++ b/britney.py @@ -988,7 +988,6 @@ class Britney(object): # look for the package in the virtual packages list and loop on them for prov in binaries[1].get(name, []): if prov not in binaries[0]: continue - package = binaries[0][prov] # A provides only satisfies: # - an unversioned dependency (per Policy Manual ยง7.5) # - a dependency without an architecture qualifier From 35e5742d853c9ea0461a7ed932adedccf82c1761 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:51 +0200 Subject: [PATCH 47/59] Remove "leading" from hints - it provides no new information --- britney.py | 1 - 1 file changed, 1 deletion(-) diff --git a/britney.py b/britney.py index 58e8eef..9598893 100755 --- a/britney.py +++ b/britney.py @@ -2417,7 +2417,6 @@ class Britney(object): if init: if not force: lundo = [] - self.output_write("leading: %s\n" % (",".join( x.uvname for x in init ))) for x in init: if x not in upgrade_me: self.output_write("failed: %s\n" % (x.uvname)) From bdcf4e826b8dbcdf5eaf8c131410ca97e0ce0575 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:51 +0200 Subject: [PATCH 48/59] Be verbose when rejecting a hint due to invalid candidates Signed-off-by: Niels Thykier --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 9598893..a9027eb 100755 --- a/britney.py +++ b/britney.py @@ -2419,7 +2419,7 @@ class Britney(object): lundo = [] for x in init: if x not in upgrade_me: - self.output_write("failed: %s\n" % (x.uvname)) + self.output_write("failed: %s is not a valid candidate (or it already migrated)\n" % (x.uvname)) return None selected.append(x) upgrade_me.remove(x) From d7baa1d7f0d7078e9f4feecd3e56c8605541bf3b Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:51 +0200 Subject: [PATCH 49/59] britney.py: Remove redundant arg to format Signed-off-by: Niels Thykier --- britney.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/britney.py b/britney.py index a9027eb..61178b9 100755 --- a/britney.py +++ b/britney.py @@ -2299,7 +2299,7 @@ class Britney(object): dependencies = self.dependencies check_packages = partial(self._check_packages, binaries) - self.output_write("recur: [%s] %s %d/%d\n" % ("", ",".join(x.uvname for x in selected), len(packages), len(extra))) + self.output_write("recur: [] %s %d/%d\n" % (",".join(x.uvname for x in selected), len(packages), len(extra))) # loop on the packages (or better, actions) while packages: From e4c7c4f2a2446fd9b5957bbb4c713593bf83f381 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 50/59] britney.py: Minor optimisation to sort_actions Avoid some cases of O(n^2) behaviour in sort_actions and reduce the size of n for the remaining O(n^2)-ish behaviour by filtering out removals early on. Signed-off-by: Niels Thykier --- britney.py | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/britney.py b/britney.py index 61178b9..eb47d46 100755 --- a/britney.py +++ b/britney.py @@ -2745,29 +2745,39 @@ class Britney(object): so the ones with most reverse dependencies are at the end of the loop. If an action depends on another one, it is put after it. """ - upgrade_me = [x.name for x in self.excuses if x.name in [y.uvname for y in self.upgrade_me]] - for e in self.excuses: - if e.name not in upgrade_me: continue - # try removes at the end of the loop - elif e.name[0] == '-': - upgrade_me.remove(e.name) - upgrade_me.append(e.name) - # otherwise, put it in a good position checking its dependencies + uvnames = frozenset(y.uvname for y in self.upgrade_me) + excuses = [e for e in self.excuses if e.name in uvnames] + removals = [] + upgrade_me = [] + + for e in excuses: + # We order removals and regular migrations differently, so + # split them out early. + if e.name[0] == '-': + removals.append(e.name) else: - pos = [] - udeps = [upgrade_me.index(x) for x in e.deps if x in upgrade_me and x != e.name] - if len(udeps) > 0: - pos.append(max(udeps)) - sdeps = [upgrade_me.index(x) for x in e.sane_deps if x in upgrade_me and x != e.name] - if len(sdeps) > 0: - pos.append(min(sdeps)) - if len(pos) == 0: continue - upgrade_me.remove(e.name) - upgrade_me.insert(max(pos)+1, e.name) - self.dependencies[e.name] = e.deps + upgrade_me.append(e.name) + + for e in excuses: + # put the item (regular migration) in a good position + # checking its dependencies + pos = [] + udeps = [upgrade_me.index(x) for x in e.deps if x in upgrade_me and x != e.name] + if udeps: + pos.append(max(udeps)) + sdeps = [upgrade_me.index(x) for x in e.sane_deps if x in upgrade_me and x != e.name] + if sdeps: + pos.append(min(sdeps)) + if not pos: + continue + upgrade_me.remove(e.name) + upgrade_me.insert(max(pos)+1, e.name) + self.dependencies[e.name] = e.deps # replace the list of actions with the new one self.upgrade_me = [ make_migrationitem(x, self.sources) for x in upgrade_me ] + self.upgrade_me.extend(make_migrationitem(x, self.sources) for x in removals) + def auto_hinter(self): """Auto-generate "easy" hints. From e45b219276279238e9b4fb3f76d38e640a6c9d7c Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 51/59] britney.py: Pre-split self.options.*_arches We always use them as lists, so we might as well just split them once and be done with it. --- britney.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/britney.py b/britney.py index eb47d46..1a07f50 100755 --- a/britney.py +++ b/britney.py @@ -430,16 +430,22 @@ class Britney(object): if not hasattr(self.options, "heidi_delta_output"): self.options.heidi_delta_output = self.options.heidi_output + "Delta" + self.options.nobreakall_arches = self.options.nobreakall_arches.split() + self.options.fucked_arches = self.options.fucked_arches.split() + self.options.break_arches = self.options.break_arches.split() + self.options.new_arches = self.options.new_arches.split() + # Sort the architecture list allarches = sorted(self.options.architectures.split()) - arches = [x for x in allarches if x in self.options.nobreakall_arches.split()] - arches += [x for x in allarches if x not in arches and x not in self.options.fucked_arches.split()] - arches += [x for x in allarches if x not in arches and x not in self.options.break_arches.split()] - arches += [x for x in allarches if x not in arches and x not in self.options.new_arches.split()] + arches = [x for x in allarches if x in self.options.nobreakall_arches] + arches += [x for x in allarches if x not in arches and x not in self.options.fucked_arches] + arches += [x for x in allarches if x not in arches and x not in self.options.break_arches] + arches += [x for x in allarches if x not in arches and x not in self.options.new_arches] arches += [x for x in allarches if x not in arches] self.options.architectures = [sys.intern(arch) for arch in arches] self.options.smooth_updates = self.options.smooth_updates.split() + def __log(self, msg, type="I"): """Print info messages according to verbosity level @@ -1045,7 +1051,7 @@ class Britney(object): # for the solving packages, update the excuse to add the dependencies for p in packages: - if arch not in self.options.break_arches.split(): + if arch not in self.options.break_arches: if p in self.sources['testing'] and self.sources['testing'][p][VERSION] == self.sources[suite][p][VERSION]: excuse.add_dep("%s/%s" % (p, arch), arch) else: @@ -1397,7 +1403,7 @@ class Britney(object): base = 'stable' text = "Not yet built on %s (relative to testing)" % (quote(arch), quote(src), quote(source_u[VERSION]), base, arch) - if arch in self.options.fucked_arches.split(): + if arch in self.options.fucked_arches: text = text + " (but %s isn't keeping up, so never mind)" % (arch) else: update_candidate = False @@ -1436,7 +1442,7 @@ class Britney(object): # if the package is architecture-dependent or the current arch is `nobreakall' # find unsatisfied dependencies for the binary package - if binary_u[ARCHITECTURE] != 'all' or arch in self.options.nobreakall_arches.split(): + if binary_u[ARCHITECTURE] != 'all' or arch in self.options.nobreakall_arches: self.excuse_unsat_deps(pkg, src, arch, suite, excuse) # if there are out-of-date packages, warn about them in the excuse and set update_candidate @@ -1458,7 +1464,7 @@ class Britney(object): "arch=%s&pkg=%s&ver=%s\" target=\"_blank\">%s: %s" % \ (quote(arch), quote(src), quote(source_u[VERSION]), arch, oodtxt) - if arch in self.options.fucked_arches.split(): + if arch in self.options.fucked_arches: text = text + " (but %s isn't keeping up, so nevermind)" % (arch) else: update_candidate = False @@ -1763,7 +1769,7 @@ class Britney(object): for arch in self.options.architectures: if requested_arch and arch != requested_arch: continue # if it is in the nobreakall ones, check arch-independent packages too - check_archall = arch in self.options.nobreakall_arches.split() + check_archall = arch in self.options.nobreakall_arches # check all the packages for this architecture nuninst[arch] = set() @@ -1807,7 +1813,7 @@ class Britney(object): elif original and arch in original: n = len(original[arch]) else: continue - if arch in self.options.break_arches.split(): + if arch in self.options.break_arches: totalbreak = totalbreak + n else: total = total + n @@ -2231,7 +2237,7 @@ class Britney(object): removals = set() all_affected = set() - nobreakall_arches = self.options.nobreakall_arches.split() + nobreakall_arches = self.options.nobreakall_arches packages_t = self.binaries['testing'] check_packages = partial(self._check_packages, packages_t) nuninst = {} @@ -2293,9 +2299,9 @@ class Britney(object): binaries = self.binaries['testing'] sources = self.sources architectures = self.options.architectures - nobreakall_arches = self.options.nobreakall_arches.split() - new_arches = self.options.new_arches.split() - break_arches = self.options.break_arches.split() + nobreakall_arches = self.options.nobreakall_arches + new_arches = self.options.new_arches + break_arches = self.options.break_arches dependencies = self.dependencies check_packages = partial(self._check_packages, binaries) @@ -2454,7 +2460,7 @@ class Britney(object): newly_uninst(nuninst_start, nuninst_end)) + "\n") if not force: - break_arches = set(self.options.break_arches.split()) + break_arches = set(self.options.break_arches) if all(x.architecture in break_arches for x in selected): # If we only migrated items from break-arches, then we # do not allow any regressions on these architectures. @@ -2528,16 +2534,16 @@ class Britney(object): allpackages = [] normpackages = self.upgrade_me[:] archpackages = {} - for a in self.options.break_arches.split(): + for a in self.options.break_arches: archpackages[a] = [p for p in normpackages if p.architecture == a] normpackages = [p for p in normpackages if p not in archpackages[a]] self.upgrade_me = normpackages self.output_write("info: main run\n") self.do_all() allpackages += self.upgrade_me - for a in self.options.break_arches.split(): + for a in self.options.break_arches: backup = self.options.break_arches - self.options.break_arches = " ".join(x for x in self.options.break_arches.split() if x != a) + self.options.break_arches = " ".join(x for x in self.options.break_arches if x != a) self.upgrade_me = archpackages[a] self.output_write("info: broken arch run for %s\n" % (a)) self.do_all() From 754688851a5bf6a61065016d33dd9d0374ea1d72 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 52/59] britney: Optimise original auto-hinter duplication handling Britney is now smart enough to produce the same result from hints regardless of the order of the items in the hint. With this in mind, we can have the original auto-hinter produce hints as sets and filter out duplicates as we produce them. Note that the hints are sorted to produce deterministic output (to make it easier to compare the hints between runs and changes). Signed-off-by: Niels Thykier --- britney.py | 51 +++++++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/britney.py b/britney.py index 1a07f50..9ad4333 100755 --- a/britney.py +++ b/britney.py @@ -2849,16 +2849,21 @@ class Britney(object): # loop on them candidates = [] mincands = [] + seen_hints = set() for e in excuses: excuse = excuses[e] if e in sources_t and sources_t[e][VERSION] == excuse.ver[1]: continue if excuse.deps: hint = find_related(e, {}, True) - if isinstance(hint, dict) and e in hint and hint not in candidates: - candidates.append(hint.items()) + if isinstance(hint, dict) and e in hint: + h = frozenset(hint.items()) + if h not in seen_hints: + candidates.append(h) + seen_hints.add(h) else: items = [ (e, excuse.ver[1]) ] + orig_size = 1 looped = False for item, ver in items: # excuses which depend on "item" or are depended on by it @@ -2866,39 +2871,21 @@ class Britney(object): (item in excuses[x].deps or x in excuses[item].deps) \ and (x, excuses[x].ver[1]) not in items ) if not looped and len(items) > 1: - mincands.append(items[:]) + orig_size = len(items) + h = frozenset(items) + if h not in seen_hints: + mincands.append(h) + seen_hints.add(h) looped = True - if (len(items) > 1 and len(items) != len(mincands[-1]) and - frozenset(items) != frozenset(mincands[-1])): - candidates.append(items) + if len(items) != orig_size: + h = frozenset(items) + if h != mincands[-1] and h not in seen_hints: + candidates.append(h) + seen_hints.add(h) for l in [ candidates, mincands ]: - to_skip = set() - for i in range(len(l)): - if i in to_skip: - continue - l_i = None - for j in range(i+1, len(l)): - if j in to_skip: - # we already know this list isn't interesting - continue - if l_i is None: - l_i = frozenset(l[i]) - l_j = frozenset(l[j]) - if l_i >= l_j: - # j is a subset of i; ignore it - to_skip.add(j) - elif l_i < l_j: - # i is a subset of j; ignore it and the rest of the - # "i" series. - # NB: We use < and not <= because the "==" case is - # already covered above - to_skip.add(i) - break - for i in range(len(l)): - if i not in to_skip: - self.do_hint("easy", "autohinter", [ MigrationItem("%s/%s" % (x[0], x[1])) for x in l[i] ]) - + for hint in l: + self.do_hint("easy", "autohinter", [ MigrationItem("%s/%s" % (x[0], x[1])) for x in sorted(hint) ]) def nuninst_arch_report(self, nuninst, arch): """Print a report of uninstallable packages for one architecture.""" From cc58511c0989009094f54b8d6c2b7b1e6672b787 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 53/59] Avoid O(n^2) duplication handling when building hints Use a set to filter out seen items to avoid doing O(n^2) de-duplication. For very large hints, this can take considerable time. Using "seen_items" to build the actual hints on the (unverified) assumption that Python can do something "smart" to turn a set into a frozenset faster than it can with a list. Signed-off-by: Niels Thykier --- britney.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/britney.py b/britney.py index 9ad4333..d36a19b 100755 --- a/britney.py +++ b/britney.py @@ -2862,23 +2862,29 @@ class Britney(object): candidates.append(h) seen_hints.add(h) else: - items = [ (e, excuse.ver[1]) ] + items = [(e, excuse.ver[1])] orig_size = 1 looped = False + seen_items = set() + seen_items.update(items) + for item, ver in items: # excuses which depend on "item" or are depended on by it - items.extend( (x, excuses[x].ver[1]) for x in excuses if \ + new_items = [(x, excuses[x].ver[1]) for x in excuses if \ (item in excuses[x].deps or x in excuses[item].deps) \ - and (x, excuses[x].ver[1]) not in items ) + and (x, excuses[x].ver[1]) not in seen_items] + items.extend(new_items) + seen_items.update(new_items) + if not looped and len(items) > 1: orig_size = len(items) - h = frozenset(items) + h = frozenset(seen_items) if h not in seen_hints: mincands.append(h) seen_hints.add(h) looped = True if len(items) != orig_size: - h = frozenset(items) + h = frozenset(seen_items) if h != mincands[-1] and h not in seen_hints: candidates.append(h) seen_hints.add(h) From ffcfa8e27e3ddd0a83db39afa6f5fb550bc7fbe5 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 54/59] britney.py: Avoid some O(n) look-ups in the auto-hinter Signed-off-by: Niels Thykier --- britney.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/britney.py b/britney.py index d36a19b..d4625b8 100755 --- a/britney.py +++ b/britney.py @@ -2805,7 +2805,9 @@ class Britney(object): type="I") # consider only excuses which are valid candidates - excuses = dict((x.name, x) for x in self.excuses if x.name in [y.uvname for y in self.upgrade_me]) + valid_excuses = frozenset(y.uvname for y in self.upgrade_me) + excuses = dict((x.name, x) for x in self.excuses if x.name in valid_excuses) + excuses_deps = dict((name, set(excuse.deps)) for name, excuse in excuses.items()) sources_t = self.sources['testing'] groups = set() @@ -2871,7 +2873,7 @@ class Britney(object): for item, ver in items: # excuses which depend on "item" or are depended on by it new_items = [(x, excuses[x].ver[1]) for x in excuses if \ - (item in excuses[x].deps or x in excuses[item].deps) \ + (item in excuses_deps[x] or x in excuses_deps[item]) \ and (x, excuses[x].ver[1]) not in seen_items] items.extend(new_items) seen_items.update(new_items) From 4d218df7d0f5a36ededa82123c115d434a821d6f Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 55/59] iter_pkg: Refactor nuninst cloning Move nuninst cloning out of the check loop and always populate the new nuninst entirely. This will allow some simplifications in other places. Signed-off-by: Niels Thykier --- britney.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/britney.py b/britney.py index d4625b8..5405f46 100755 --- a/britney.py +++ b/britney.py @@ -2291,9 +2291,9 @@ class Britney(object): position = len(packages) if nuninst: - nuninst_comp = nuninst.copy() + nuninst_comp = nuninst else: - nuninst_comp = self.nuninst_orig.copy() + nuninst_comp = self.nuninst_orig # local copies for better performance binaries = self.binaries['testing'] @@ -2332,18 +2332,36 @@ class Britney(object): self.output_write("trying: %s\n" % (item.uvname)) better = True - nuninst = {} # apply the changes affected, undo = self.doop_source(item, lundo) + # Copy nuninst_comp - we have to deep clone affected + # architectures. + + # NB: We do this *after* updating testing and we have to filter out + # removed binaries. Otherwise, uninstallable binaries that were + # removed by the item would still be counted. + if item.architecture == 'source': + # Assume that all architectures are affected and deep + # copy nuninst_comp + nuninst = {} + for arch in architectures: + nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) + nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) + else: + # Shallow clone nuninst_comp except for the affected + # architecture, which is deep cloned. + arch = item.architecture + nuninst = nuninst_comp.copy() + nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) + nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) + + # check the affected packages on all the architectures for arch in (item.architecture == 'source' and architectures or (item.architecture,)): check_archall = arch in nobreakall_arches - nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) - nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) - check_packages(arch, affected, check_archall, nuninst) # if the uninstallability counter is worse than before, break the loop @@ -2368,8 +2386,7 @@ class Britney(object): self.output_write(" all: %s\n" % (" ".join( x.uvname for x in selected ))) else: self.output_write(" most: (%d) .. %s\n" % (len(selected), " ".join(x.uvname for x in selected[-20:]))) - for k in nuninst: - nuninst_comp[k] = nuninst[k] + nuninst_comp = nuninst else: self.output_write("skipped: %s (%d <- %d)\n" % (item.uvname, len(extra), len(packages))) self.output_write(" got: %s\n" % (self.eval_nuninst(nuninst, item.architecture != 'source' and nuninst_comp or None))) From 8f27919cf87d1ed5d12b8c1130b06a469f47be7d Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 2 Jun 2015 21:33:52 +0200 Subject: [PATCH 56/59] Create a clone_nuninst function for iter_packages{,_hint} Signed-off-by: Niels Thykier --- britney.py | 33 ++++++++++----------------------- britney_util.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/britney.py b/britney.py index 5405f46..3b2651d 100755 --- a/britney.py +++ b/britney.py @@ -204,7 +204,8 @@ from britney_util import (old_libraries_format, same_source, undo_changes, read_nuninst, write_nuninst, write_heidi, eval_uninst, newly_uninst, make_migrationitem, write_excuses, write_heidi_delta, write_controlfiles, - old_libraries, is_nuninst_asgood_generous) + old_libraries, is_nuninst_asgood_generous, + clone_nuninst) from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC, SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS, PROVIDES, RDEPENDS, RCONFLICTS, MULTIARCH, ESSENTIAL) @@ -2240,7 +2241,6 @@ class Britney(object): nobreakall_arches = self.options.nobreakall_arches packages_t = self.binaries['testing'] check_packages = partial(self._check_packages, packages_t) - nuninst = {} for item in hinted_packages: @@ -2258,15 +2258,11 @@ class Britney(object): lundo.append((undo,item)) # deep copy nuninst (in case the hint is undone) - # NB: We do this *after* updating testing and we have to filter out + # NB: We do this *after* updating testing as we have to filter out # removed binaries. Otherwise, uninstallable binaries that were # removed by the hint would still be counted. - for arch in self.options.architectures: - nuninst_arch = self.nuninst_orig[arch] - nuninst_arch_all = self.nuninst_orig[arch + '+all'] - binaries_t_a = packages_t[arch][0] - nuninst[arch] = set(x for x in nuninst_arch if x in binaries_t_a) - nuninst[arch + '+all'] = set(x for x in nuninst_arch_all if x in binaries_t_a) + nuninst = clone_nuninst(self.nuninst_orig, packages_t, + self.options.architectures) for arch in self.options.architectures: check_archall = arch in nobreakall_arches @@ -2339,27 +2335,18 @@ class Britney(object): # Copy nuninst_comp - we have to deep clone affected # architectures. - # NB: We do this *after* updating testing and we have to filter out + # NB: We do this *after* updating testing as we have to filter out # removed binaries. Otherwise, uninstallable binaries that were # removed by the item would still be counted. if item.architecture == 'source': - # Assume that all architectures are affected and deep - # copy nuninst_comp - nuninst = {} - for arch in architectures: - nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) - nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) + affected_architectures = architectures else: - # Shallow clone nuninst_comp except for the affected - # architecture, which is deep cloned. - arch = item.architecture - nuninst = nuninst_comp.copy() - nuninst[arch] = set(x for x in nuninst_comp[arch] if x in binaries[arch][0]) - nuninst[arch + "+all"] = set(x for x in nuninst_comp[arch + "+all"] if x in binaries[arch][0]) + affected_architectures = [item.architecture] + nuninst = clone_nuninst(nuninst_comp, binaries, affected_architectures) # check the affected packages on all the architectures - for arch in (item.architecture == 'source' and architectures or (item.architecture,)): + for arch in affected_architectures: check_archall = arch in nobreakall_arches check_packages(arch, affected, check_archall, nuninst) diff --git a/britney_util.py b/britney_util.py index 1043ad3..346bac3 100644 --- a/britney_util.py +++ b/britney_util.py @@ -595,3 +595,18 @@ def is_nuninst_asgood_generous(architectures, old, new, break_arches=frozenset() continue diff = diff + (len(new[arch]) - len(old[arch])) return diff <= 0 + + +def clone_nuninst(nuninst, packages_s, architectures): + """Selectively deep clone nuninst + + Given nuninst table, the package table for a given suite and + a list of architectures, this function will clone the nuninst + table. Only the listed architectures will be deep cloned - + the rest will only be shallow cloned. + """ + clone = nuninst.copy() + for arch in architectures: + clone[arch] = set(x for x in nuninst[arch] if x in packages_s[arch][0]) + clone[arch + "+all"] = set(x for x in nuninst[arch + "+all"] if x in packages_s[arch][0]) + return clone From 025d0dd3cfff721e2c0e1acec684afa736b83b13 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Tue, 11 Aug 2015 21:38:36 +0000 Subject: [PATCH 57/59] excuses: Point to the d-i release manager when block-udeb is involved. Signed-off-by: Cyril Brulebois --- britney.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/britney.py b/britney.py index 3b2651d..e82a626 100755 --- a/britney.py +++ b/britney.py @@ -1088,8 +1088,11 @@ class Britney(object): # if the package is blocked, skip it for hint in self.hints.search('block', package=pkg, removal=True): - excuse.addhtml("Not touching package, as requested by %s " - "(check https://release.debian.org/jessie/freeze_policy.html if update is needed)" % hint.user) + tooltip = "check https://release.debian.org/jessie/freeze_policy.html if update is needed" + # redirect people to d-i RM for udeb things: + if block_cmd == 'block-udeb': + tooltip = "please contact the d-i release manager if an update is needed" + excuse.addhtml("Not touching package, as requested by %s (%s)" % (hint.user, tooltip)) excuse.addhtml("Not considered") excuse.addreason("block") self.excuses.append(excuse) From 753087510bc62abac93f94e3f858ca51facb9ace Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Tue, 11 Aug 2015 22:25:23 +0000 Subject: [PATCH 58/59] Revert "excuses: Point to the d-i release manager when block-udeb is involved." This reverts commit 6adee798cd0bec3b64c8a6315667d68ab3c16bd9. Patching the wrong location considered unhelpful. Signed-off-by: Cyril Brulebois --- britney.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/britney.py b/britney.py index e82a626..3b2651d 100755 --- a/britney.py +++ b/britney.py @@ -1088,11 +1088,8 @@ class Britney(object): # if the package is blocked, skip it for hint in self.hints.search('block', package=pkg, removal=True): - tooltip = "check https://release.debian.org/jessie/freeze_policy.html if update is needed" - # redirect people to d-i RM for udeb things: - if block_cmd == 'block-udeb': - tooltip = "please contact the d-i release manager if an update is needed" - excuse.addhtml("Not touching package, as requested by %s (%s)" % (hint.user, tooltip)) + excuse.addhtml("Not touching package, as requested by %s " + "(check https://release.debian.org/jessie/freeze_policy.html if update is needed)" % hint.user) excuse.addhtml("Not considered") excuse.addreason("block") self.excuses.append(excuse) From 85070f38d834995493d3bb49a12d22870a70ba88 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Tue, 11 Aug 2015 22:26:57 +0000 Subject: [PATCH 59/59] =?UTF-8?q?excuses:=20New=20try=20for=20block-udeb?= =?UTF-8?q?=20=E2=86=92=20d-i=20RM.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cyril Brulebois --- britney.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/britney.py b/britney.py index 3b2651d..36dc546 100755 --- a/britney.py +++ b/britney.py @@ -1339,9 +1339,12 @@ class Britney(object): excuse.addhtml("%s request by %s ignored due to version mismatch: %s" % (unblock_cmd.capitalize(), unblocks[0].user, unblocks[0].version)) if suite == 'unstable' or block_cmd == 'block-udeb': - excuse.addhtml("Not touching package due to %s request by %s " - "(check https://release.debian.org/jessie/freeze_policy.html if update is needed)" % - (block_cmd, blocked[block_cmd].user)) + tooltip = "check https://release.debian.org/jessie/freeze_policy.html if update is needed" + # redirect people to d-i RM for udeb things: + if block_cmd == 'block-udeb': + tooltip = "please contact the d-i release manager if an update is needed" + excuse.addhtml("Not touching package due to %s request by %s (%s)" % + (block_cmd, blocked[block_cmd].user, tooltip)) excuse.addreason("block") else: excuse.addhtml("NEEDS APPROVAL BY RM")