From cf79fe5618218e53dc087e47c1b711f7369c4246 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Tue, 22 Jan 2013 13:02:13 +0000 Subject: [PATCH 01/16] Add jmw Signed-off-by: Adam D. Barratt --- britney.conf | 1 + britney_nobreakall.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/britney.conf b/britney.conf index 4af20ed..ae082a3 100644 --- a/britney.conf +++ b/britney.conf @@ -47,6 +47,7 @@ HINTS_JCRISTAU = STANDARD force force-hint HINTS_FAW = HELPERS HINTS_NTHYKIER = STANDARD HINTS_KIBI = STANDARD +HINTS_JMW = STANDARD HINTS_FREEZE = block block-all block-udeb HINTS_FREEZE-EXCEPTION = unblock unblock-udeb HINTS_SATBRITNEY = easy diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf index 905c99f..c7ea1cb 100644 --- a/britney_nobreakall.conf +++ b/britney_nobreakall.conf @@ -47,6 +47,7 @@ HINTS_JCRISTAU = STANDARD force force-hint HINTS_FAW = HELPERS HINTS_NTHYKIER = STANDARD HINTS_KIBI = STANDARD +HINTS_JMW = STANDARD HINTS_FREEZE = block block-all block-udeb HINTS_FREEZE-EXCEPTION = unblock unblock-udeb HINTS_SATBRITNEY = easy From 74d1b70096f69d335a552d900d45731be2bbd93b Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 2 Feb 2013 22:47:56 +0000 Subject: [PATCH 02/16] Don't flag unbuilt binary *pu binNMUs for removal in excuses Originally when binNMUs for packages in testing were scheduled, the binaries would be installed into tpu with no accompanying source. This allowed the "removed binary" portions of should_upgrade_srcarch() to be skipped (as britney had generated a faux source record). dak now adds the source package to tpu in such cases which lead to the "removed binary" checks being applied to binNMUs in tpu with potentially destructive consequences. For example, if a package with amd64 and i386 binaries in testing were binNMUed on just amd64, britney would notice that there were no i386 binaries in tpu and subsequently remove the i386 binaries from testing as well. In order to resolve this, we skip the check for removed binaries when building excuses for a binary-only migration via *pu. Signed-off-by: Adam D. Barratt --- britney.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/britney.py b/britney.py index ab87016..16151eb 100755 --- a/britney.py +++ b/britney.py @@ -1213,17 +1213,20 @@ class Britney(object): if not anywrongver and (anyworthdoing or not self.sources[suite][src][FAKESRC]): srcv = self.sources[suite][src][VERSION] ssrc = self.same_source(source_t[VERSION], srcv) - # for every binary package produced by this source in testing for this architecture - for pkg in sorted([x.split("/")[0] for x in self.sources['testing'][src][BINARIES] if x.endswith("/"+arch)]): - # if the package is architecture-independent, then ignore it - if self.binaries['testing'][arch][0][pkg][ARCHITECTURE] == 'all': - excuse.addhtml("Ignoring removal of %s as it is arch: all" % (pkg)) - continue - # if the package is not produced by the new source package, then remove it from testing - if pkg not in self.binaries[suite][arch][0]: - tpkgv = self.binaries['testing'][arch][0][pkg][VERSION] - excuse.addhtml("Removed binary: %s %s" % (pkg, tpkgv)) - if ssrc: anyworthdoing = True + # if this is a binary-only migration via *pu, we never want to try + # removing binary packages + if not (ssrc and suite != 'unstable'): + # for every binary package produced by this source in testing for this architecture + for pkg in sorted([x.split("/")[0] for x in self.sources['testing'][src][BINARIES] if x.endswith("/"+arch)]): + # if the package is architecture-independent, then ignore it + if self.binaries['testing'][arch][0][pkg][ARCHITECTURE] == 'all': + excuse.addhtml("Ignoring removal of %s as it is arch: all" % (pkg)) + continue + # if the package is not produced by the new source package, then remove it from testing + if pkg not in self.binaries[suite][arch][0]: + tpkgv = self.binaries['testing'][arch][0][pkg][VERSION] + excuse.addhtml("Removed binary: %s %s" % (pkg, tpkgv)) + if ssrc: anyworthdoing = True # if there is nothing wrong and there is something worth doing, this is a valid candidate if not anywrongver and anyworthdoing: From 9f4af0171ea3bb2562ff43f9269175d164862d43 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Wed, 6 Feb 2013 17:18:46 +0000 Subject: [PATCH 03/16] Fix tpu out-of-date checks for sources with multiple versions in tpu When checking whether a tpu source has built on a particular arch, we should only consider binaries produced by the latest version of the source package in tpu. Signed-off-by: Adam D. Barratt --- britney.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 16151eb..8489182 100755 --- a/britney.py +++ b/britney.py @@ -1374,7 +1374,9 @@ class Britney(object): if not src in self.sources["testing"] or \ (len([x for x in self.sources["testing"][src][BINARIES] if x.endswith("/"+arch) and self.binaries["testing"][arch][0][x.split("/")[0]][ARCHITECTURE] != 'all' ]) == 0) or \ - (len([x for x in self.sources[suite][src][BINARIES] if x.endswith("/"+arch) and self.binaries[suite][arch][0][x.split("/")[0]][ARCHITECTURE] != 'all' ]) > 0): + (len([x for x in self.binaries[suite][arch][0].values() \ + if x[SOURCE] == src and x[SOURCEVER] == source_u[VERSION] and \ + x[ARCHITECTURE] != 'all' ]) > 0): continue if suite == 'tpu': From 25cc2c3b36822accfedca56dcb6ddf22a85154b7 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Wed, 6 Feb 2013 19:05:16 +0000 Subject: [PATCH 04/16] Split the tpu o-o-d checks up to make them more readable Signed-off-by: Adam D. Barratt --- britney.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/britney.py b/britney.py index 8489182..ca9209b 100755 --- a/britney.py +++ b/britney.py @@ -1366,15 +1366,20 @@ class Britney(object): if suite in ['pu', 'tpu']: # o-o-d(ish) checks for (t-)p-u for arch in self.options.architectures: - # If the package isn't in testing or the testing - # package produces no packages on this architecture, - # then it can't be out-of-date. We assume that if - # the (t-)p-u package has produced any binaries for - # this architecture then it is ok - - if not src in self.sources["testing"] or \ - (len([x for x in self.sources["testing"][src][BINARIES] if x.endswith("/"+arch) and self.binaries["testing"][arch][0][x.split("/")[0]][ARCHITECTURE] != 'all' ]) == 0) or \ - (len([x for x in self.binaries[suite][arch][0].values() \ + if src not in self.sources["testing"]: + continue + + # if the package in testing has no binaries on this + # architecture, it can't be out-of-date + if (len([x for x in self.sources["testing"][src][BINARIES] + if x.endswith("/"+arch) and self.binaries["testing"][arch][0][x.split("/")[0]][ARCHITECTURE] != 'all' ]) == 0): + continue + + # if the (t-)p-u package has produced any binaries on + # this architecture then we assume it's ok. this allows for + # uploads to (t-)p-u which intentionally drop binary + # packages + if (len([x for x in self.binaries[suite][arch][0].values() \ if x[SOURCE] == src and x[SOURCEVER] == source_u[VERSION] and \ x[ARCHITECTURE] != 'all' ]) > 0): continue From acec4ff6e37815cda91517725bc8ffeae778b900 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Wed, 6 Feb 2013 19:09:04 +0000 Subject: [PATCH 05/16] Use any() in tpu o-o-d checks rather than throwaway lists The test only needs to consider whether any binaries exist on a given arch, not how many of them there are (or indeed which binaries they are) Signed-off-by: Adam D. Barratt --- britney.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/britney.py b/britney.py index ca9209b..4edce54 100755 --- a/britney.py +++ b/britney.py @@ -1371,17 +1371,17 @@ class Britney(object): # if the package in testing has no binaries on this # architecture, it can't be out-of-date - if (len([x for x in self.sources["testing"][src][BINARIES] - if x.endswith("/"+arch) and self.binaries["testing"][arch][0][x.split("/")[0]][ARCHITECTURE] != 'all' ]) == 0): + if not any(x for x in self.sources["testing"][src][BINARIES] + if x.endswith("/"+arch) and self.binaries["testing"][arch][0][x.split("/")[0]][ARCHITECTURE] != 'all'): continue # if the (t-)p-u package has produced any binaries on # this architecture then we assume it's ok. this allows for # uploads to (t-)p-u which intentionally drop binary # packages - if (len([x for x in self.binaries[suite][arch][0].values() \ - if x[SOURCE] == src and x[SOURCEVER] == source_u[VERSION] and \ - x[ARCHITECTURE] != 'all' ]) > 0): + if any(x for x in self.binaries[suite][arch][0].values() \ + if x[SOURCE] == src and x[SOURCEVER] == source_u[VERSION] and \ + x[ARCHITECTURE] != 'all'): continue if suite == 'tpu': From 2a590ff09566d80c2c91d3973ba75f1eb0bf25ed Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Wed, 6 Feb 2013 19:56:04 +0000 Subject: [PATCH 06/16] Replace a single-use list in the smooth updates checks with any() Signed-off-by: Adam D. Barratt --- britney.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/britney.py b/britney.py index 4edce54..4e567ce 100755 --- a/britney.py +++ b/britney.py @@ -1891,9 +1891,8 @@ class Britney(object): # outside of the current source for p in check: binary, parch = p.split("/") - rdeps = [ bin for bin in binaries[parch][0][binary][RDEPENDS] \ - if bin in [y.split("/")[0] for y in smoothbins] ] - if len(rdeps) > 0: + if any(bin for bin in binaries[parch][0][binary][RDEPENDS] \ + if bin in [y.split("/")[0] for y in smoothbins]): smoothbins.append(p) # remove all the binaries which aren't being smooth updated From 8c15568b8fcb780c4d5c2e06e46b551ab2859eb7 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sun, 3 Mar 2013 17:23:38 +0000 Subject: [PATCH 07/16] Don't remove arch:all packages when migrating binNMUs via *pu A binNMU does not rebuild architecture:all packages. For migrations via unstable this is not a problem as the packages corresponding to the source upload are still present. However, for *pu migrations, the set of packages considered only includes architecture-specific packages. In order to avoid installability issues with packages in testing which depend on the arch:all packages, we leave the existing arch:all packages in testing and only consider the arch-specific packages for migration. Signed-off-by: Adam D. Barratt --- britney.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/britney.py b/britney.py index 4e567ce..6007e6f 100755 --- a/britney.py +++ b/britney.py @@ -1846,7 +1846,16 @@ class Britney(object): # first, build a list of eligible binaries for p in source[BINARIES]: binary, parch = p.split("/") - if item.architecture != 'source' and parch != item.architecture: continue + if item.architecture != 'source': + # for a binary migration, binaries should not be removed: + # - unless they are for the correct architecture + if parch != item.architecture: continue + # - if they are arch:all and the migration is via *pu, + # as the packages will not have been rebuilt and the + # source suite will not contain them + if binaries[parch][0][binary][ARCHITECTURE] == 'all' and \ + item.suite != 'unstable': + continue # do not remove binaries which have been hijacked by other sources if binaries[parch][0][binary][SOURCE] != item.package: continue bins.append(p) From 8e72456651448e9d083131e55a4e356bef1c4dad Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Thu, 14 Mar 2013 19:58:14 +0000 Subject: [PATCH 08/16] Completer: don't crash if there are candidates in t-p-u Signed-off-by: Adam D. Barratt --- completer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/completer.py b/completer.py index cda4eb5..36345d5 100644 --- a/completer.py +++ b/completer.py @@ -40,6 +40,8 @@ class Completer(object): if pkg[0] == '-': suite = 'testing' pkg = pkg[1:] + if "_" in pkg: + (pkg, suite) = pkg.split("_") if "/" in pkg: pkg = pkg.split("/")[0] name = "%s/%s" % (e.name, britney.sources[suite][pkg][0]) # 0 == VERSION From 33bc6cb7fbdbd986e7f7639a9406bb2bfa7642d1 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 16:30:18 +0000 Subject: [PATCH 09/16] hint_tester: accept all hints For those hints which don't cause an immediate run (i.e. other than easy, hint and force-hint), re-build the excuses after adding the hint so that the actions are accounted for in later hints. Signed-off-by: Adam D. Barratt --- britney.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/britney.py b/britney.py index 6007e6f..386b1ff 100755 --- a/britney.py +++ b/britney.py @@ -2533,6 +2533,11 @@ class Britney(object): # quit the hint tester if input and input[0] in ('quit', 'exit'): break + elif input and input[0] in ('remove', 'approve', 'urgent', 'age-days', + 'block', 'block-udeb', 'unblock', 'unblock-udeb', + 'block-all', 'force'): + self.hints.add_hint(' '.join(input), 'hint-tester') + self.write_excuses() # run a hint elif input and input[0] in ('easy', 'hint', 'force-hint'): try: From 0463c666f093fce15db87393fe99dcb83794fd14 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 16:31:04 +0000 Subject: [PATCH 10/16] write_excuses: clear the list of excuses before building Although this isn't an issue during normal runs, the excuses might be built multiple times during a hint-tester run and should not accumulate during the run. Signed-off-by: Adam D. Barratt --- britney.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/britney.py b/britney.py index 386b1ff..252f722 100755 --- a/britney.py +++ b/britney.py @@ -1581,6 +1581,8 @@ class Britney(object): # if a package is going to be removed, it will have a "-" prefix upgrade_me = [] + self.excuses = [] + # for every source package in testing, check if it should be removed for pkg in sources['testing']: if should_remove_source(pkg): From d9a9a26c57072f34342fee605832373d7b29723d Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 16:39:23 +0000 Subject: [PATCH 11/16] Completer: support all hints Signed-off-by: Adam D. Barratt --- completer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/completer.py b/completer.py index 36345d5..9797b9e 100644 --- a/completer.py +++ b/completer.py @@ -28,7 +28,10 @@ class Completer(object): Creates a completer for a given britney. """ self.matches = [] - self.cmds = ['easy', 'hint', 'force-hint', 'exit', 'quit'] + self.cmds = ['easy', 'hint', 'force-hint', 'force', + 'force', 'age-days', 'urgent', 'block-all', + 'block', 'block-udeb', 'unblock', 'unblock-udeb', + 'exit', 'quit'] self.britney = britney # generate a completion list from excuses. # - it might contain too many items, but meh From de56c719b6eb1465a5fa94a5a427d9a8fae53093 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 16:59:19 +0000 Subject: [PATCH 12/16] Completer: add forgotten "remove" to command completion Signed-off-by: Adam D. Barratt --- completer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/completer.py b/completer.py index 9797b9e..851d466 100644 --- a/completer.py +++ b/completer.py @@ -28,7 +28,7 @@ class Completer(object): Creates a completer for a given britney. """ self.matches = [] - self.cmds = ['easy', 'hint', 'force-hint', 'force', + self.cmds = ['easy', 'hint', 'force-hint', 'force', 'remove', 'force', 'age-days', 'urgent', 'block-all', 'block', 'block-udeb', 'unblock', 'unblock-udeb', 'exit', 'quit'] From c345dea3b5b9c01a43442a09cb9ac512c6951d49 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 17:10:11 +0000 Subject: [PATCH 13/16] Completer: when processing a "remove", only offer packages from testing Signed-off-by: Adam D. Barratt --- completer.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/completer.py b/completer.py index 851d466..ae927b6 100644 --- a/completer.py +++ b/completer.py @@ -50,7 +50,9 @@ class Completer(object): name = "%s/%s" % (e.name, britney.sources[suite][pkg][0]) # 0 == VERSION complete.append(name) self.packages = sorted(complete) - + testing = britney.sources['testing'] + self.testing_packages = sorted("%s/%s" % (pkg, testing[pkg][0]) for pkg in testing) + def completer(self, text, state): """readline completer (see the readline API)""" @@ -64,11 +66,15 @@ class Completer(object): self.matches = [x for x in self.cmds if x.startswith(text)] else: # complete pkg/[arch/]version - start = bisect.bisect_left(self.packages, text) - while start < len(self.packages): - if not self.packages[start].startswith(text): + if words[0] == 'remove': + packages = self.testing_packages + else: + packages = self.packages + start = bisect.bisect_left(packages, text) + while start < len(packages): + if not packages[start].startswith(text): break - self.matches.append(self.packages[start]) + self.matches.append(packages[start]) start += 1 if len(self.matches) > state: From af86002deae72ff9853751949bfc5621c90f6579 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 17:12:04 +0000 Subject: [PATCH 14/16] Completer: copyright foo Signed-off-by: Adam D. Barratt --- completer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/completer.py b/completer.py index ae927b6..fa36676 100644 --- a/completer.py +++ b/completer.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (C) 2011 Niels Thykier +# Copyright (C) 2013 Adam D. Barratt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From ce72b31bc6cd25ba15d6b0cc4692c75ecea7525e Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 17:19:01 +0000 Subject: [PATCH 15/16] Completer: offer tpu packages when completing "approve" hints Signed-off-by: Adam D. Barratt --- completer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/completer.py b/completer.py index fa36676..6b89a10 100644 --- a/completer.py +++ b/completer.py @@ -37,6 +37,7 @@ class Completer(object): # generate a completion list from excuses. # - it might contain too many items, but meh complete = [] + tpu = [] for e in britney.excuses: ver = None pkg = e.name @@ -50,7 +51,10 @@ class Completer(object): pkg = pkg.split("/")[0] name = "%s/%s" % (e.name, britney.sources[suite][pkg][0]) # 0 == VERSION complete.append(name) + if suite == 'tpu': + tpu.append(name) self.packages = sorted(complete) + self.tpu_packages = sorted(tpu) testing = britney.sources['testing'] self.testing_packages = sorted("%s/%s" % (pkg, testing[pkg][0]) for pkg in testing) @@ -69,6 +73,8 @@ class Completer(object): # complete pkg/[arch/]version if words[0] == 'remove': packages = self.testing_packages + elif words[0] == 'approve': + packages = self.tpu_packages else: packages = self.packages start = bisect.bisect_left(packages, text) From 7933a720aa11ac45450df25fe69353252016cf81 Mon Sep 17 00:00:00 2001 From: "Adam D. Barratt" Date: Sat, 16 Mar 2013 17:19:49 +0000 Subject: [PATCH 16/16] Completer: add "approve" to the command completion Signed-off-by: Adam D. Barratt --- completer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/completer.py b/completer.py index 6b89a10..e5b0ca0 100644 --- a/completer.py +++ b/completer.py @@ -32,7 +32,7 @@ class Completer(object): self.cmds = ['easy', 'hint', 'force-hint', 'force', 'remove', 'force', 'age-days', 'urgent', 'block-all', 'block', 'block-udeb', 'unblock', 'unblock-udeb', - 'exit', 'quit'] + 'approve', 'exit', 'quit'] self.britney = britney # generate a completion list from excuses. # - it might contain too many items, but meh