#!/usr/bin/env python # Copyright 2001-6 Anthony Towns # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. import sys, re, string, time, whrandom, math import britney if len(sys.argv) != 4: print "Must specify testing, unstable, testing-updates directories." sys.exit(1) testingdir = sys.argv[1] unstabledir = sys.argv[2] testingupdatesdir = sys.argv[3] # Configuration information expected_arches = 12 allarches = [ 'i386', 'sparc', 'alpha', 'powerpc', 'arm', 'm68k', 'hppa', 'ia64', 'mips', 'mipsel', 's390', 'amd64' ] mindays = { "low" : 10, "medium" : 5, "high" : 2, "critical" : 0, "emergency" : 0 } defaulturgency = "low" # if you're not in this list, arch: all packages are allowed to break on you nobreakarchallarches = ['i386'] # if you're in this list, your packages may not stay in sync with the source fuckedarches = ['m68k', 'sparc'] # if you're in this list, your uninstallability count may increase breakarches = ['m68k', 'sparc'] # new architectures newarches = [] allarches.sort() arches = [ x for x in allarches if x in nobreakarchallarches ] arches += [ x for x in allarches if x not in arches and x not in fuckedarches ] arches += [ x for x in allarches if x not in arches and x not in breakarches ] arches += [ x for x in allarches if x not in arches and x not in newarches ] arches += [ x for x in allarches if x not in arches ] # Subs def same_source(sv1, sv2): if sv1 == sv2: return 1 m = re.match(r'^(.*)\+b\d+$', sv1) if m: sv1 = m.group(1) m = re.match(r'^(.*)\+b\d+$', sv2) if m: sv2 = m.group(1) if sv1 == sv2: return 1 if re.search("-", sv1) or re.search("-", sv2): m = re.match(r'^(.*-[^.]+)\.0\.\d+$', sv1) if m: sv1 = m.group(1) m = re.match(r'^(.*-[^.]+\.[^.]+)\.\d+$', sv1) if m: sv1 = m.group(1) m = re.match(r'^(.*-[^.]+)\.0\.\d+$', sv2) if m: sv2 = m.group(1) m = re.match(r'^(.*-[^.]+\.[^.]+)\.\d+$', sv2) if m: sv2 = m.group(1) return (sv1 == sv2) else: m = re.match(r'^([^-]+)\.0\.\d+$', sv1) if m and sv2 == m.group(1): return 1 m = re.match(r'^([^-]+)\.0\.\d+$', sv2) if m and sv1 == m.group(1): return 1 return 0 def read_approvals(dir, approver, approved): f = open("%s/%s" % (dir, approver)) line = f.readline() while line: l = string.split(line) if len(l) == 2: [pkg,ver] = l approved["%s_%s" % (pkg, ver)] = approver line = f.readline() f.close() def read_bugs(file): bugsperpkg = {} f = open(file) line = f.readline() while line: l = string.split(line) if len(l) == 2: bugsperpkg[l[0]] = string.atoi(l[1]) line = f.readline() f.close() return bugsperpkg def write_bugs(file, bugs): f = open(file, 'w') pkgs = bugs.keys() pkgs.sort() for pkg in pkgs: if bugs[pkg] == 0: continue f.write("%s %d\n" % (pkg, bugs[pkg])) f.close() def read_dates(file): dates = {} f = open(file) line = f.readline() while line: l = string.split(line) if len(l) == 3: dates[l[0]] = (l[1], string.atoi(l[2])) line = f.readline() f.close() return dates def write_dates(file, dates): f = open(file, 'w') pkgs = dates.keys() pkgs.sort() for pkg in dates.keys(): f.write("%s %s %d\n" % ((pkg,) + dates[pkg])) f.close() def read_urgencies(file, testing, unstable): urgency = {} f = open(file) line = f.readline() while line: l = string.split(line) if len(l) == 3: uo = urgency.get(l[0], defaulturgency) mo = mindays.get(uo, mindays[defaulturgency]) mn = mindays.get(l[2], mindays[defaulturgency]) if mo <= mn: line = f.readline() continue tsrcv = testing.get_version(l[0]) if tsrcv and britney.versioncmp(tsrcv, l[1]) >= 0: line = f.readline() continue usrcv = unstable.get_version(l[0]) if not usrcv or britney.versioncmp(usrcv, l[1]) < 0: line = f.readline() continue urgency[l[0]] = l[2] line = f.readline() f.close() return urgency def read_hints(dir, hinter, hints, allowed): res = {} for k in allowed: res[k] = [] try: f = open("%s/%s" % (dir, hinter)) except IOError: return res while 1: line = f.readline() if not line: break l = string.split(line) if len(l) == 0 or line[0] == "#": continue type = l[0] if type == "finished": break if type not in allowed: continue def mysplit(str): x = str.rfind("/") if x == -1: return [str] return [str[:x], str[x+1:]] if type in ["easy", "hint", "force-hint"]: l = [ tuple(mysplit(y)) for y in l[1:] ] l = [ k for k in l if len(k) == 2 ] res[type].append((hinter, l)) if type in ["block-all"]: l = [ (y, hinter) for y in l[1:] ] res[type].extend(l) if type in ["block"]: l = [ (y, hinter) for y in l[1:] ] res[type].extend(l) if type in ["remove", "approve", "unblock", "force", "urgent"]: l = [ tuple(mysplit(y)+[hinter]) for y in l[1:] ] l = [ k for k in l if len(k) == 3 ] l = [ (p, (v,h)) for (p,v,h) in l ] res[type].extend(l) f.close() return res class Excuse: reemail = re.compile(r"<.*?>") def __init__(self, name): self.name = name self.ver = ("-", "-") self.maint = None self.pri = None self.date = None self.urgency = None self.daysold = None self.mindays = None self.section = None self.dontinvalidate = 0 self.invalid_deps = [] self.deps = [] self.break_deps = [] self.bugs = [] self.htmlline = [] def set_vers(self, tver, uver): if tver: self.ver = (tver, self.ver[1]) if uver: self.ver = (self.ver[0], uver) def set_maint(self, maint): self.maint = self.reemail.sub("",maint) # self.maint = maint def set_section(self, section): self.section = section def set_priority(self, pri): self.pri = pri def set_date(self, date): self.date = date def set_urgency(self, date): self.urgency = date def add_dep(self, name): if name not in self.deps: self.deps.append(name) def add_break_dep(self, name, arch): if (name, arch) not in self.break_deps: self.break_deps.append( (name, arch) ) def invalidate_dep(self, name): if name not in self.invalid_deps: self.invalid_deps.append(name) def setdaysold(self, daysold, mindays): self.daysold = daysold self.mindays = mindays def addhtml(self, note): self.htmlline.append(note) def html(self): res = "%s (%s to %s)\n\n" return res def should_remove_source(src, orig, new, excs): if new.is_present(src): return 0 okay = 1 exc = Excuse("-" + src) exc.set_vers(orig.get_version(src), None) m = orig.get_field(src, "Maintainer") if m: exc.set_maint(string.strip(m)) s = orig.get_field(src, "Section") if s: exc.set_section(string.strip(s)) if hints["block"].has_key("-" + src): blocked = hints["block"]["-" + src] exc.addhtml("Not touching package, as requested by %s (contact debian-release if update is needed)" % (blocked)) okay = 0 if okay: exc.addhtml("Valid candidate") else: exc.addhtml("Not considered") excs.append(exc) return okay def should_upgrade_srcarch(src, arch, suite, tsrcv, orig, opkgsa, new, npkgsa, excs): # binnmu this arch? anywrongver = 0 anyworthdoing = 0 ref = "%s/%s" % (src, arch) if suite: ref = ref + "_%s" % (suite) e = Excuse(ref) e.set_vers(tsrcv, tsrcv) m = new.get_field(src, "Maintainer") if m: e.set_maint(string.strip(m)) s = new.get_field(src, "Section") if s: e.set_section(string.strip(s)) if hints["remove"].has_key(src): if same_source(tsrcv, hints["remove"][src][0]): e.addhtml("Removal request by %s" % (hints["remove"][src][1])) e.addhtml("Trying to remove package, not update it") e.addhtml("Not considered") excs.append(e) return 0 for pkg in new.binaries(src, arch): pkgv = npkgsa.get_version(pkg) pkgsv = npkgsa.get_sourcever(pkg) if npkgsa.is_arch_all(pkg): e.addhtml("Ignoring %s %s (from %s) as it is arch: all" % (pkg, pkgv, pkgsv)) continue if not same_source(tsrcv, pkgsv): anywrongver = 1 e.addhtml("From wrong source: %s %s (%s not %s)" % ( pkg, pkgv, pkgsv, tsrcv)) break excuse_unsat_deps(pkg, arch, opkgsa, npkgsa, e, 0) if not opkgsa.is_present(pkg): e.addhtml("New binary: %s (%s)" % (pkg, pkgv)) anyworthdoing = 1 continue tpkgv = opkgsa.get_version(pkg) if britney.versioncmp(tpkgv, pkgv) > 0: anywrongver = 1 e.addhtml("Not downgrading: %s (%s to %s)" % ( pkg, tpkgv, pkgv)) break elif britney.versioncmp(tpkgv, pkgv) < 0: e.addhtml("Updated binary: %s (%s to %s)" % ( pkg, tpkgv, pkgv)) anyworthdoing = 1 if not anywrongver and (anyworthdoing or not new.is_fake(src)): srcv = new.get_version(src) ssrc = same_source(tsrcv, srcv) for pkg in orig.binaries(src, arch): if opkgsa.is_arch_all(pkg): e.addhtml("Ignoring removal of %s as it is arch: all" % (pkg)) continue if not npkgsa.is_present(pkg): tpkgv = opkgsa.get_version(pkg) e.addhtml("Removed binary: %s %s" % ( pkg, tpkgv)) if ssrc: anyworthdoing = 1 if not anywrongver and anyworthdoing: e.addhtml("Valid candidate") excs.append(e) return 1 else: if anyworthdoing: e.addhtml("Not considered") excs.append(e) return 0 def excuse_unsat_deps(pkg, arch, tpkgsarch, upkgsarch, exc, ignore_break=0): for d in ['Pre-Depends', 'Depends']: udt = tpkgsarch.unsatisfiable_deps(upkgsarch, pkg, d) udu = upkgsarch.unsatisfiable_deps(upkgsarch, pkg, d) for t,u in map(None, udt, udu): if t[1]: continue l = [] for e in u[1]: s = upkgsarch.get_source(e) if s not in l: l.append(s) if src in l: continue if l == []: exc.addhtml("%s/%s unsatisfiable %s: %s" % (pkg, arch, d, t[0])) for s in l: if ignore_break or arch not in breakarches: exc.add_dep(s) else: exc.add_break_dep(s, arch) def should_upgrade_src(src, suite, orig, origpkgs, new, newpkgs, approvals, excs): srcv = new.get_version(src) if orig.is_present(src): tsrcv = orig.get_version(src) if britney.versioncmp(srcv, tsrcv) == 0: # Candidate for binnmus only return 0 else: tsrcv = None updatecand = 1 ref = src if suite: ref = ref + "_tpu" exc = Excuse(ref) exc.set_vers(tsrcv, srcv) m = new.get_field(src, "Maintainer") if m: exc.set_maint(string.strip(m)) s = new.get_field(src, "Section") if s: exc.set_section(string.strip(s)) if tsrcv and britney.versioncmp(srcv, tsrcv) < 0: # Version in unstable is older! exc.addhtml("ALERT: %s is newer in testing (%s %s)" % (src, tsrcv, srcv)) excs.append(exc) return 0 if unstable.is_fake(src): exc.addhtml("%s source package doesn't exist" % (src)) updatecand = 0 urgency = unstableurg.get(src, defaulturgency) if not tsrcv and urgency != defaulturgency: exc.addhtml("Ignoring %s urgency setting for NEW package" % (urgency)) urgency = defaulturgency if hints["remove"].has_key(src): if (tsrcv and same_source(tsrcv, hints["remove"][src][0])) or \ same_source(srcv, hints["remove"][src][0]): exc.addhtml("Removal request by %s" % (hints["remove"][src][1])) exc.addhtml("Trying to remove package, not update it") updatecand = 0 blocked = None if hints["block"].has_key(src): blocked = hints["block"][src] elif hints["block-all"].has_key("source"): blocked = hints["block-all"]["source"] if blocked: ubv = hints["unblock"].get(src,(None,None))[0] if ubv != None and same_source(ubv, srcv): exc.addhtml("Ignoring request to block package by %s, due to unblock request by %s" % (blocked, hints["unblock"][src][1])) else: if ubv != None: exc.addhtml("Unblock request by %s ignored due to version mismatch: %s" % (hints["unblock"][src][1], hints["unblock"][src][0])) exc.addhtml("Not touching package, as requested by %s (contact debian-release if update is needed)" % (blocked)) updatecand = 0 if suite == None: if not unstabledates.has_key(src): unstabledates[src] = (srcv, datenow) elif not same_source(unstabledates[src][0], srcv): unstabledates[src] = (srcv, datenow) daysold = datenow - unstabledates[src][1] mymindays = mindays[urgency] exc.setdaysold(daysold, mymindays) if daysold < mymindays: if hints["urgent"].has_key(src) and same_source(srcv, hints["urgent"][src][0]): exc.addhtml("Too young, but urgency pushed by %s" % (hints["urgent"][src][1])) else: updatecand = 0 pkgs = { src: ["source"] } anybins = 0 for arch in arches: oodbins = {} for pkg in new.binaries(src,arch): anybins = 1 if not pkgs.has_key(pkg): pkgs[pkg] = [] pkgs[pkg].append(arch) pkgsv = newpkgs[arch].get_sourcever(pkg) if not same_source(srcv, pkgsv): if not oodbins.has_key(pkgsv): oodbins[pkgsv] = [] oodbins[pkgsv].append(pkg) continue if newpkgs[arch].isnt_arch_all(pkg) or \ arch in nobreakarchallarches: excuse_unsat_deps(pkg, arch, origpkgs[arch], newpkgs[arch], exc) if oodbins: oodtxt = "" for v in oodbins.keys(): if oodtxt: oodtxt = oodtxt + "; " oodtxt = oodtxt + "%s (from %s)" % \ (", ".join(oodbins[v]), arch, src, v, v) text = "out of date on %s: %s" % (arch, src, srcv, arch, oodtxt) if arch in fuckedarches: text = text + " (but %s isn't keeping up," % \ (arch) + " so nevermind)" else: updatecand = 0 if datenow != unstabledates[src][1]: exc.addhtml(text) if not anybins: exc.addhtml("%s has no binaries on any arch" % src) updatecand = 0 if suite == None: for pkg in pkgs.keys(): if not testingbugs.has_key(pkg): testingbugs[pkg] = 0 if not unstablebugs.has_key(pkg): unstablebugs[pkg] = 0 if unstablebugs[pkg] > testingbugs[pkg]: exc.addhtml("%s (%s) is buggy! (%d > %d)" % \ (pkg, ", ".join(pkgs[pkg]), pkg, unstablebugs[pkg], testingbugs[pkg])) updatecand = 0 elif unstablebugs[pkg] > 0: exc.addhtml("%s (%s) is (less) buggy! (%d <= %d)" % \ (pkg, ", ".join(pkgs[pkg]), pkg, unstablebugs[pkg], testingbugs[pkg])) if not updatecand and hints["force"].has_key(src) and same_source(srcv, hints["force"][src][0]) : exc.dontinvalidate = 1 exc.addhtml("Should ignore, but forced by %s" % (hints["force"][src][1])) updatecand = 1 if approvals: if approvals.has_key("%s_%s" % (src, srcv)): exc.addhtml("Approved by %s" % approvals["%s_%s" % (src, srcv)]) else: exc.addhtml("NEEDS APPROVAL BY RM") updatecand = 0 if updatecand: exc.addhtml("Valid candidate") else: exc.addhtml("Not considered") excuses.append(exc) return updatecand ### # Brute force stuff class UpgradeRun: def __init__(self, sn, u, tu, ps): self.srcsn = sn self.unstable = u self.testingupdates = tu self.packages = ps self.sortpkgs() self.output = open("update.OUTPUT_py", "w"); self.arches = [ x for x in arches if x in srcsn.arches ] self.srcsnpkgs = {} for arch in arches: self.srcsnpkgs[arch] = self.srcsn.Packages(arch) #def __del__(): # self.output.close() def sortpkgs(self): p = self.packages p.sort() self.packages = p def writeout(self, text): self.output.write(text) sys.stdout.write(text) self.output.flush() sys.stdout.flush() def doop_source(self, op): # removals = "-", # arch = "/", # normal = "" which = self.unstable if "_" in op: ind = string.index(op, "_") if op[ind+1:] == "tpu": which = self.testingupdates op = op[:ind] if op[0] == "-": self.srcsn.remove_source(op[1:]) elif "/" in op: ind = string.index(op, "/") self.srcsn.upgrade_arch(which, op[:ind], op[ind+1:]) else: self.srcsn.upgrade_source(which, op) def get_nuninst(self): nuninst = {} for arch in self.arches: con = self.srcsnpkgs[arch].packages if arch not in nobreakarchallarches: con = filter( self.srcsnpkgs[arch].isnt_arch_all, con) nuninst[arch] = filter( self.srcsnpkgs[arch].is_uninstallable, con) return nuninst def get_improved_nuninst(self, old): new = {} for arch in self.arches: con = self.srcsnpkgs[arch].packages if arch not in nobreakarchallarches: con = filter( self.srcsnpkgs[arch].isnt_arch_all, con) new[arch] = filter( self.srcsnpkgs[arch].is_uninstallable, con) if arch in breakarches: continue if len(new[arch]) > len(old[arch]): return (0, new) return (1, new) def arch_improved_nuninst(self, old, arch): new = old.copy() if "_" in arch: arch = arch[:arch.index("_")] con = self.srcsnpkgs[arch].packages if arch not in nobreakarchallarches: con = filter(self.srcsnpkgs[arch].isnt_arch_all, con) new[arch] = filter(self.srcsnpkgs[arch].is_uninstallable, con) if arch not in newarches and len(new[arch]) > len(old[arch]): return (0, new) return (1, new) def is_nuninst_asgood(self, old, new): for arch in self.arches: if arch in breakarches: continue if len(new[arch]) > len(old[arch]): return 0 return 1 def is_nuninst_asgood_generous(self, old, new): diff = 0 for arch in self.arches: if arch in breakarches: continue diff = diff + (len(new[arch]) - len(old[arch])) return diff <= 0 def eval_nuninst(self, nuninst): res = [] total = 0 totalbreak = 0 for arch in self.arches: if nuninst.has_key(arch): n = len(nuninst[arch]) if arch in breakarches: totalbreak = totalbreak + n else: total = total + n res.append("%s-%d" % (arch[0], n)) return "%d+%d: %s" % (total, totalbreak, ":".join(res)) def slist_subtract(self, base, sub): res = [] for x in base: if x not in sub: res.append(x) return res def newlyuninst(self, nuold, nunew): res = {} for arch in self.arches: if not nuold.has_key(arch) or not nunew.has_key(arch): continue res[arch] = \ self.slist_subtract(nunew[arch], nuold[arch]) return res def eval_uninst(self, nuninst): res = "" for arch in self.arches: if nuninst.has_key(arch) and nuninst[arch] != []: res = res + " * %s: %s\n" % (arch, ", ".join(nuninst[arch])) return res def do_all(self, maxdepth = 0, init = []): self.selected = [] self.selected_committed = 0 packages = self.packages[:] earlyabort = 0 if maxdepth == "easy": earlyabort = 1 maxdepth = 0 # meaningless to try forcing something _and_ recurse force = 0 if maxdepth < 0: force = 1 maxdepth = 0 earlyabort = 1 nuninst_start = self.get_nuninst() if init: self.writeout("leading: %s\n" % (",".join(init))) for x in init: if x not in packages: self.writeout("failed: %s\n" % (x)) return None y = packages.index(x) self.selected.append(packages.pop(y)) for x in init: self.doop_source(x) if force: self.nuninst_orig = self.get_nuninst() else: self.nuninst_orig = nuninst_start self.writeout("start: %s\n" % self.eval_nuninst(nuninst_start)) self.writeout("orig: %s\n" % self.eval_nuninst(self.nuninst_orig)) if earlyabort: nuninst_end = self.get_nuninst() self.writeout("easy: %s\n" % (self.eval_nuninst(nuninst_end))) self.writeout(self.eval_uninst( self.newlyuninst(self.nuninst_orig, nuninst_end))) self.writeout("\n") if not self.is_nuninst_asgood_generous( self.nuninst_orig, nuninst_end): nuninst_end, respackages = None, None else: respackages = packages[:] self.selected_committed = len(self.selected) else: nuninst_end, respackages = \ self.iter_some(maxdepth, packages, []) if nuninst_end: assert(len(self.selected) == self.selected_committed) self.writeout("final: %s\n" % ",".join(self.selected)) self.writeout("start: %s\n" % self.eval_nuninst(nuninst_start)) self.writeout(" orig: %s\n" % self.eval_nuninst(self.nuninst_orig)) self.writeout(" end: %s\n" % self.eval_nuninst(nuninst_end)) if force: self.writeout("force breaks:\n") self.writeout(self.eval_uninst( self.newlyuninst(nuninst_start, nuninst_end))) self.writeout("\n") if not self.is_nuninst_asgood_generous( self.nuninst_orig, nuninst_end): print "NON-None RETURN THAT'S NOT BETTER" self.srcsn.commit_changes() self.writeout("SUCCESS (%d/%d)\n" % (len(self.packages), len(respackages))) self.packages = respackages self.sortpkgs() return self.selected else: assert(len(self.selected) == len(init)) assert(self.selected_committed == 0) for x in init: self.srcsn.undo_change() if self.srcsn.can_undo: print "MORE OPS LEFT TO UNDO THAN DONE" self.writeout("FAILED\n") return None def iter_end(self, available): extra = [] count = 0 nuninst_comp = self.get_nuninst() while available: x = available.pop(0) self.writeout("trying: %s\n" % (x)) self.doop_source(x) if "/" in x: better, nuninst_new = self.arch_improved_nuninst( nuninst_comp, x[x.index("/")+1:]) else: better, nuninst_new = self.get_improved_nuninst( nuninst_comp) if better: self.selected.append(x) count = count + 1 available.extend(extra) extra = [] self.writeout("accepted: %s\n" % (x)) self.writeout(" ori: %s\n" % (self.eval_nuninst(self.nuninst_orig))) self.writeout(" pre: %s\n" % (self.eval_nuninst(nuninst_comp))) self.writeout(" now: %s\n" % (self.eval_nuninst(nuninst_new))) if len(self.selected) <= 20: self.writeout(" all: %s\n" % ( " ".join(self.selected))) else: self.writeout(" most: (%d) .. %s\n" % (len(self.selected), " ".join(self.selected[-20:]))) nuninst_comp = nuninst_new else: self.writeout("skipped: %s (%d <- %d)\n" % ( x, len(extra), len(available))) self.writeout(" got: %s\n%s" % ( self.eval_nuninst(nuninst_new), self.eval_uninst(self.newlyuninst( nuninst_comp, nuninst_new)))) self.srcsn.undo_change() extra.append(x) self.writeout(" finish: [%s]\n" % ",".join(self.selected[self.selected_committed:])) self.writeout("endloop: %s\n" % (self.eval_nuninst(self.nuninst_orig))) self.writeout(" now: %s\n" % (self.eval_nuninst(nuninst_comp))) self.writeout(self.eval_uninst( self.newlyuninst(self.nuninst_orig, nuninst_comp))) self.writeout("\n") if self.is_nuninst_asgood_generous(self.nuninst_orig, nuninst_comp): self.writeout("Apparently successful\n") self.selected_committed = len(self.selected) return (nuninst_comp, extra) while count > 0: self.srcsn.undo_change() self.selected.pop() count = count - 1 return (None, None) def iter_some(self, depth, available, extra): self.writeout("recur: [%s] %s %d/%d\n" % ( ",".join(self.selected[:self.selected_committed]), ",".join(self.selected[self.selected_committed:]), len(available), len(extra))) if depth == 0: extra.extend(available) return self.iter_end(extra) nuninst = None while len(available) > depth: x = available.pop(0) if not skiphint(x): self.doop_source(x) self.selected.append(x) res = self.iter_some(depth - 1, available[:], extra[:]) if res[0]: nuninst = res[0] available = filter(lambda x, y=res[1]: x in y, available + extra) # reset nuninst_orig too self.nuninst_orig = nuninst extra = [] continue self.srcsn.undo_change() self.selected.pop() extra.append(x) return (nuninst, extra) # Package information testing = britney.Sources(testingdir, arches) testingpkgs = {} for arch in arches: testingpkgs[arch] = testing.Packages(arch) testingbugs = read_bugs(testingdir + "/Bugs") unstable = britney.Sources(unstabledir, arches) unstablepkgs = {} for arch in arches: unstablepkgs[arch] = unstable.Packages(arch) unstablebugs = read_bugs(unstabledir + "/Bugs") unstabledates = read_dates(testingdir + "/Dates") unstableurg = read_urgencies(testingdir + "/Urgency", testing, unstable) testingupdates = britney.Sources(testingupdatesdir, arches) testingupdatespkgs = {} for arch in arches: testingupdatespkgs[arch] = testingupdates.Packages(arch) testingupdatesapproved = {} # pkg_ver -> who for approver in ["ajt", "security-team", "ftpmaster", "cjwatson", "vorlon"]: read_approvals(testingupdatesdir + "/Approved", approver, testingupdatesapproved) hlphints = ["easy", "hint", "remove", "block", "unblock", "approve"] stdhints = ["easy", "hint", "remove", "block", "unblock", "urgent", "approve"] allhints = ["force", "force-hint", "block-all"] + stdhints hintsallowed = { "ajt": allhints, "rmurray": allhints, "cjwatson": allhints, "vorlon": allhints, "aba": allhints, "joeyh": stdhints + ["force"], "djpig": stdhints, "he": hlphints, "adeodato": hlphints, "ballombe": hlphints, "luk": hlphints, "freeze": ["block", "block-all"], "ftpteam": ["block"] } hints = {"easy":[], "hint":[], "force-hint":[], "remove":[], "block":[], "block-all":[], "unblock":[], "force":[], "urgent":[], "approve":[]} for who in hintsallowed.keys(): h = read_hints(unstabledir + "/Hints", who, hints, hintsallowed[who]) for k in hintsallowed[who]: hints[k].extend(h[k]) for x in ["block", "block-all", "unblock", "force", "urgent", "remove"]: z = {} for a, b in hints[x]: if z.has_key(a): print "Overriding %s[%s] = %s with %s" % (x, a, z[a], b) z[a] = b hints[x] = z for p, vh in hints["approve"]: (v,h) = vh testingupdatesapproved["%s_%s" % (p,v)] = h hints["unblock"]["%s" % p] = (v,h) def maxver(pkg, source, pkgs): maxver = source.get_version(pkg) for arch in arches: pkgv = pkgs[arch].get_version(pkg) if pkgv == None: continue if maxver == None or britney.versioncmp(pkgv, maxver) > 0: maxver = pkgv return maxver for pkg in testingbugs.keys() + unstablebugs.keys(): if not testingbugs.has_key(pkg): testingbugs[pkg] = 0 if not unstablebugs.has_key(pkg): unstablebugs[pkg] = 0 maxvert = maxver(pkg, testing, testingpkgs) if maxvert == None: testingbugs[pkg] = 0 continue if testingbugs[pkg] == unstablebugs[pkg]: continue maxveru = maxver(pkg, unstable, unstablepkgs) if maxveru == None: continue if britney.versioncmp(maxvert, maxveru) >= 0: testingbugs[pkg] = unstablebugs[pkg] datenow = int(((time.time() / (60*60)) - 15) / 24); # Next, work out which packages are candidates to be changed. upgrademe = [] excuses = [] # Packages to be removed for src in testing.sources: if should_remove_source(src, testing, unstable, excuses): upgrademe.append("-" + src) # Packages to be upgraded from unstable: for src in unstable.sources: if testing.is_present(src): tsrcv = testing.get_version(src) # silly optimisation for arch in arches: if should_upgrade_srcarch(src, arch, None, tsrcv, testing, testingpkgs[arch], unstable, unstablepkgs[arch], excuses): upgrademe.append("%s/%s" % (src, arch)) if should_upgrade_src(src, None, testing, testingpkgs, unstable, unstablepkgs, None, excuses): upgrademe.append(src) for src in testingupdates.sources: if testing.is_present(src): tsrcv = testing.get_version(src) # silly optimisation for arch in arches: if should_upgrade_srcarch(src, arch, "tpu", tsrcv, testing, testingpkgs[arch], testingupdates, testingupdatespkgs[arch], excuses): upgrademe.append("%s/%s_tpu" % (src, arch)) if should_upgrade_src(src, "tpu", testing, testingpkgs, testingupdates, testingupdatespkgs, testingupdatesapproved, excuses): upgrademe.append("%s_tpu" % src) for src in hints["remove"].keys(): if src in upgrademe: continue if ("-"+src) in upgrademe: continue if not testing.is_present(src): continue tsrcv = testing.get_version(src) if not same_source(tsrcv, hints["remove"][src][0]): continue upgrademe.append("-%s" % (src)) exc = Excuse("-%s" % (src)) exc.set_vers(tsrcv, None) exc.addhtml("Removal request by %s" % (hints["remove"][src][1])) exc.addhtml("Package is broken, will try to remove") excuses.append(exc) def cmpexcuses(el, er): return cmp(el.daysold, er.daysold) or cmp(el.name, er.name) excuses.sort(cmpexcuses) def reversed_exc_deps(excuses): res = {} for exc in excuses: for d in exc.deps: if not res.has_key(d): res[d] = [] res[d].append(exc.name) return res def invalidate(excuses, valid, invalid): i = 0 exclookup = {} for e in excuses: exclookup[e.name] = e revdeps = reversed_exc_deps(excuses) while i < len(invalid): if not revdeps.has_key(invalid[i]): i += 1 continue if (invalid[i] + "_tpu") in valid: i += 1 continue for x in revdeps[invalid[i]]: if x in valid and exclookup[x].dontinvalidate: continue exclookup[x].invalidate_dep(invalid[i]) if x in valid: p = valid.index(x) invalid.append(valid.pop(p)) exclookup[x].addhtml("Invalidated by dependency") exclookup[x].addhtml("Not considered") i = i + 1 unconsidered = [] for exc in excuses: if exc.name not in upgrademe: unconsidered.append(exc.name) for exc in excuses: for d in exc.deps: if d not in upgrademe and d not in unconsidered: exc.addhtml("Unpossible dep: %s -> %s" % (exc.name, d)) invalidate(excuses, upgrademe, unconsidered) f = open("update.EXCUSES_py", 'w') f.write("\n") f.write("excuses...") f.write("\n") f.write("

Generated: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "

\n") f.write("\n") f.close() del excuses # Changes srcsn = britney.SourcesNote(arches) # Initialise new testing to be the old testing for src in testing.sources: srcsn.upgrade_source(testing, src) srcsn.commit_changes() #print "Things to do:" #for x in upgrademe: # print " " + x run = UpgradeRun(srcsn, unstable, testingupdates, upgrademe) def skiphint(candidate): if "/" in candidate and candidate[candidate.rfind("/")+1:] in breakarches: return 1 return 0 def do_hint(type, who, pkgvers): hintinfo = {"easy": "easy", "hint": 0, "force-hint": -1, } hintdoall = hintinfo[type] run.writeout("Trying %s from %s: %s\n" % (type, who, " ".join( ["%s/%s" % (p,v) for (p,v) in pkgvers] ))) ok = 1 for xp,v in pkgvers: # is this version of this package present in unstable? # (if it's also present in testing, do_all will skip it) if "/" in xp: p = xp[:xp.find("/")] else: p = xp if p[0] == "-": pass elif p.endswith("_tpu"): if britney.versioncmp(run.testingupdates.get_version(p[:-4]),v) != 0: ok = 0 run.writeout(" Version mismatch, %s %s != %s\n" % (p, v, run.testingupdates.get_version(p[:-4]))) elif run.unstable.get_version(p) == None: ok = 0 run.writeout(" Source %s has no version in unstable\n" % p) elif britney.versioncmp(run.unstable.get_version(p), v) != 0: ok = 0 run.writeout(" Version mismatch, %s %s != %s\n" % (p, v, run.unstable.get_version(p))) if ok: run.do_all(hintdoall, map(lambda hp: hp[0], x[1])) else: run.writeout("Not using hint\n") return ok run.writeout("Generated on: %s\n" % (time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())))) run.writeout("Arch order is: %s\n" % ", ".join(arches)) for x in hints["easy"]: do_hint("easy", x[0], x[1]) for x in hints["force-hint"]: do_hint("force-hint", x[0], x[1]) allpackages = [] normpackages = run.packages[:] archpackages = {} for a in breakarches: la = len(a) + 1 archpackages[a] = [ p for p in normpackages if p[-la:] == "/" + a ] normpackages = [ p for p in normpackages if p[-la:] != "/" + a ] run.packages = normpackages run.writeout("info: main run\n"); run.do_all() allpackages += run.packages for a in breakarches: x = breakarches breakarches = [ ba for ba in breakarches if ba != a ] run.packages = archpackages[a] run.writeout("info: broken arch run for %s\n" % (a)) run.do_all() #run.do_all(1) breakarches = x allpackages += run.packages run.packages = allpackages #run.do_all(0,["caudium", "sablotron"]) hintcnt = 1 rand = whrandom.whrandom() rand.seed(23,187,96) for q in range(datenow): rand.random() q = rand.random() q = 1.0 run.writeout("Current value is %f\n" % (q)) if q < 0.2: q = 0.2 run.writeout("Current value bumped to %f\n" % (q)) maxloops = int(math.ceil(math.log(100/(q**0.5)) / math.log(1+len(run.packages)))) maxloops = 1 run.writeout("Max loops for q=%.2f is %d\n" % (q, maxloops)) for x in hints["hint"]: if hintcnt > 50: run.writeout("Skipping remaining hints...") break if len(x[1]) < maxloops: run.writeout("Skipping simple hint from %s (%d<%d): %s\n" % (x[0], len(x[1]), maxloops, str(x[1]))) continue if do_hint("hint", x[0], x[1]): hintcnt += 1 for i in range(1,maxloops): run.do_all(i) if maxloops <= 1 and len(run.packages) < 500: # too many to do all of them, let's try 5 at random num_at_random = 5 if len(run.packages) > num_at_random: run.writeout("Trying %d at random\n" % num_at_random) for k in range(num_at_random): special = rand.choice(run.packages) if skiphint(special): continue run.writeout("Randomly trying %s\n" % (special)) run.do_all(0, [special]) run.srcsn.write_notes(testingdir) write_bugs(testingdir + "/Bugs", testingbugs) write_dates(testingdir + "/Dates", unstabledates) f = open(testingdir + '/HeidiResult', 'w') for arch in arches: pkgs = srcsn.Packages(arch) for pkg in pkgs.packages: pkgv = pkgs.get_version(pkg) pkgarch = pkgs.get_field(pkg, 'Architecture') pkgsec = pkgs.get_field(pkg, 'Section') if pkgsec == None: pkgsec = 'unknown\n' pkgarch = pkgarch[:-1] pkgsec = pkgsec[:-1] f.write('%s %s %s %s\n' % (pkg, pkgv, pkgarch, pkgsec)) for src in srcsn.sources: srcv = srcsn.get_version(src) srcsec = srcsn.get_field(src, 'Section') if srcsec == None: srcsec = 'unknown\n' if srcsn.is_fake(src): srcsec = 'faux\n' srcsec = srcsec[:-1] f.write('%s %s source %s\n' % (src, srcv, srcsec)) f.close() if len(arches) != expected_arches: sys.exit(1)