britney2-ubuntu/lib/update_out.py

#!/usr/bin/env python

# Copyright 2001-6 Anthony Towns

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

import sys, re, string, time, whrandom, math
import britney

if len(sys.argv) != 4:
	print "Must specify testing, unstable, testing-updates directories."
	sys.exit(1)

testingdir = sys.argv[1]
unstabledir = sys.argv[2]
testingupdatesdir = sys.argv[3]

# Configuration information

expected_arches = 13
allarches = [ 'i386', 'sparc', 'alpha', 'powerpc', 'armel', 'hppa', 'ia64', 'mips', 'mipsel', 's390', 'amd64' , 'kfreebsd-i386', 'kfreebsd-amd64']

mindays = { "low" : 10, "medium" : 5, "high" : 2, "critical" : 0, 
	    "emergency" : 0 }
defaulturgency = "low"

# if you're not in this list, arch: all packages are allowed to break on you
nobreakarchallarches = ['i386']
# if you're in this list, your packages may not stay in sync with the source
fuckedarches = ['kfreebsd-i386','kfreebsd-amd64']
# if you're in this list, your uninstallability count may increase
breakarches = ['kfreebsd-i386','kfreebsd-amd64']
# new architectures
newarches = ['kfreebsd-i386','kfreebsd-amd64']

allarches.sort()
arches = [ x for x in allarches if x in nobreakarchallarches ]
arches += [ x for x in allarches if x not in arches and x not in fuckedarches ]
arches += [ x for x in allarches if x not in arches and x not in breakarches ]
arches += [ x for x in allarches if x not in arches and x not in newarches ]
arches += [ x for x in allarches if x not in arches ]

# Subs

def same_source(sv1, sv2):
	if sv1 == sv2:
		return 1

	m = re.match(r'^(.*)\+b\d+$', sv1)
	if m: sv1 = m.group(1)
	m = re.match(r'^(.*)\+b\d+$', sv2)
	if m: sv2 = m.group(1)

	if sv1 == sv2:
		return 1

	if re.search("-", sv1) or re.search("-", sv2):
		m = re.match(r'^(.*-[^.]+)\.0\.\d+$', sv1)
		if m: sv1 = m.group(1)
		m = re.match(r'^(.*-[^.]+\.[^.]+)\.\d+$', sv1)
		if m: sv1 = m.group(1)

		m = re.match(r'^(.*-[^.]+)\.0\.\d+$', sv2)
		if m: sv2 = m.group(1)
		m = re.match(r'^(.*-[^.]+\.[^.]+)\.\d+$', sv2)
		if m: sv2 = m.group(1)

		return (sv1 == sv2)
	else:
		m = re.match(r'^([^-]+)\.0\.\d+$', sv1)
		if m and sv2 == m.group(1): return 1

		m = re.match(r'^([^-]+)\.0\.\d+$', sv2)
		if m and sv1 == m.group(1): return 1

		return 0

def read_approvals(dir, approver, approved):
	f = open("%s/%s" % (dir, approver))
	line = f.readline()
	while line:
		l = string.split(line)
		if len(l) == 2:
			[pkg,ver] = l
			approved["%s_%s" % (pkg, ver)] = approver
		line = f.readline()
	f.close()

def read_bugs(file):
	bugsperpkg = {}

	f = open(file)
	line = f.readline()
	while line:
		l = string.split(line)
		if len(l) == 2:
			bugsperpkg[l[0]] = string.atoi(l[1])
		line = f.readline()
	f.close()
	return bugsperpkg

def write_bugs(file, bugs):
	f = open(file, 'w')
	pkgs = bugs.keys()
	pkgs.sort()
	for pkg in pkgs:
		if bugs[pkg] == 0: continue
		f.write("%s %d\n" % (pkg, bugs[pkg]))
	f.close()

def read_dates(file):
	dates = {}

	f = open(file)
	line = f.readline()
	while line:
		l = string.split(line)
		if len(l) == 3:
			dates[l[0]] = (l[1], string.atoi(l[2]))
		line = f.readline()
	f.close()
	return dates

def write_dates(file, dates):
	f = open(file, 'w')
	pkgs = dates.keys()
	pkgs.sort()
	for pkg in dates.keys():
		f.write("%s %s %d\n" % ((pkg,) + dates[pkg]))
	f.close()

def read_urgencies(file, testing, unstable):
	urgency = {}

	f = open(file)
	line = f.readline()
	while line:
		l = string.split(line)
		if len(l) == 3:
			uo = urgency.get(l[0], defaulturgency)
			mo = mindays.get(uo, mindays[defaulturgency])
			mn = mindays.get(l[2], mindays[defaulturgency])
			if mo <= mn: 
				line = f.readline()
				continue

			tsrcv = testing.get_version(l[0])
			if tsrcv and britney.versioncmp(tsrcv, l[1]) >= 0:
				line = f.readline()
				continue
			usrcv = unstable.get_version(l[0])
			if not usrcv or britney.versioncmp(usrcv, l[1]) < 0:
				line = f.readline()
				continue
			
			urgency[l[0]] = l[2]

		line = f.readline()
	f.close()
	return urgency

def read_hints(dir, hinter, hints, allowed):
	res = {}
	for k in allowed:
		res[k] = []

	try:
		f = open("%s/%s" % (dir, hinter))
	except IOError:
		return res

	while 1:
		line = f.readline()
		if not line: break

		l = string.split(line)
		if len(l) == 0 or line[0] == "#": 
			continue

		type = l[0]

		if type == "finished":
			break

		if type not in allowed:
			continue

		def mysplit(str):
			x = str.rfind("/")
			if x == -1: return [str]
			return [str[:x], str[x+1:]]

		if type in ["easy", "hint", "force-hint"]:
			l = [ tuple(mysplit(y)) for y in l[1:] ]
			l = [ k for k in l if len(k) == 2 ]
			res[type].append((hinter, l))

		if type in ["block-all"]:
			l = [ (y, hinter) for y in l[1:] ]
			res[type].extend(l)

		if type in ["block"]:
			l = [ (y, hinter) for y in l[1:] ]
			res[type].extend(l)

		if type in ["remove", "approve", "unblock", "force", "urgent"]:
			l = [ tuple(mysplit(y)+[hinter]) for y in l[1:] ]
			l = [ k for k in l if len(k) == 3 ]
			l = [ (p, (v,h)) for (p,v,h) in l ]
			res[type].extend(l)

	f.close()
	return res

class Excuse:
	reemail = re.compile(r"<.*?>")

	def __init__(self, name):
		self.name = name
		self.ver = ("-", "-")
		self.maint = None
		self.pri = None
		self.date = None
		self.urgency = None
		self.daysold = None
		self.mindays = None
		self.section = None
		self.dontinvalidate = 0

		self.invalid_deps = []
		self.deps = []
		self.break_deps = []
		self.bugs = []
		self.htmlline = []

	def set_vers(self, tver, uver):
		if tver: self.ver = (tver, self.ver[1])
		if uver: self.ver = (self.ver[0], uver)

	def set_maint(self, maint):
		self.maint = self.reemail.sub("",maint)
#		self.maint = maint

	def set_section(self, section):
		self.section = section

	def set_priority(self, pri):
		self.pri = pri

	def set_date(self, date):
		self.date = date

	def set_urgency(self, date):
		self.urgency = date

	def add_dep(self, name):
		if name not in self.deps: self.deps.append(name)
	def add_break_dep(self, name, arch):
		if (name, arch) not in self.break_deps:
			self.break_deps.append( (name, arch) )

	def invalidate_dep(self, name):
		if name not in self.invalid_deps: self.invalid_deps.append(name)

	def setdaysold(self, daysold, mindays):
		self.daysold = daysold
		self.mindays = mindays

	def addhtml(self, note):
		self.htmlline.append(note)

	def html(self):
		res = "<a id=\"%s\" name=\"%s\">%s</a> (%s to %s)\n<ul>\n" % \
			(self.name, self.name, self.name, self.ver[0], self.ver[1])
		if self.maint:
			res = res + "<li>Maintainer: %s\n" % (self.maint)
		if self.section and string.find(self.section, "/") > -1:
			res = res + "<li>Section: %s\n" % (self.section)
		if self.daysold != None:
			if self.daysold < self.mindays:
			    res = res + ("<li>Too young, only %d of %d days old\n" %
				(self.daysold, self.mindays))
			else:
			    res = res + ("<li>%d days old (needed %d days)\n" %
				(self.daysold, self.mindays))
		for x in self.htmlline:
			res = res + "<li>" + x + "\n"
		for x in self.deps:
			if x in self.invalid_deps:
				res = res + "<li>Depends: %s <a href=\"#%s\">%s</a> (not considered)\n" % (self.name, x, x)
			else:
				res = res + "<li>Depends: %s <a href=\"#%s\">%s</a>\n" % (self.name, x, x)
		for (n,a) in self.break_deps:
			if n not in self.deps:
				res += "<li>Ignoring %s depends: <a href=\"#%s\">%s</a>\n" % (a, n, n)
		res = res + "</ul>\n"
		return res
		


def should_remove_source(src, orig, new, excs):
	if new.is_present(src): return 0

	okay = 1

	exc = Excuse("-" + src)

	exc.set_vers(orig.get_version(src), None)
	m = orig.get_field(src, "Maintainer")
	if m: exc.set_maint(string.strip(m))
	s = orig.get_field(src, "Section")
	if s: exc.set_section(string.strip(s))

	if hints["block"].has_key("-" + src):
		blocked = hints["block"]["-" + src]
		exc.addhtml("Not touching package, as requested by %s (contact debian-release if update is needed)" % (blocked))
		okay = 0

	if okay:
		exc.addhtml("Valid candidate")
	else:
		exc.addhtml("Not considered")

	excs.append(exc)

	return okay

def should_upgrade_srcarch(src, arch, suite, tsrcv, orig, opkgsa, new, npkgsa, excs):
	# binnmu this arch?
	anywrongver = 0
	anyworthdoing = 0

	ref = "%s/%s" % (src, arch)
	if suite: ref = ref + "_%s" % (suite)

	e = Excuse(ref)
	e.set_vers(tsrcv, tsrcv)
	m = new.get_field(src, "Maintainer")
	if m: e.set_maint(string.strip(m))
	s = new.get_field(src, "Section")
	if s: e.set_section(string.strip(s))

	if hints["remove"].has_key(src):
		if same_source(tsrcv, hints["remove"][src][0]):
			e.addhtml("Removal request by %s" %
				(hints["remove"][src][1]))
			e.addhtml("Trying to remove package, not update it")
			e.addhtml("Not considered")
			excs.append(e)
			return 0

	for pkg in new.binaries(src, arch):
		pkgv = npkgsa.get_version(pkg)
		pkgsv = npkgsa.get_sourcever(pkg)

		if npkgsa.is_arch_all(pkg):
			e.addhtml("Ignoring %s %s (from %s) as it is arch: all"
				% (pkg, pkgv, pkgsv))
			continue

		if not same_source(tsrcv, pkgsv):
			anywrongver = 1
			e.addhtml("From wrong source: %s %s (%s not %s)" % (
				pkg, pkgv, pkgsv, tsrcv))
			break

		excuse_unsat_deps(pkg, arch, opkgsa, npkgsa, e, 0)

		if not opkgsa.is_present(pkg):
			e.addhtml("New binary: %s (%s)" % (pkg, pkgv))
			anyworthdoing = 1
			continue

		tpkgv = opkgsa.get_version(pkg)
		if britney.versioncmp(tpkgv, pkgv) > 0:
			anywrongver = 1
			e.addhtml("Not downgrading: %s (%s to %s)" % (
				pkg, tpkgv, pkgv))
			break
		elif britney.versioncmp(tpkgv, pkgv) < 0:
			e.addhtml("Updated binary: %s (%s to %s)" % (
				pkg, tpkgv, pkgv))
			anyworthdoing = 1

	if not anywrongver and (anyworthdoing or not new.is_fake(src)):
		srcv = new.get_version(src)
		ssrc = same_source(tsrcv, srcv)
		for pkg in orig.binaries(src, arch):
			if opkgsa.is_arch_all(pkg):
				e.addhtml("Ignoring removal of %s as it is arch: all"
					% (pkg))
				continue
			if not npkgsa.is_present(pkg):
				tpkgv = opkgsa.get_version(pkg)
				e.addhtml("Removed binary: %s %s" % (
					pkg, tpkgv))
				if ssrc: anyworthdoing = 1

	if not anywrongver and anyworthdoing:
		e.addhtml("Valid candidate")
		excs.append(e)
		return 1
	else:
		if anyworthdoing:
			e.addhtml("Not considered")
			excs.append(e)
		return 0	

def excuse_unsat_deps(pkg, arch, tpkgsarch, upkgsarch, exc, ignore_break=0):
	for d in ['Pre-Depends', 'Depends']:
		udt = tpkgsarch.unsatisfiable_deps(upkgsarch, pkg, d)
		udu = upkgsarch.unsatisfiable_deps(upkgsarch, pkg, d)

		for t,u in map(None, udt, udu):
			if t[1]: continue
			l = []
			for e in u[1]:
				s = upkgsarch.get_source(e)
				if s not in l: l.append(s)
			if src in l: continue
			if l == []:
				exc.addhtml("%s/%s unsatisfiable %s: %s" % (pkg, arch, d, t[0]))
			for s in l:
				if ignore_break or arch not in breakarches:
					exc.add_dep(s)
				else:
					exc.add_break_dep(s, arch)

def should_upgrade_src(src, suite, orig, origpkgs, new, newpkgs, approvals, 
                       excs):
	srcv = new.get_version(src)

	if orig.is_present(src):
		tsrcv = orig.get_version(src)
		if britney.versioncmp(srcv, tsrcv) == 0:
			# Candidate for binnmus only	
			return 0
	else:
		tsrcv = None

	updatecand = 1

	ref = src
	if suite: ref = ref + "_tpu"

	exc = Excuse(ref)
	exc.set_vers(tsrcv, srcv)
	m = new.get_field(src, "Maintainer")
	if m: exc.set_maint(string.strip(m))
	s = new.get_field(src, "Section")
	if s: exc.set_section(string.strip(s))

	if tsrcv and britney.versioncmp(srcv, tsrcv) < 0:
		# Version in unstable is older!
		exc.addhtml("ALERT: %s is newer in testing (%s %s)" % (src, tsrcv, srcv))
		excs.append(exc)
		return 0

	if unstable.is_fake(src):
		exc.addhtml("%s source package doesn't exist" % (src))
		updatecand = 0

	urgency = unstableurg.get(src, defaulturgency)
	if not tsrcv and urgency != defaulturgency:
		exc.addhtml("Ignoring %s urgency setting for NEW package" % (urgency))
		urgency = defaulturgency

        if hints["remove"].has_key(src):
                if (tsrcv and same_source(tsrcv, hints["remove"][src][0])) or \
		  same_source(srcv, hints["remove"][src][0]):
                        exc.addhtml("Removal request by %s" %
                                (hints["remove"][src][1]))
                        exc.addhtml("Trying to remove package, not update it")
			updatecand = 0

	blocked = None
	if hints["block"].has_key(src):
		blocked = hints["block"][src]
	elif hints["block-all"].has_key("source"):
		blocked = hints["block-all"]["source"]

	if blocked:
		ubv = hints["unblock"].get(src,(None,None))[0]
		if ubv != None and same_source(ubv, srcv):
			exc.addhtml("Ignoring request to block package by %s, due to unblock request by %s" % (blocked, hints["unblock"][src][1]))
		else:
			if ubv != None:
				exc.addhtml("Unblock request by %s ignored due to version mismatch: %s" % (hints["unblock"][src][1], hints["unblock"][src][0]))
			exc.addhtml("Not touching package, as requested by %s (contact debian-release if update is needed)" % (blocked))
			updatecand = 0

	if suite == None:
		if not unstabledates.has_key(src):
			unstabledates[src] = (srcv, datenow)
		elif not same_source(unstabledates[src][0], srcv):
			unstabledates[src] = (srcv, datenow)

		daysold = datenow - unstabledates[src][1]
		mymindays = mindays[urgency]
		exc.setdaysold(daysold, mymindays)
		if daysold < mymindays:
			if hints["urgent"].has_key(src) and same_source(srcv, hints["urgent"][src][0]):
				exc.addhtml("Too young, but urgency pushed by %s" % (hints["urgent"][src][1]))
			else:
				updatecand = 0

	pkgs = { src: ["source"] }
	anybins = 0
	for arch in arches:
		oodbins = {}
		for pkg in new.binaries(src,arch):
			anybins = 1
			if not pkgs.has_key(pkg): pkgs[pkg] = []
			pkgs[pkg].append(arch)

			pkgsv = newpkgs[arch].get_sourcever(pkg)
			if not same_source(srcv, pkgsv):
				if not oodbins.has_key(pkgsv):
					oodbins[pkgsv] = []
				oodbins[pkgsv].append(pkg)
				continue

			if newpkgs[arch].isnt_arch_all(pkg) or \
			  arch in nobreakarchallarches:
				excuse_unsat_deps(pkg, arch, 
					origpkgs[arch], newpkgs[arch], exc)

		if oodbins:
			oodtxt = ""
			for v in oodbins.keys():
				if oodtxt: oodtxt = oodtxt + "; "
				oodtxt = oodtxt + "%s (from <a href=\"http://buildd.debian.org/status/logs.php?arch=%s&pkg=%s&ver=%s\" target=\"_blank\">%s</a>)" % \
					(", ".join(oodbins[v]), arch, src, v, v)
			text = "out of date on <a href=\"http://buildd.debian.org/status/logs.php?arch=%s&pkg=%s&ver=%s\" target=\"_blank\">%s</a>: %s" % (arch, src, srcv, arch, oodtxt)

			if arch in fuckedarches:
				text = text + " (but %s isn't keeping up," % \
					(arch) + " so nevermind)"
			else:
				updatecand = 0

			if datenow != unstabledates[src][1]:
				exc.addhtml(text)

	if not anybins:
		exc.addhtml("%s has no binaries on any arch" % src)
		updatecand = 0

	if suite == None:
		for pkg in pkgs.keys():
			if not testingbugs.has_key(pkg): testingbugs[pkg] = 0
			if not unstablebugs.has_key(pkg): unstablebugs[pkg] = 0

			if unstablebugs[pkg] > testingbugs[pkg]:
				exc.addhtml("%s (%s) is <a href=\"http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=pkg&data=%s&sev-inc=critical&sev-inc=grave&sev-inc=serious\" target=\"_blank\">buggy</a>! (%d > %d)" % \
					(pkg, ", ".join(pkgs[pkg]), pkg,
					unstablebugs[pkg], testingbugs[pkg]))
				updatecand = 0
			elif unstablebugs[pkg] > 0:
				exc.addhtml("%s (%s) is (less) <a href=\"http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=pkg&data=%s&sev-inc=critical&sev-inc=grave&sev-inc=serious\" target=\"_blank\">buggy</a>! (%d <= %d)" % \
					(pkg, ", ".join(pkgs[pkg]), pkg,
					unstablebugs[pkg], testingbugs[pkg]))

	if not updatecand and hints["force"].has_key(src) and same_source(srcv, hints["force"][src][0]) :
		exc.dontinvalidate = 1
		exc.addhtml("Should ignore, but forced by %s" % (hints["force"][src][1]))
		updatecand = 1

	if approvals:
		if approvals.has_key("%s_%s" % (src, srcv)):
			exc.addhtml("Approved by %s" % 
			              approvals["%s_%s" % (src, srcv)])
		else:
			exc.addhtml("NEEDS APPROVAL BY RM")
			updatecand = 0

	if updatecand:
		exc.addhtml("Valid candidate")
	else:
		exc.addhtml("Not considered")

	excuses.append(exc)

	return updatecand

###

# Brute force stuff

class UpgradeRun:
	def __init__(self, sn, u, tu, ps):
		self.srcsn = sn
		self.unstable = u
		self.testingupdates = tu
		self.packages = ps

		self.sortpkgs()

		self.output = open("update.OUTPUT_py", "w");
		
		self.arches = [ x for x in arches if x in srcsn.arches ]
		self.srcsnpkgs = {}
		for arch in arches:
			self.srcsnpkgs[arch] = self.srcsn.Packages(arch)

	#def __del__():
	#	self.output.close()

	def sortpkgs(self):
		p = self.packages
		p.sort()
		self.packages = p

	def writeout(self, text):
		self.output.write(text)
		sys.stdout.write(text)
		self.output.flush()
		sys.stdout.flush()

	def doop_source(self, op):
		# removals = "-<source>", 
		# arch = "<source>/<arch>", 
		# normal = "<source>"
		which = self.unstable
		if "_" in op:
			ind = string.index(op, "_")
			if op[ind+1:] == "tpu":
				which = self.testingupdates
			op = op[:ind]

		if op[0] == "-":
			self.srcsn.remove_source(op[1:])
		elif "/" in op:
			ind = string.index(op, "/")
			self.srcsn.upgrade_arch(which, op[:ind], op[ind+1:])
		else:
			self.srcsn.upgrade_source(which, op)

	def get_nuninst(self):
		nuninst = {}
		for arch in self.arches:
			con = self.srcsnpkgs[arch].packages
			if arch not in nobreakarchallarches:
				con = filter(
					self.srcsnpkgs[arch].isnt_arch_all,
					con)
			nuninst[arch] = filter(
				self.srcsnpkgs[arch].is_uninstallable,
				con)
		return nuninst

	def get_improved_nuninst(self, old):
		new = {}
		for arch in self.arches:
		        con = self.srcsnpkgs[arch].packages
			if arch not in nobreakarchallarches:
				con = filter(
					self.srcsnpkgs[arch].isnt_arch_all,
					con)
			new[arch] = filter(
				self.srcsnpkgs[arch].is_uninstallable, con)
			if arch in breakarches: continue
			if len(new[arch]) > len(old[arch]):
				return (0, new)
		return (1, new)

	def arch_improved_nuninst(self, old, arch):
		new = old.copy()
		if "_" in arch: arch = arch[:arch.index("_")]
		con = self.srcsnpkgs[arch].packages
		if arch not in nobreakarchallarches:
			con = filter(self.srcsnpkgs[arch].isnt_arch_all, con)
		new[arch] = filter(self.srcsnpkgs[arch].is_uninstallable, con)
		if arch not in newarches and len(new[arch]) > len(old[arch]):
			return (0, new)
		return (1, new)

	def is_nuninst_asgood(self, old, new):
		for arch in self.arches:
			if arch in breakarches: continue
			if len(new[arch]) > len(old[arch]):
				return 0
		return 1

	def is_nuninst_asgood_generous(self, old, new):
		diff = 0
		for arch in self.arches:
			if arch in breakarches: continue
			diff = diff + (len(new[arch]) - len(old[arch]))
		return diff <= 0

	def eval_nuninst(self, nuninst):
		res = []
		total = 0
		totalbreak = 0
		for arch in self.arches:
			if nuninst.has_key(arch):
				n = len(nuninst[arch])
				if arch in breakarches:
					totalbreak = totalbreak + n
				else:
					total = total + n
				res.append("%s-%d" % (arch[0], n))
		return "%d+%d: %s" % (total, totalbreak, ":".join(res))

	def slist_subtract(self, base, sub):
		res = []
		for x in base:
			if x not in sub: res.append(x)
		return res

	def newlyuninst(self, nuold, nunew):
		res = {}
		for arch in self.arches:
			if not nuold.has_key(arch) or not nunew.has_key(arch):
				continue
			res[arch] = \
				self.slist_subtract(nunew[arch], nuold[arch])
		return res

	def eval_uninst(self, nuninst):
		res = ""
		for arch in self.arches:
			if nuninst.has_key(arch) and nuninst[arch] != []:
				res = res + "    * %s: %s\n" % (arch, 
					", ".join(nuninst[arch]))
		return res

	def do_all(self, maxdepth = 0, init = []):
		self.selected = []
		self.selected_committed = 0
		packages = self.packages[:]

		earlyabort = 0
		if maxdepth == "easy":
			earlyabort = 1
			maxdepth = 0

		# meaningless to try forcing something _and_ recurse
		force = 0
		if maxdepth < 0:
			force = 1
			maxdepth = 0
			earlyabort = 1

		nuninst_start = self.get_nuninst()

		if init:
			self.writeout("leading: %s\n" % (",".join(init)))
			
		for x in init:
			if x not in packages:
				self.writeout("failed: %s\n" % (x))
				return None
			y = packages.index(x)
			self.selected.append(packages.pop(y))

		for x in init:
			self.doop_source(x)
	
		if force:
			self.nuninst_orig = self.get_nuninst()
		else:
			self.nuninst_orig = nuninst_start

		self.writeout("start: %s\n" %
			self.eval_nuninst(nuninst_start))
		self.writeout("orig: %s\n" %
			self.eval_nuninst(self.nuninst_orig))

		if earlyabort:
			nuninst_end = self.get_nuninst()
		        self.writeout("easy: %s\n" %
		   	    (self.eval_nuninst(nuninst_end)))
		        self.writeout(self.eval_uninst(
			    self.newlyuninst(self.nuninst_orig, nuninst_end)))
		        self.writeout("\n")
			if not self.is_nuninst_asgood_generous(
							self.nuninst_orig, 
							nuninst_end):
				nuninst_end, respackages = None, None
			else:
				respackages = packages[:]
				self.selected_committed = len(self.selected)
		else:
			nuninst_end, respackages = \
				self.iter_some(maxdepth, packages, [])

		if nuninst_end:
			assert(len(self.selected) == self.selected_committed)

			self.writeout("final: %s\n" % 
				",".join(self.selected))
			self.writeout("start: %s\n" %
				self.eval_nuninst(nuninst_start))
			self.writeout(" orig: %s\n" %
				self.eval_nuninst(self.nuninst_orig))
			self.writeout("  end: %s\n" %
				self.eval_nuninst(nuninst_end))

			if force:
		            self.writeout("force breaks:\n")
		            self.writeout(self.eval_uninst(
			        self.newlyuninst(nuninst_start, nuninst_end)))
		            self.writeout("\n")

			if not self.is_nuninst_asgood_generous(
							self.nuninst_orig, 
							nuninst_end):
				print "NON-None RETURN THAT'S NOT BETTER"

			self.srcsn.commit_changes()

			self.writeout("SUCCESS (%d/%d)\n" % 
				(len(self.packages), len(respackages)))
			self.packages = respackages
			self.sortpkgs()

			return self.selected

		else:
			assert(len(self.selected) == len(init))
			assert(self.selected_committed == 0)

			for x in init:
				self.srcsn.undo_change()
			if self.srcsn.can_undo:
				print "MORE OPS LEFT TO UNDO THAN DONE"

			self.writeout("FAILED\n")
			return None

	def iter_end(self, available):
		extra = []
		count = 0
		nuninst_comp = self.get_nuninst()
		while available:
			x = available.pop(0)
			self.writeout("trying: %s\n" % (x))

			self.doop_source(x)

			if "/" in x:
				better, nuninst_new = self.arch_improved_nuninst(
					nuninst_comp, x[x.index("/")+1:])
			else:
				better, nuninst_new = self.get_improved_nuninst(
					nuninst_comp)

			if better:
				self.selected.append(x)
				count = count + 1
				available.extend(extra)
				extra = []

				self.writeout("accepted: %s\n" % (x))
				self.writeout("   ori: %s\n" % 
					(self.eval_nuninst(self.nuninst_orig)))
				self.writeout("   pre: %s\n" % 
					(self.eval_nuninst(nuninst_comp)))
				self.writeout("   now: %s\n" %
					(self.eval_nuninst(nuninst_new)))
				if len(self.selected) <= 20:
					self.writeout("   all: %s\n" % (
						" ".join(self.selected)))
				else:
					self.writeout("  most: (%d) .. %s\n" %
					  (len(self.selected),
					  " ".join(self.selected[-20:])))

				nuninst_comp = nuninst_new
			else:
				self.writeout("skipped: %s (%d <- %d)\n" % (
					x, len(extra), len(available)))
				self.writeout("    got: %s\n%s" % (
					self.eval_nuninst(nuninst_new),
					self.eval_uninst(self.newlyuninst(
					  nuninst_comp, nuninst_new))))

				self.srcsn.undo_change()
				extra.append(x)
		self.writeout(" finish: [%s]\n" %
		    ",".join(self.selected[self.selected_committed:]))
		self.writeout("endloop: %s\n" % 
			(self.eval_nuninst(self.nuninst_orig)))
		self.writeout("    now: %s\n" %
			(self.eval_nuninst(nuninst_comp)))
		self.writeout(self.eval_uninst(
			self.newlyuninst(self.nuninst_orig, nuninst_comp)))
		self.writeout("\n")

		if self.is_nuninst_asgood_generous(self.nuninst_orig,
		  nuninst_comp):
			self.writeout("Apparently successful\n")
			self.selected_committed = len(self.selected)
			return (nuninst_comp, extra)

		while count > 0:
			self.srcsn.undo_change()
			self.selected.pop()
			count = count - 1

		return (None, None)

	def iter_some(self, depth, available, extra):
		self.writeout("recur: [%s] %s %d/%d\n" % (
		    ",".join(self.selected[:self.selected_committed]), 
		    ",".join(self.selected[self.selected_committed:]), 
		    len(available), len(extra)))

		if depth == 0:
			extra.extend(available)
			return self.iter_end(extra)

		nuninst = None
		
		while len(available) > depth:
			x = available.pop(0)

			if not skiphint(x):
				self.doop_source(x)
				self.selected.append(x)

				res = self.iter_some(depth - 1, available[:], extra[:])
				if res[0]:
					nuninst = res[0]
					available = filter(lambda x, y=res[1]: x in y,
						available + extra)
					# reset nuninst_orig too
					self.nuninst_orig = nuninst
					extra = []
					continue

				self.srcsn.undo_change()
				self.selected.pop()

			extra.append(x)

		return (nuninst, extra)

# Package information

testing = britney.Sources(testingdir, arches)
testingpkgs = {}
for arch in arches:
	testingpkgs[arch] = testing.Packages(arch)
testingbugs = read_bugs(testingdir + "/Bugs")

unstable = britney.Sources(unstabledir, arches)
unstablepkgs = {}
for arch in arches:
	unstablepkgs[arch] = unstable.Packages(arch)
unstablebugs = read_bugs(unstabledir + "/Bugs")
unstabledates = read_dates(testingdir + "/Dates")
unstableurg = read_urgencies(testingdir + "/Urgency", testing, unstable)

testingupdates = britney.Sources(testingupdatesdir, arches)
testingupdatespkgs = {}
for arch in arches:
	testingupdatespkgs[arch] = testingupdates.Packages(arch)
testingupdatesapproved = {} # pkg_ver -> who
for approver in ["ajt", "security-team", "ftpmaster", "cjwatson", "vorlon"]:
	read_approvals(testingupdatesdir + "/Approved", approver,
		testingupdatesapproved)

hlphints = ["easy", "hint", "remove", "block", "unblock", "approve"]
stdhints = ["easy", "hint", "remove", "block", "unblock", "urgent", "approve"]
allhints = ["force", "force-hint", "block-all"] + stdhints
hintsallowed = {
	"ajt": allhints,
	"rmurray": allhints,
	"cjwatson": allhints,
	"vorlon": allhints,
	"aba": allhints,

	"joeyh": stdhints + ["force"],
	"djpig": stdhints,

	"he": hlphints,
	"adeodato": hlphints,
	"ballombe": hlphints,
	"luk": hlphints,

	"freeze": ["block", "block-all"],
	"ftpteam": ["block"]
}

hints = {"easy":[], "hint":[], "force-hint":[], "remove":[], "block":[], "block-all":[], "unblock":[], "force":[], "urgent":[], "approve":[]}
for who in hintsallowed.keys():
	h = read_hints(unstabledir + "/Hints", who, hints, hintsallowed[who])
	for k in hintsallowed[who]:
		hints[k].extend(h[k])

for x in ["block", "block-all", "unblock", "force", "urgent", "remove"]:
	z = {}
	for a, b in hints[x]:
		if z.has_key(a):
			print "Overriding %s[%s] = %s with %s" % (x, a, z[a], b)
		z[a] = b
	hints[x] = z

for p, vh in hints["approve"]:
	(v,h) = vh
	testingupdatesapproved["%s_%s" % (p,v)] = h
	hints["unblock"]["%s" % p] = (v,h)

def maxver(pkg, source, pkgs):
	maxver = source.get_version(pkg)
	for arch in arches:
		pkgv = pkgs[arch].get_version(pkg)
		if pkgv == None: continue
		if maxver == None or britney.versioncmp(pkgv, maxver) > 0:
			maxver = pkgv
	return maxver

for pkg in testingbugs.keys() + unstablebugs.keys():
	if not testingbugs.has_key(pkg): testingbugs[pkg] = 0
	if not unstablebugs.has_key(pkg): unstablebugs[pkg] = 0

	maxvert = maxver(pkg, testing, testingpkgs)
	if maxvert == None: 
		testingbugs[pkg] = 0
		continue

	if testingbugs[pkg] == unstablebugs[pkg]: continue
	maxveru = maxver(pkg, unstable, unstablepkgs)
	
	if maxveru == None: 
		continue
	if britney.versioncmp(maxvert, maxveru) >= 0:
		testingbugs[pkg] = unstablebugs[pkg]

datenow = int(((time.time() / (60*60)) - 15) / 24);

# Next, work out which packages are candidates to be changed.

upgrademe = []
excuses = []

# Packages to be removed
for src in testing.sources:
	if should_remove_source(src, testing, unstable,  excuses):
		upgrademe.append("-" + src)

# Packages to be upgraded from unstable:
for src in unstable.sources:
	if testing.is_present(src):
		tsrcv = testing.get_version(src) # silly optimisation
		for arch in arches:
			if should_upgrade_srcarch(src, arch, None, tsrcv, 
			                          testing, testingpkgs[arch], 
			                          unstable, unstablepkgs[arch],
						  excuses):
				upgrademe.append("%s/%s" % (src, arch))

	if should_upgrade_src(src, None, testing, testingpkgs, 
	                      unstable, unstablepkgs, None, excuses):
		upgrademe.append(src)

for src in testingupdates.sources:
	if testing.is_present(src):
		tsrcv = testing.get_version(src) # silly optimisation
		for arch in arches:
			if should_upgrade_srcarch(src, arch, "tpu", tsrcv, 
			                          testing, testingpkgs[arch], 
			                          testingupdates,
						   testingupdatespkgs[arch],
						  excuses):
				upgrademe.append("%s/%s_tpu" % (src, arch))

	if should_upgrade_src(src, "tpu", testing, testingpkgs,
	                      testingupdates, testingupdatespkgs, 
	                      testingupdatesapproved, excuses):
		upgrademe.append("%s_tpu" % src)

for src in hints["remove"].keys():
	if src in upgrademe: continue
	if ("-"+src) in upgrademe: continue
	if not testing.is_present(src): continue

	tsrcv = testing.get_version(src)
	if not same_source(tsrcv, hints["remove"][src][0]): continue

	upgrademe.append("-%s" % (src))
	exc = Excuse("-%s" % (src))
	exc.set_vers(tsrcv, None)
	exc.addhtml("Removal request by %s" % (hints["remove"][src][1]))
	exc.addhtml("Package is broken, will try to remove")
	excuses.append(exc)

def cmpexcuses(el, er):
	return cmp(el.daysold, er.daysold) or cmp(el.name, er.name)
excuses.sort(cmpexcuses)

def reversed_exc_deps(excuses):
	res = {}
	for exc in excuses:
		for d in exc.deps:
			if not res.has_key(d): res[d] = []
			res[d].append(exc.name)
	return res

def invalidate(excuses, valid, invalid):
	i = 0
	exclookup = {}
	for e in excuses:
		exclookup[e.name] = e
	revdeps = reversed_exc_deps(excuses)
	while i < len(invalid):
		if not revdeps.has_key(invalid[i]): 
			i += 1
			continue
		if (invalid[i] + "_tpu") in valid:
			i += 1
			continue
		for x in revdeps[invalid[i]]:
			if x in valid and exclookup[x].dontinvalidate:
				continue

			exclookup[x].invalidate_dep(invalid[i])
			if x in valid:
				p = valid.index(x)
				invalid.append(valid.pop(p))
				exclookup[x].addhtml("Invalidated by dependency")
				exclookup[x].addhtml("Not considered")
		i = i + 1

unconsidered = []
for exc in excuses:
	if exc.name not in upgrademe: unconsidered.append(exc.name)

for exc in excuses:
	for d in exc.deps:
		if d not in upgrademe and d not in unconsidered:
			exc.addhtml("Unpossible dep: %s -> %s" % (exc.name, d))

invalidate(excuses, upgrademe, unconsidered)

f = open("update.EXCUSES_py", 'w')

f.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n")
f.write("<html><head><title>excuses...</title>")
f.write("<meta http-equiv=\"Content-Type\" content=\"text/html;charset=utf-8\"></head><body>\n")
f.write("<p>Generated: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "</p>\n")
f.write("<ul>\n")

for exc in excuses:
	f.write("<li>%s" % exc.html())
f.write("</ul></body></html>\n")

f.close()

del excuses

# Changes

srcsn = britney.SourcesNote(arches)

# Initialise new testing to be the old testing

for src in testing.sources:
	srcsn.upgrade_source(testing, src)

srcsn.commit_changes()

#print "Things to do:"
#for x in upgrademe:
#	print "        " + x

run = UpgradeRun(srcsn, unstable, testingupdates, upgrademe)

def skiphint(candidate):
	if "/" in candidate and candidate[candidate.rfind("/")+1:] in breakarches:
		return 1
	return 0

def do_hint(type, who, pkgvers):
	hintinfo = {"easy":       "easy",
		    "hint":       0,
        	    "force-hint": -1,
		   }
	hintdoall = hintinfo[type]

	run.writeout("Trying %s from %s: %s\n" % (type, who, 
		" ".join( ["%s/%s" % (p,v) for (p,v) in pkgvers] )))

	ok = 1
	for xp,v in pkgvers:
		# is this version of this package present in unstable?
		# (if it's also present in testing, do_all will skip it)
		if "/" in xp:
			p = xp[:xp.find("/")]
		else:
			p = xp

		if p[0] == "-":
			pass
		elif p.endswith("_tpu"):
			if britney.versioncmp(run.testingupdates.get_version(p[:-4]),v) != 0:
				ok = 0
				run.writeout(" Version mismatch, %s %s != %s\n" %
					     (p, v, run.testingupdates.get_version(p[:-4])))
		elif run.unstable.get_version(p) == None:
			ok = 0
			run.writeout(" Source %s has no version in unstable\n" % p)
		elif britney.versioncmp(run.unstable.get_version(p), v) != 0:
			ok = 0
			run.writeout(" Version mismatch, %s %s != %s\n" %
				(p, v, run.unstable.get_version(p)))
	if ok:
		run.do_all(hintdoall, map(lambda hp: hp[0], x[1]))
	else:
		run.writeout("Not using hint\n")
	return ok

run.writeout("Generated on: %s\n" % (time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time()))))
run.writeout("Arch order is: %s\n" % ", ".join(arches))

for x in hints["easy"]:
	do_hint("easy", x[0], x[1])
for x in hints["force-hint"]:
	do_hint("force-hint", x[0], x[1])

allpackages = []
normpackages = run.packages[:]
archpackages = {}
for a in breakarches:
    la = len(a) + 1
    archpackages[a] = [ p for p in normpackages if p[-la:] == "/" + a ]
    normpackages = [ p for p in normpackages if p[-la:] != "/" + a ]
run.packages = normpackages
run.writeout("info: main run\n");
run.do_all()
allpackages += run.packages
for a in breakarches:
    x = breakarches
    breakarches = [ ba for ba in breakarches if ba != a ]
    run.packages = archpackages[a]
    run.writeout("info: broken arch run for %s\n" % (a))
    run.do_all()
    #run.do_all(1)
    breakarches = x
    allpackages += run.packages
run.packages = allpackages

#run.do_all(0,["caudium", "sablotron"])

hintcnt = 1

rand = whrandom.whrandom()
rand.seed(23,187,96)
for q in range(datenow): 
	rand.random()
q = rand.random()
q = 1.0
run.writeout("Current value is %f\n" % (q))
if q < 0.2:
	q = 0.2
	run.writeout("Current value bumped to %f\n" % (q))
maxloops = int(math.ceil(math.log(100/(q**0.5)) / math.log(1+len(run.packages))))
maxloops = 1
run.writeout("Max loops for q=%.2f is %d\n" % (q, maxloops))

for x in hints["hint"]:
	if hintcnt > 50:
		run.writeout("Skipping remaining hints...")
		break

	if len(x[1]) < maxloops:
		run.writeout("Skipping simple hint from %s (%d<%d): %s\n" 
			% (x[0], len(x[1]), maxloops, str(x[1])))
		continue

	if do_hint("hint", x[0], x[1]):
		hintcnt += 1

for i in range(1,maxloops):
	run.do_all(i)

if maxloops <= 1 and len(run.packages) < 500:
	# too many to do all of them, let's try 5 at random
	num_at_random = 5
	if len(run.packages) > num_at_random:
		run.writeout("Trying %d at random\n" % num_at_random)
		for k in range(num_at_random):
			special = rand.choice(run.packages)
			if skiphint(special): continue
			run.writeout("Randomly trying %s\n" % (special))
			run.do_all(0, [special])

run.srcsn.write_notes(testingdir)

write_bugs(testingdir + "/Bugs", testingbugs)
write_dates(testingdir + "/Dates", unstabledates)

f = open(testingdir + '/HeidiResult', 'w')

for arch in arches:
	pkgs = srcsn.Packages(arch)
	for pkg in pkgs.packages:
		pkgv = pkgs.get_version(pkg)
		pkgarch = pkgs.get_field(pkg, 'Architecture')
		pkgsec = pkgs.get_field(pkg, 'Section')
		if pkgsec == None: pkgsec = 'unknown\n'
		pkgarch = pkgarch[:-1]
		pkgsec = pkgsec[:-1]
		f.write('%s %s %s %s\n' % (pkg, pkgv, pkgarch, pkgsec))

for src in srcsn.sources:
	srcv = srcsn.get_version(src)
	srcsec = srcsn.get_field(src, 'Section')
	if srcsec == None: srcsec = 'unknown\n'
	if srcsn.is_fake(src): srcsec = 'faux\n'
	srcsec = srcsec[:-1]
	f.write('%s %s source %s\n' % (src, srcv, srcsec))

f.close()

if len(arches) != expected_arches:
	sys.exit(1)