Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-03-12 03:41:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge with trunk, port to Python 3

Browse Source
This commit is contained in:
Martin Pitt 2015-08-24 20:46:42 +02:00
commit 32f33baf09
14 changed files with 568 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
autopkgtest.py
View File

@ -16,8 +16,6 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
from __future__ import print_function
import os
import time
import json
@ -25,7 +23,8 @@ import tarfile
import io
import copy
import itertools
from urllib import urlencode, urlopen
from urllib.parse import urlencode
from urllib.request import urlopen
import apt_pkg
import kombu
@ -209,7 +208,7 @@ class AutoPackageTest(object):
tests.append((rdep_src, rdep_src_info[VERSION]))
reported_pkgs.add(rdep_src)
tests.sort(key=lambda (s, v): s)
tests.sort(key=lambda s_v: s_v[0])
return tests
#
@ -321,7 +320,7 @@ class AutoPackageTest(object):
try:
f = urlopen(url)
if f.getcode() == 200:
result_paths = f.read().strip().splitlines()
result_paths = f.read().decode().strip().splitlines()
elif f.getcode() == 204: # No content
result_paths = []
else:
@ -417,9 +416,9 @@ class AutoPackageTest(object):
'''Return (src, arch) set for failed tests for given trigger pkg'''
result = set()
for src, srcinfo in self.test_results.iteritems():
for arch, (stamp, vermap, ever_passed) in srcinfo.iteritems():
for ver, (passed, triggers) in vermap.iteritems():
for src, srcinfo in self.test_results.items():
for arch, (stamp, vermap, ever_passed) in srcinfo.items():
for ver, (passed, triggers) in vermap.items():
if not passed:
# triggers might contain tuples or lists (after loading
# from json), so iterate/check manually
@ -491,7 +490,7 @@ class AutoPackageTest(object):
# update results from swift for all packages that we are waiting
# for, and remove pending tests that we have results for on all
# arches
for pkg, verinfo in copy.deepcopy(self.pending_tests.items()):
for pkg, verinfo in copy.deepcopy(self.pending_tests).items():
for archinfo in verinfo.values():
for arch in archinfo:
self.fetch_swift_results(self.britney.options.adt_swift_url, pkg, arch)

10
boottest.py
View File

@ -11,7 +11,7 @@
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
from __future__ import print_function
from collections import defaultdict
from contextlib import closing
@ -20,7 +20,7 @@ import subprocess
import tempfile
from textwrap import dedent
import time
import urllib
import urllib.request
import apt_pkg
@ -90,7 +90,7 @@ class TouchManifest(object):
print("I: [%s] - saving it to %s" %
(time.asctime(), self.path))
try:
response = urllib.urlopen(url)
response = urllib.request.urlopen(url)
if response.code == 200:
# Only [re]create the manifest file if one was successfully
# downloaded. This allows for an existing image to be used
@ -98,7 +98,7 @@ class TouchManifest(object):
path_dir = os.path.dirname(self.path)
if not os.path.exists(path_dir):
os.makedirs(path_dir)
with open(self.path, 'w') as fp:
with open(self.path, 'wb') as fp:
fp.write(response.read())
success = True
break
@ -266,7 +266,7 @@ class BootTest(object):
if not self.britney.options.verbose:
return
for src in sorted(self.pkglist):
for ver in sorted(self.pkglist[src], cmp=apt_pkg.version_compare):
for ver in sorted(self.pkglist[src]):
status = self.pkglist[src][ver]
print("I: [%s] - Collected boottest status for %s_%s: "
"%s" % (time.asctime(), src, ver, status))

524
britney.py
View File

File diff suppressed because it is too large Load Diff

49
britney_util.py
View File

@ -24,7 +24,7 @@
import apt_pkg
from functools import partial
from datetime import datetime
from itertools import chain, ifilter, ifilterfalse, izip, repeat
from itertools import chain, repeat, filterfalse
import os
import re
import time
@ -76,8 +76,8 @@ def ifilter_except(container, iterable=None):
iterators that are not known on beforehand.
"""
if iterable is not None:
return ifilterfalse(container.__contains__, iterable)
return partial(ifilterfalse, container.__contains__)
return filterfalse(container.__contains__, iterable)
return partial(filterfalse, container.__contains__)
def ifilter_only(container, iterable=None):
@ -89,8 +89,8 @@ def ifilter_only(container, iterable=None):
iterators that are not known on beforehand.
"""
if iterable is not None:
return ifilter(container.__contains__, iterable)
return partial(ifilter, container.__contains__)
return filter(container.__contains__, iterable)
return partial(filter, container.__contains__)
# iter_except is from the "itertools" recipe
@ -120,7 +120,7 @@ def iter_except(func, exception, first=None):
def undo_changes(lundo, inst_tester, sources, binaries,
BINARIES=BINARIES, PROVIDES=PROVIDES):
BINARIES=BINARIES):
"""Undoes one or more changes to testing
* lundo is a list of (undo, item)-tuples
@ -225,7 +225,7 @@ def register_reverses(packages, provides, check_doubles=True, iterator=None,
the loops.
"""
if iterator is None:
iterator = packages.iterkeys()
iterator = packages.keys()
else:
iterator = ifilter_only(packages, iterator)
@ -303,7 +303,7 @@ def compute_reverse_tree(packages_s, pkg, arch,
# generate the next iteration, which is the reverse-dependencies of
# the current iteration
rev_deps = set(revfilt(flatten( binaries[x][RDEPENDS] for x in binfilt(rev_deps) )))
return izip(seen, repeat(arch))
return zip(seen, repeat(arch))
def write_nuninst(filename, nuninst):
@ -312,7 +312,7 @@ def write_nuninst(filename, nuninst):
Write the non-installable report derived from "nuninst" to the
file denoted by "filename".
"""
with open(filename, 'w') as f:
with open(filename, 'w', encoding='utf-8') as f:
# Having two fields with (almost) identical dates seems a bit
# redundant.
f.write("Built on: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "\n")
@ -329,7 +329,7 @@ def read_nuninst(filename, architectures):
will be included in the report.
"""
nuninst = {}
with open(filename) as f:
with open(filename, encoding='ascii') as f:
for r in f:
if ":" not in r: continue
arch, packages = r.strip().split(":", 1)
@ -387,7 +387,7 @@ def write_heidi(filename, sources_t, packages_t,
The "X=X" parameters are optimizations to avoid "load global" in
the loops.
"""
with open(filename, 'w') as f:
with open(filename, 'w', encoding='ascii') as f:
# write binary packages
for arch in sorted(packages_t):
@ -426,7 +426,7 @@ def write_heidi_delta(filename, all_selected):
The order corresponds to that shown in update_output.
"""
with open(filename, "w") as fd:
with open(filename, "w", encoding='ascii') as fd:
fd.write("#HeidiDelta\n")
@ -463,7 +463,7 @@ def write_excuses(excuses, dest_file, output_format="yaml"):
"""
if output_format == "yaml":
ensuredir(os.path.dirname(dest_file))
with open(dest_file, 'w') as f:
with open(dest_file, 'w', encoding='utf-8') as f:
excuselist = []
for e in excuses:
excuselist.append(e.excusedata())
@ -473,7 +473,7 @@ def write_excuses(excuses, dest_file, output_format="yaml"):
f.write(yaml.dump(excusesdata, default_flow_style=False, allow_unicode=True))
elif output_format == "legacy-html":
ensuredir(os.path.dirname(dest_file))
with open(dest_file, 'w') as f:
with open(dest_file, 'w', encoding='utf-8') as f:
f.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n")
f.write("<html><head><title>excuses...</title>")
f.write("<meta http-equiv=\"Content-Type\" content=\"text/html;charset=utf-8\"></head><body>\n")
@ -491,13 +491,13 @@ def write_sources(sources_s, filename):
"""Write a sources file from Britney's state for a given suite
Britney discards fields she does not care about, so the resulting
file omitts a lot of regular fields.
file omits a lot of regular fields.
"""
key_pairs = ((VERSION, 'Version'), (SECTION, 'Section'),
(MAINTAINER, 'Maintainer'))
with open(filename, 'w') as f:
with open(filename, 'w', encoding='utf-8') as f:
for src in sources_s:
src_data = sources_s[src]
output = "Package: %s\n" % src
@ -528,7 +528,7 @@ def write_controlfiles(sources, packages, suite, basedir):
for arch in packages_s:
filename = os.path.join(basedir, 'Packages_%s' % arch)
binaries = packages_s[arch][0]
with open(filename, 'w') as f:
with open(filename, 'w', encoding='utf-8') as f:
for pkg in binaries:
output = "Package: %s\n" % pkg
bin_data = binaries[pkg]
@ -605,3 +605,18 @@ def is_nuninst_asgood_generous(architectures, old, new, break_arches=frozenset()
continue
diff = diff + (len(new[arch]) - len(old[arch]))
return diff <= 0
def clone_nuninst(nuninst, packages_s, architectures):
"""Selectively deep clone nuninst
Given nuninst table, the package table for a given suite and
a list of architectures, this function will clone the nuninst
table. Only the listed architectures will be deep cloned -
the rest will only be shallow cloned.
"""
clone = nuninst.copy()
for arch in architectures:
clone[arch] = set(x for x in nuninst[arch] if x in packages_s[arch][0])
clone[arch + "+all"] = set(x for x in nuninst[arch + "+all"] if x in packages_s[arch][0])
return clone

1
completer.py
View File

@ -39,7 +39,6 @@ class Completer(object):
complete = []
tpu = []
for e in britney.excuses:
ver = None
pkg = e.name
suite = 'unstable'
if pkg[0] == '-':

25
excuse.py
View File

@ -15,8 +15,6 @@
# GNU General Public License for more details.
import re
import string
class Excuse(object):
"""Excuse class
@ -65,6 +63,11 @@ class Excuse(object):
self.reason = {}
self.htmlline = []
def sortkey(self):
if self.daysold == None:
return (-1, self.name)
return (self.daysold, self.name)
@property
def is_valid(self):
return self._is_valid
@ -151,7 +154,7 @@ class Excuse(object):
(self.name, self.name, lp_pkg, self.name, lp_old, lp_new))
if self.maint:
res = res + "<li>Maintainer: %s\n" % (self.maint)
if self.section and string.find(self.section, "/") > -1:
if self.section and self.section.find("/") > -1:
res = res + "<li>Section: %s\n" % (self.section)
if self.daysold != None:
if self.mindays == 0:
@ -165,7 +168,7 @@ class Excuse(object):
for x in self.htmlline:
res = res + "<li>" + x + "\n"
lastdep = ""
for x in sorted(self.deps, lambda x,y: cmp(x.split('/')[0], y.split('/')[0])):
for x in sorted(self.deps, key=lambda x: x.split('/')[0]):
dep = x.split('/')[0]
if dep == lastdep: continue
lastdep = dep
@ -198,14 +201,8 @@ class Excuse(object):
(self.name, self.ver[0], self.ver[1]))
if self.maint:
maint = self.maint
# ugly hack to work around strange encoding in pyyaml
# should go away with pyyaml in python 3
try:
maint.decode('ascii')
except UnicodeDecodeError:
maint = unicode(self.maint,'utf-8')
res.append("Maintainer: %s" % maint)
if self.section and string.find(self.section, "/") > -1:
if self.section and self.section.find("/") > -1:
res.append("Section: %s" % (self.section))
if self.daysold != None:
if self.mindays == 0:
@ -219,7 +216,7 @@ class Excuse(object):
for x in self.htmlline:
res.append("" + x + "")
lastdep = ""
for x in sorted(self.deps, lambda x,y: cmp(x.split('/')[0], y.split('/')[0])):
for x in sorted(self.deps, key=lambda x: x.split('/')[0]):
dep = x.split('/')[0]
if dep == lastdep: continue
lastdep = dep
@ -246,10 +243,10 @@ class Excuse(object):
excusedata["new-bugs"] = sorted(self.newbugs)
excusedata["old-bugs"] = sorted(self.oldbugs)
if self.forced:
excusedata["forced-reason"] = self.reason.keys()
excusedata["forced-reason"] = list(self.reason.keys())
excusedata["reason"] = []
else:
excusedata["reason"] = self.reason.keys()
excusedata["reason"] = list(self.reason.keys())
excusedata["is-candidate"] = self.is_valid
return excusedata

6
hints.py
View File

@ -12,6 +12,8 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
from __future__ import print_function
from migrationitem import MigrationItem
class HintCollection(object):
@ -22,7 +24,7 @@ class HintCollection(object):
return self.search(type)
def search(self, type=None, onlyactive=True, package=None, \
version=None, days=None, removal=None):
version=None, removal=None):
return [ hint for hint in self._hints if
(type is None or type == hint.type) and
@ -36,7 +38,7 @@ class HintCollection(object):
try:
self._hints.append(Hint(hint, user))
except AssertionError:
print "Ignoring broken hint %r from %s" % (hint, user)
print("Ignoring broken hint %r from %s" % (hint, user))
class Hint(object):
NO_VERSION = [ 'block', 'block-all', 'block-udeb' ]

5
installability/builder.py
View File

@ -29,7 +29,7 @@ class _RelationBuilder(object):
self._new_breaks = set(binary_data[1])
def add_dependency_clause(self, or_clause, frozenset=frozenset):
def add_dependency_clause(self, or_clause):
"""Add a dependency clause
The clause must be a sequence of (name, version, architecture)
@ -48,7 +48,6 @@ class _RelationBuilder(object):
clause = self._itbuilder._intern_set(or_clause)
binary = self._binary
itbuilder = self._itbuilder
package_table = itbuilder._package_table
okay = False
for dep_tuple in clause:
okay = True
@ -388,7 +387,7 @@ class InstallabilityTesterBuilder(object):
ekey = (deps, con, rdeps)
find_eqv_table[ekey].append(pkg)
for pkg_list in find_eqv_table.itervalues():
for pkg_list in find_eqv_table.values():
if len(pkg_list) < 2:
continue

37
installability/solver.py
View File

@ -14,7 +14,8 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
from functools import partial
from __future__ import print_function
import os
from installability.tester import InstallabilityTester
@ -43,8 +44,8 @@ class InstallabilitySolver(InstallabilityTester):
- NB: arch:all packages are "re-mapped" to given architecture.
(simplifies caches and dependency checking)
"""
InstallabilityTester.__init__(self, universe, revuniverse, testing,
broken, essentials, safe_set, eqv_table)
super().__init__(universe, revuniverse, testing,
broken, essentials, safe_set, eqv_table)
def solve_groups(self, groups):
@ -106,7 +107,7 @@ class InstallabilitySolver(InstallabilityTester):
# "Self-conflicts" => ignore
continue
if debug_solver and other not in order[key]['before']:
print "N: Conflict induced order: %s before %s" % (key, other)
print("N: Conflict induced order: %s before %s" % (key, other))
order[key]['before'].add(other)
order[other]['after'].add(key)
@ -125,7 +126,7 @@ class InstallabilitySolver(InstallabilityTester):
# "Self-dependency" => ignore
continue
if debug_solver and other not in order[key]['after']:
print "N: Removal induced order: %s before %s" % (key, other)
print("N: Removal induced order: %s before %s" % (key, other))
order[key]['after'].add(other)
order[other]['before'].add(key)
@ -162,13 +163,13 @@ class InstallabilitySolver(InstallabilityTester):
for other in (other_adds - other_rms):
if debug_solver and other != key and other not in order[key]['after']:
print "N: Dependency induced order (add): %s before %s" % (key, other)
print("N: Dependency induced order (add): %s before %s" % (key, other))
order[key]['after'].add(other)
order[other]['before'].add(key)
for other in (other_rms - other_adds):
if debug_solver and other != key and other not in order[key]['before']:
print "N: Dependency induced order (remove): %s before %s" % (key, other)
print("N: Dependency induced order (remove): %s before %s" % (key, other))
order[key]['before'].add(other)
order[other]['after'].add(key)
@ -208,7 +209,7 @@ class InstallabilitySolver(InstallabilityTester):
merged[n] = scc_id
del order[n]
if debug_solver:
print "N: SCC: %s -- %s" % (scc_id, str(sorted(com)))
print("N: SCC: %s -- %s" % (scc_id, str(sorted(com))))
for com in comps:
node = com[0]
@ -223,27 +224,27 @@ class InstallabilitySolver(InstallabilityTester):
if debug_solver:
print "N: -- PARTIAL ORDER --"
print("N: -- PARTIAL ORDER --")
for com in sorted(order):
if debug_solver and order[com]['before']:
print "N: %s <= %s" % (com, str(sorted(order[com]['before'])))
print("N: %s <= %s" % (com, str(sorted(order[com]['before']))))
if not order[com]['after']:
# This component can be scheduled immediately, add it
# to "check"
check.add(com)
elif debug_solver:
print "N: %s >= %s" % (com, str(sorted(order[com]['after'])))
print("N: %s >= %s" % (com, str(sorted(order[com]['after']))))
if debug_solver:
print "N: -- END PARTIAL ORDER --"
print "N: -- LINEARIZED ORDER --"
print("N: -- END PARTIAL ORDER --")
print("N: -- LINEARIZED ORDER --")
for cur in iter_except(check.pop, KeyError):
if order[cur]['after'] <= emitted:
# This item is ready to be emitted right now
if debug_solver:
print "N: %s -- %s" % (cur, sorted(scc[cur]))
print("N: %s -- %s" % (cur, sorted(scc[cur])))
emitted.add(cur)
result.append([key2item[x] for x in scc[cur]])
if order[cur]['before']:
@ -254,7 +255,7 @@ class InstallabilitySolver(InstallabilityTester):
check.update(order[cur]['before'] - emitted)
if debug_solver:
print "N: -- END LINEARIZED ORDER --"
print("N: -- END LINEARIZED ORDER --")
return result
@ -301,8 +302,8 @@ class InstallabilitySolver(InstallabilityTester):
return result
def _dump_groups(self, groups):
print "N: === Groups ==="
print("N: === Groups ===")
for (item, adds, rms) in groups:
print "N: %s => A: %s, R: %s" % (str(item), str(adds), str(rms))
print "N: === END Groups ==="
print("N: %s => A: %s, R: %s" % (str(item), str(adds), str(rms)))
print("N: === END Groups ===")

193
installability/tester.py
View File

@ -12,11 +12,13 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
from collections import defaultdict
from functools import partial
from itertools import ifilter, ifilterfalse
from itertools import chain, filterfalse
from britney_util import iter_except
class InstallabilityTester(object):
def __init__(self, universe, revuniverse, testing, broken, essentials,
@ -52,6 +54,7 @@ class InstallabilityTester(object):
self._revuniverse = revuniverse
self._safe_set = safe_set
self._eqv_table = eqv_table
self._stats = InstallabilityStats()
# Cache of packages known to be broken - we deliberately do not
# include "broken" in it. See _optimize for more info.
@ -84,7 +87,7 @@ class InstallabilityTester(object):
eqv_table = self._eqv_table
testing = self._testing
tcopy = [x for x in testing]
for t in ifilterfalse(cache_inst.__contains__, tcopy):
for t in filterfalse(cache_inst.__contains__, tcopy):
if t in cbroken:
continue
res = check_inst(t)
@ -97,13 +100,16 @@ class InstallabilityTester(object):
testing -= eqv_set
cbroken |= eqv_set
@property
def stats(self):
return self._stats
def are_equivalent(self, p1, p2):
"""Test if p1 and p2 are equivalent
Returns True if p1 and p2 have the same "signature" in
the package dependency graph (i.e. relations can not tell
them appart sematically except for their name)
them apart semantically except for their name)
"""
eqv_table = self._eqv_table
return p1 in eqv_table and p2 in eqv_table[p1]
@ -113,7 +119,7 @@ class InstallabilityTester(object):
"""Add a binary package to "testing"
If the package is not known, this method will throw an
Keyrror.
KeyError.
"""
t = (pkg_name, pkg_version, pkg_arch)
@ -125,6 +131,8 @@ class InstallabilityTester(object):
self._testing.add(t)
elif t not in self._testing:
self._testing.add(t)
if self._cache_inst:
self._stats.cache_drops += 1
self._cache_inst = set()
if self._cache_broken:
# Re-add broken packages as some of them may now be installable
@ -163,6 +171,7 @@ class InstallabilityTester(object):
if t not in self._broken and t in self._cache_inst:
# It is in our cache (and not guaranteed to be broken) - throw out the cache
self._cache_inst = set()
self._stats.cache_drops += 1
return True
@ -176,17 +185,21 @@ class InstallabilityTester(object):
Returns False otherwise.
"""
self._stats.is_installable_calls += 1
t = (pkg_name, pkg_version, pkg_arch)
if t not in self._universe:
raise KeyError(str(t))
if t not in self._testing or t in self._broken:
self._stats.cache_hits += 1
return False
if t in self._cache_inst:
self._stats.cache_hits += 1
return True
self._stats.cache_misses += 1
return self._check_inst(t)
@ -194,8 +207,9 @@ class InstallabilityTester(object):
# See the explanation of musts, never and choices below.
cache_inst = self._cache_inst
stats = self._stats
if t in cache_inst and not never:
if musts and t in cache_inst and not never:
# use the inst cache only for direct queries/simple queries.
cache = True
if choices:
@ -213,7 +227,6 @@ class InstallabilityTester(object):
if cache:
return True
universe = self._universe
testing = self._testing
cbroken = self._cache_broken
@ -254,15 +267,14 @@ class InstallabilityTester(object):
# set conflicts with t - either way, t is f***ed
cbroken.add(t)
testing.remove(t)
stats.conflicts_essential += 1
return False
musts.update(start)
never.update(ess_never)
# curry check_loop
check_loop = partial(self._check_loop, universe, testing,
eqv_table, musts, never, choices,
cbroken)
eqv_table, stats, musts, never, cbroken)
# Useful things to remember:
#
@ -277,7 +289,7 @@ class InstallabilityTester(object):
#
# * check never includes choices (these are always in choices)
#
# * A package is installable if never and musts are disjoined
# * A package is installable if never and musts are disjointed
# and both check and choices are empty.
# - exception: _pick_choice may determine the installability
# of t via recursion (calls _check_inst). In this case
@ -297,9 +309,9 @@ class InstallabilityTester(object):
rebuild.
"""
# We already satisfied/chosen at least one of the litterals
# We already satisfied/chosen at least one of the literals
# in the choice, so the choice is gone
for choice in ifilter(musts.isdisjoint, choices):
for choice in filter(musts.isdisjoint, choices):
# cbroken is needed here because (in theory) it could
# have changed since the choice was discovered and it
# is smaller than testing (so presumably faster)
@ -307,7 +319,7 @@ class InstallabilityTester(object):
if len(remain) > 1 and not remain.isdisjoint(safe_set):
first = None
for r in ifilter(safe_set.__contains__, remain):
for r in filter(safe_set.__contains__, remain):
# don't bother giving extra arguments to _check_inst. "safe" packages are
# usually trivial to satisfy on their own and will not involve conflicts
# (so never will not help)
@ -317,6 +329,7 @@ class InstallabilityTester(object):
if first:
musts.add(first)
check.add(first)
stats.choice_resolved_using_safe_set += 1
continue
# None of the safe set choices are installable, so drop them
remain -= safe_set
@ -325,11 +338,13 @@ class InstallabilityTester(object):
# the choice was reduced to one package we haven't checked - check that
check.update(remain)
musts.update(remain)
stats.choice_presolved += 1
continue
if not remain:
# all alternatives would violate the conflicts or are uninstallable
# => package is not installable
stats.choice_presolved += 1
return None
# The choice is still deferred
@ -346,8 +361,8 @@ class InstallabilityTester(object):
choices_tmp = set()
check_tmp = set([p])
if not self._check_loop(universe, testing, eqv_table,
musts_copy, never_tmp,
choices_tmp, cbroken,
stats, musts_copy, never_tmp,
cbroken, choices_tmp,
check_tmp):
# p cannot be chosen/is broken (unlikely, but ...)
continue
@ -363,6 +378,7 @@ class InstallabilityTester(object):
# routine, but to conserve stack-space, we return
# and expect to be called again later.
musts.update(musts_copy)
stats.choice_resolved_without_restore_point += 1
return False
if not musts.isdisjoint(never_tmp):
@ -370,6 +386,7 @@ class InstallabilityTester(object):
# t uninstallable, so p is a no-go.
continue
stats.backtrace_restore_point_created += 1
# We are not sure that p is safe, setup a backtrack
# point and recurse.
never_tmp |= never
@ -386,6 +403,7 @@ class InstallabilityTester(object):
# to satisfy the dependencies, so pretend to conflict
# with it - hopefully it will reduce future choices.
never.add(p)
stats.backtrace_restore_point_used += 1
# Optimization for the last case; avoid the recursive call
# and just assume the last will lead to a solution. If it
@ -393,11 +411,12 @@ class InstallabilityTester(object):
# have to back-track anyway.
check.add(last)
musts.add(last)
stats.backtrace_last_option += 1
return False
# END _pick_choice
while check:
if not check_loop(check):
if not check_loop(choices, check):
verdict = False
break
@ -417,12 +436,14 @@ class InstallabilityTester(object):
if verdict:
# if t is installable, then so are all packages in musts
self._cache_inst.update(musts)
stats.solved_installable += 1
else:
stats.solved_uninstallable += 1
return verdict
def _check_loop(self, universe, testing, eqv_table, musts, never,
choices, cbroken, check, len=len,
def _check_loop(self, universe, testing, eqv_table, stats, musts, never,
cbroken, choices, check, len=len,
frozenset=frozenset):
"""Finds all guaranteed dependencies via "check".
@ -431,7 +452,7 @@ class InstallabilityTester(object):
returns True, then t is installable.
"""
# Local variables for faster access...
not_satisfied = partial(ifilter, musts.isdisjoint)
not_satisfied = partial(filter, musts.isdisjoint)
# While we have guaranteed dependencies (in check), examine all
# of them.
@ -453,7 +474,7 @@ class InstallabilityTester(object):
# so "obviously" we can never choose any of its conflicts
never.update(cons & testing)
# depgroup can be satisifed by picking something that is
# depgroup can be satisfied by picking something that is
# already in musts - lets pick that (again). :)
for depgroup in not_satisfied(deps):
@ -492,13 +513,19 @@ class InstallabilityTester(object):
# _build_eqv_packages_table method for more
# information on how this works.
new_cand = set(x for x in candidates if x not in possible_eqv)
stats.eqv_table_times_used += 1
for chosen in iter_except(possible_eqv.pop, KeyError):
new_cand.add(chosen)
possible_eqv -= eqv_table[chosen]
stats.eqv_table_total_number_of_alternatives_eliminated += len(candidates) - len(new_cand)
if len(new_cand) == 1:
check.update(new_cand)
musts.update(new_cand)
stats.eqv_table_reduced_to_one += 1
continue
elif len(candidates) == len(new_cand):
stats.eqv_table_reduced_by_zero += 1
candidates = frozenset(new_cand)
# defer this choice till later
choices.add(candidates)
@ -513,18 +540,19 @@ class InstallabilityTester(object):
eqv_table = self._eqv_table
cbroken = self._cache_broken
universe = self._universe
stats = self._stats
safe_set = self._safe_set
ess_base = set(x for x in self._essentials if x[2] == arch and x in testing)
start = set(ess_base)
ess_never = set()
ess_choices = set()
not_satisified = partial(ifilter, start.isdisjoint)
not_satisified = partial(filter, start.isdisjoint)
while ess_base:
self._check_loop(universe, testing, eqv_table,
start, ess_never, ess_choices,
cbroken, ess_base)
self._check_loop(universe, testing, eqv_table, stats,
start, ess_never, cbroken,
ess_choices, ess_base)
if ess_choices:
# Try to break choices where possible
nchoice = set()
@ -548,3 +576,118 @@ class InstallabilityTester(object):
return self._cache_ess[arch]
def compute_stats(self):
universe = self._universe
eqv_table = self._eqv_table
graph_stats = defaultdict(ArchStats)
seen_eqv = defaultdict(set)
for pkg in universe:
(pkg_name, pkg_version, pkg_arch) = pkg
deps, con = universe[pkg]
arch_stats = graph_stats[pkg_arch]
arch_stats.nodes += 1
if pkg in eqv_table and pkg not in seen_eqv[pkg_arch]:
eqv = [e for e in eqv_table[pkg] if e[2] == pkg_arch]
arch_stats.eqv_nodes += len(eqv)
arch_stats.add_dep_edges(deps)
arch_stats.add_con_edges(con)
for stat in graph_stats.values():
stat.compute_all()
return graph_stats
class InstallabilityStats(object):
def __init__(self):
self.cache_hits = 0
self.cache_misses = 0
self.cache_drops = 0
self.backtrace_restore_point_created = 0
self.backtrace_restore_point_used = 0
self.backtrace_last_option = 0
self.choice_presolved = 0
self.choice_resolved_using_safe_set = 0
self.choice_resolved_without_restore_point = 0
self.is_installable_calls = 0
self.solved_installable = 0
self.solved_uninstallable = 0
self.conflicts_essential = 0
self.eqv_table_times_used = 0
self.eqv_table_reduced_to_one = 0
self.eqv_table_reduced_by_zero = 0
self.eqv_table_total_number_of_alternatives_eliminated = 0
def stats(self):
formats = [
"Requests - is_installable: {is_installable_calls}",
"Cache - hits: {cache_hits}, misses: {cache_misses}, drops: {cache_drops}",
"Choices - pre-solved: {choice_presolved}, safe-set: {choice_resolved_using_safe_set}, No RP: {choice_resolved_without_restore_point}",
"Backtrace - RP created: {backtrace_restore_point_created}, RP used: {backtrace_restore_point_used}, reached last option: {backtrace_last_option}",
"Solved - installable: {solved_installable}, uninstallable: {solved_uninstallable}, conflicts essential: {conflicts_essential}",
"Eqv - times used: {eqv_table_times_used}, perfect reductions: {eqv_table_reduced_to_one}, failed reductions: {eqv_table_reduced_by_zero}, total no. of alternatives pruned: {eqv_table_total_number_of_alternatives_eliminated}",
]
return [x.format(**self.__dict__) for x in formats]
class ArchStats(object):
def __init__(self):
self.nodes = 0
self.eqv_nodes = 0
self.dep_edges = []
self.con_edges = []
self.stats = defaultdict(lambda: defaultdict(int))
def stat(self, statname):
return self.stats[statname]
def stat_summary(self):
text = []
for statname in ['nodes', 'dependency-clauses', 'dependency-clause-alternatives', 'negative-dependency-clauses']:
stat = self.stats[statname]
if statname != 'nodes':
format_str = "%s, max: %d, min: %d, median: %d, average: %f (%d/%d)"
values = [statname, stat['max'], stat['min'], stat['median'], stat['average'], stat['sum'], stat['size']]
if 'average-per-node' in stat:
format_str += ", average-per-node: %f"
values.append(stat['average-per-node'])
else:
format_str = "nodes: %d, eqv-nodes: %d"
values = (self.nodes, self.eqv_nodes)
text.append(format_str % tuple(values))
return text
def add_dep_edges(self, edges):
self.dep_edges.append(edges)
def add_con_edges(self, edges):
self.con_edges.append(edges)
def _list_stats(self, stat_name, sorted_list, average_per_node=False):
if sorted_list:
stats = self.stats[stat_name]
stats['max'] = sorted_list[-1]
stats['min'] = sorted_list[0]
stats['sum'] = sum(sorted_list)
stats['size'] = len(sorted_list)
stats['average'] = float(stats['sum'])/len(sorted_list)
stats['median'] = sorted_list[len(sorted_list)//2]
if average_per_node:
stats['average-per-node'] = float(stats['sum'])/self.nodes
def compute_all(self):
dep_edges = self.dep_edges
con_edges = self.con_edges
sorted_no_dep_edges = sorted(len(x) for x in dep_edges)
sorted_size_dep_edges = sorted(len(x) for x in chain.from_iterable(dep_edges))
sorted_no_con_edges = sorted(len(x) for x in con_edges)
self._list_stats('dependency-clauses', sorted_no_dep_edges)
self._list_stats('dependency-clause-alternatives', sorted_size_dep_edges, average_per_node=True)
self._list_stats('negative-dependency-clauses', sorted_no_con_edges)

7
migrationitem.py
View File

@ -54,6 +54,9 @@ class MigrationItem(object):
def __hash__(self):
return hash((self.uvname, self.version))
def __lt__(self, other):
return (self.uvname, self.version) < (other.uvname, other.version)
@property
def name(self):
return self._name
@ -142,5 +145,5 @@ class MigrationItem(object):
return self._uvname
class UnversionnedMigrationItem(MigrationItem):
def __init__(self, name = None):
MigrationItem.__init__(self, name = name, versionned = False)
def __init__(self, name=None):
super().__init__(name=name, versionned=False)

3
tests/mock_swift.py
View File

@ -113,6 +113,8 @@ class AutoPkgTestSwiftServer:
def start(self):
assert self.server_pid is None, 'already started'
if self.log:
self.log.close()
self.log = tempfile.TemporaryFile()
p = os.fork()
if p:
@ -123,6 +125,7 @@ class AutoPkgTestSwiftServer:
if s.connect_ex(('127.0.0.1', self.port)) == 0:
break
time.sleep(0.1)
s.close()
return
# child; quiesce logging on stderr

6
tests/test_autopkgtest.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/python3
# (C) 2014 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
@ -87,10 +87,10 @@ class TestAutoPkgTest(TestBase):
if excuses_expect:
for re in excuses_expect:
self.assertRegexpMatches(excuses, re, excuses)
self.assertRegex(excuses, re, excuses)
if excuses_no_expect:
for re in excuses_no_expect:
self.assertNotRegexpMatches(excuses, re, excuses)
self.assertNotRegex(excuses, re, excuses)
self.amqp_requests = set()
try:

15
tests/test_boottest.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/python3
# (C) 2014 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
@ -48,7 +48,7 @@ class TestTouchManifest(unittest.TestCase):
self.imagesdir = os.path.join(self.path, 'boottest/images')
os.makedirs(self.imagesdir)
self.addCleanup(shutil.rmtree, self.path)
_p = mock.patch('urllib.urlopen')
_p = mock.patch('urllib.request.urlopen')
self.mocked_urlopen = _p.start()
self.mocked_urlopen.side_effect = [
FakeResponse(code=404),
@ -71,7 +71,7 @@ class TestTouchManifest(unittest.TestCase):
def test_fetch(self):
# Missing manifest file is fetched dynamically
self.mocked_urlopen.side_effect = [
FakeResponse(code=200, content='foo 1.0'),
FakeResponse(code=200, content=b'foo 1.0'),
]
manifest = boottest.TouchManifest('ubuntu-touch', 'vivid')
self.assertNotEqual([], manifest._manifest)
@ -244,10 +244,10 @@ args.func()
# print('-------\nout: %s\n-----' % out)
if expect:
for re in expect:
self.assertRegexpMatches(excuses, re)
self.assertRegex(excuses, re)
if no_expect:
for re in no_expect:
self.assertNotRegexpMatches(excuses, re)
self.assertNotRegex(excuses, re)
def test_runs(self):
# `Britney` runs and considers binary packages for boottesting
@ -283,8 +283,9 @@ args.func()
# '<source> <version>\n'
test_input_path = os.path.join(
self.data.path, 'boottest/work/test_input')
self.assertEqual(
['green 1.1~beta\n'], open(test_input_path).readlines())
with open(test_input_path) as f:
self.assertEqual(
['green 1.1~beta\n'], f.readlines())
def test_pass(self):
# `Britney` updates boottesting information in excuses when the