From f961f74ef311cfff6754448bd456828640536f0a Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 17 Jun 2013 13:56:21 +0100 Subject: [PATCH] Run autopkgtests for valid candidates, and wait for the results. --- autopkgtest.py | 168 +++++++++++++++++++++++++++++++++++++++++++++++++ britney.conf | 5 ++ britney.py | 48 +++++++++++++- 3 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 autopkgtest.py diff --git a/autopkgtest.py b/autopkgtest.py new file mode 100644 index 0000000..7953770 --- /dev/null +++ b/autopkgtest.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 Canonical Ltd. +# Author: Colin Watson +# Partly based on code in auto-package-testing by +# Jean-Baptiste Lallement + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +from __future__ import print_function + +from collections import defaultdict +from contextlib import closing +import logging +import os +import subprocess +import tempfile +from textwrap import dedent + + +adt_britney = os.path.expanduser("~/auto-package-testing/jenkins/adt-britney") + + +class AutoPackageTest(object): + """autopkgtest integration + + Look for autopkgtest jobs to run for each update that is otherwise a + valid candidate, and collect the results. If an update causes any + autopkgtest jobs to be run, then they must all pass before the update is + accepted. + """ + + def __init__(self, series, debug=False): + self.series = series + self.debug = debug + self.read() + self.rc_path = None + + def _ensure_rc_file(self): + if self.rc_path: + return + self.rc_path = os.path.expanduser( + "~/proposed-migration/autopkgtest/rc.%s" % self.series) + with open(self.rc_path, "w") as rc_file: + home = os.path.expanduser("~") + print(dedent("""\ + aptroot: ~/.chdist/%s-proposed-amd64/ + apturi: file:%s/mirror/ubuntu + components: main restricted universe multiverse + rsync_host: rsync://10.189.74.2/adt/ + datadir: ~/proposed-migration/autopkgtest/data""" % + (self.series, home)), file=rc_file) + + @property + def _request_path(self): + return os.path.expanduser( + "~/proposed-migration/autopkgtest/work/adt.request.%s" % + self.series) + + @property + def _result_path(self): + return os.path.expanduser( + "~/proposed-migration/autopkgtest/work/adt.result.%s" % + self.series) + + def _parse(self, path): + if os.path.exists(path): + with open(path) as f: + for line in f: + line = line.strip() + if line.startswith("Suite:") or line.startswith("Date:"): + continue + linebits = line.split() + if len(linebits) < 2: + logging.warning( + "Invalid line format: '%s', skipped" % line) + continue + yield linebits + + def read(self): + self.pkglist = defaultdict(dict) + self.pkgcauses = defaultdict(lambda: defaultdict(list)) + for linebits in self._parse(self._result_path): + src = linebits.pop(0) + ver = linebits.pop(0) + self.pkglist[src][ver] = { + "status": "NEW", + "causes": {}, + } + try: + status = linebits.pop(0).upper() + self.pkglist[src][ver]["status"] = status + while True: + trigsrc = linebits.pop(0) + trigver = linebits.pop(0) + self.pkglist[src][ver]["causes"][trigsrc] = trigver + self.pkgcauses[trigsrc][trigver].append((status, src, ver)) + except IndexError: + # End of the list + pass + + def _adt_britney(self, *args): + command = [ + adt_britney, + "-c", self.rc_path, "-r", self.series, "-a", "amd64", "-PU", + ] + if self.debug: + command.append("-d") + command.extend(args) + subprocess.check_call(command) + + def request(self, packages): + self._ensure_rc_file() + request_path = self._request_path + if os.path.exists(request_path): + os.unlink(request_path) + with closing(tempfile.NamedTemporaryFile(mode="w")) as request_file: + for src, ver in packages: + if src in self.pkglist and ver in self.pkglist[src]: + continue + print("%s %s" % (src, ver), file=request_file) + request_file.flush() + self._adt_britney("request", "-O", request_path, request_file.name) + for linebits in self._parse(request_path): + # Make sure that there's an entry in pkgcauses for each new + # request, so that results() gives useful information without + # relying on the submit/collect cycle. This improves behaviour + # in dry-run mode. + src = linebits.pop(0) + ver = linebits.pop(0) + try: + status = linebits.pop(0).upper() + while True: + trigsrc = linebits.pop(0) + trigver = linebits.pop(0) + for status, csrc, cver in self.pkgcauses[src][ver]: + if csrc == trigsrc and cver == trigver: + break + else: + self.pkgcauses[src][ver].append( + (status, trigsrc, trigver)) + except IndexError: + # End of the list + pass + + def submit(self): + self._ensure_rc_file() + request_path = self._request_path + if os.path.exists(request_path): + self._adt_britney("submit", request_path) + + def collect(self): + self._ensure_rc_file() + result_path = self._result_path + self._adt_britney("collect", "-O", result_path) + self.read() + + def results(self, trigsrc, trigver): + for status, src, ver in self.pkgcauses[trigsrc][trigver]: + yield status, src, ver diff --git a/britney.conf b/britney.conf index 5cb81ea..a182cd5 100644 --- a/britney.conf +++ b/britney.conf @@ -52,3 +52,8 @@ HINTS_STGRABER = ALL SMOOTH_UPDATES = badgers REMOVE_OBSOLETE = no + +# autopkgtest needs to know the series name; set to the empty string to +# disable autopkgtest +ADT_SERIES = saucy +ADT_DEBUG = no diff --git a/britney.py b/britney.py index 2457de6..006a425 100755 --- a/britney.py +++ b/britney.py @@ -213,6 +213,7 @@ from excuse import Excuse from migrationitem import MigrationItem, HintItem from hints import HintCollection from britney import buildSystem +from autopkgtest import AutoPackageTest __author__ = 'Fabio Tranchitella and the Debian Release Team' @@ -916,7 +917,7 @@ class Britney(object): elif len(l) == 1: # All current hints require at least one argument self.__log("Malformed hint found in %s: '%s'" % (filename, line), type="W") - elif l[0] in ["approve", "block", "block-all", "block-udeb", "unblock", "unblock-udeb", "force", "urgent", "remove"]: + elif l[0] in ["approve", "block", "block-all", "block-udeb", "unblock", "unblock-udeb", "force", "force-autopkgtest", "urgent", "remove"]: for package in l[1:]: hints.add_hint('%s %s' % (l[0], package), who) elif l[0] in ["age-days"]: @@ -925,7 +926,7 @@ class Britney(object): else: hints.add_hint(l, who) - for x in ["approve", "block", "block-all", "block-udeb", "unblock", "unblock-udeb", "force", "urgent", "remove", "age-days"]: + for x in ["approve", "block", "block-all", "block-udeb", "unblock", "unblock-udeb", "force", "force-autopkgtest", "urgent", "remove", "age-days"]: z = {} for hint in hints[x]: package = hint.package @@ -1755,6 +1756,49 @@ class Britney(object): # extract the not considered packages, which are in the excuses but not in upgrade_me unconsidered = [e.name for e in self.excuses if e.name not in upgrade_me] + if self.options.adt_series: + # trigger autopkgtests for valid candidates + adt_debug = getattr(self.options, "adt_debug", "no") == "yes" + autopkgtest = AutoPackageTest( + self.options.adt_series, debug=adt_debug) + autopkgtest_packages = [] + autopkgtest_excuses = [] + for e in self.excuses: + if e.name not in upgrade_me: + continue + # skip removals, binary-only candidates, and proposed-updates + if e.name.startswith("-") or "/" in e.name or "_" in e.name: + pass + if e.ver[1] == "-": + continue + autopkgtest_excuses.append(e) + autopkgtest_packages.append((e.name, e.ver[1])) + autopkgtest.request(autopkgtest_packages) + if not self.options.dry_run: + autopkgtest.submit() + autopkgtest.collect() + for e in autopkgtest_excuses: + adtpass = True + for status, adtsrc, adtver in autopkgtest.results( + e.name, e.ver[1]): + e.addhtml( + "autopkgtest for %s %s: %s" % (adtsrc, adtver, status)) + if status != "PASS": + adtpass = False + if not adtpass: + forces = [ + x for x in self.hints.search( + 'force-autopkgtest', package=e.name) + if self.same_source(e.ver[1], x.version) ] + if forces: + e.addhtml( + "Should ignore, but forced by %s" % forces[0].user) + else: + upgrade_me.remove(e.name) + unconsidered.append(e.name) + e.addhtml("Not considered") + e.is_valid = False + # invalidate impossible excuses for e in self.excuses: # parts[0] == package name