Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-02-23 11:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move age-handling into a separate file

Browse Source 
Signed-off-by: Niels Thykier <niels@thykier.net>
This commit is contained in:
Niels Thykier 2016-03-25 15:23:34 +00:00
parent 2e81e55c56
commit 5ec3aea43a
4 changed files with 279 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
britney.py
View File

@ -57,7 +57,7 @@ Other than source and binary packages, Britney loads the following data:
of a source package (see Britney.read_dates).
* Urgencies, which contains the urgency of the upload of a given
version of a source package (see Britney.read_urgencies).
version of a source package (see AgePolicy._read_urgencies).
* Hints, which contains lists of commands which modify the standard behaviour
of Britney (see Britney.read_hints).
@ -206,6 +206,7 @@ from britney_util import (old_libraries_format, undo_changes,
write_excuses, write_heidi_delta, write_controlfiles,
old_libraries, is_nuninst_asgood_generous,
clone_nuninst, check_installability)
from policies.policy import AgePolicy, PolicyVerdict
from consts import (VERSION, SECTION, BINARIES, MAINTAINER, FAKESRC,
SOURCE, SOURCEVER, ARCHITECTURE, DEPENDS, CONFLICTS,
PROVIDES, MULTIARCH, ESSENTIAL)
@ -248,10 +249,9 @@ class Britney(object):
This method initializes and populates the data lists, which contain all
the information needed by the other methods of the class.
"""
# britney's "day" begins at 3pm
self.date_now = int(((time.time() / (60*60)) - 15) / 24)
# parse the command line arguments
self.policies = []
self.__parse_arguments()
MigrationItem.set_architectures(self.options.architectures)
@ -336,10 +336,9 @@ class Britney(object):
self.bugs = {'unstable': self.read_bugs(self.options.unstable),
'testing': self.read_bugs(self.options.testing),}
self.normalize_bugs()
# read additional data
self.dates = self.read_dates(self.options.testing)
self.urgencies = self.read_urgencies(self.options.testing)
for policy in self.policies:
policy.hints = self.hints
policy.initialise(self)
def merge_pkg_entries(self, package, parch, pkg_entry1, pkg_entry2,
check_fields=check_fields, check_field_name=check_field_name):
@ -402,7 +401,7 @@ class Britney(object):
# minimum days for unstable-testing transition and the list of hints
# are handled as an ad-hoc case
self.MINDAYS = {}
MINDAYS = {}
self.HINTS = {'command-line': self.HINTS_ALL}
with open(self.options.config, encoding='utf-8') as config:
for line in config:
@ -411,7 +410,7 @@ class Britney(object):
k = k.strip()
v = v.strip()
if k.startswith("MINDAYS_"):
self.MINDAYS[k.split("_")[1].lower()] = int(v)
MINDAYS[k.split("_")[1].lower()] = int(v)
elif k.startswith("HINTS_"):
self.HINTS[k.split("_")[1].lower()] = \
reduce(lambda x,y: x+y, [hasattr(self, "HINTS_" + i) and getattr(self, "HINTS_" + i) or (i,) for i in v.split()])
@ -452,6 +451,8 @@ class Britney(object):
self.options.ignore_cruft == "0":
self.options.ignore_cruft = False
self.policies.append(AgePolicy(self.options, MINDAYS))
def log(self, msg, type="I"):
"""Print info messages according to verbosity level
@ -863,90 +864,6 @@ class Britney(object):
if maxvert is None:
self.bugs['testing'][pkg] = []
def read_dates(self, basedir):
"""Read the upload date for the packages from the specified directory
The upload dates are read from the `Dates' file within the directory
specified as `basedir' parameter. The file contains rows with the
format:
<package-name> <version> <date-of-upload>
The dates are expressed as the number of days from 1970-01-01.
The method returns a dictionary where the key is the binary package
name and the value is a tuple with two items, the version and the date.
"""
dates = {}
filename = os.path.join(basedir, "Dates")
self.log("Loading upload data from %s" % filename)
for line in open(filename, encoding='ascii'):
l = line.split()
if len(l) != 3: continue
try:
dates[l[0]] = (l[1], int(l[2]))
except ValueError:
self.log("Dates, unable to parse \"%s\"" % line, type="E")
return dates
def write_dates(self, basedir, dates):
"""Write the upload date for the packages to the specified directory
For a more detailed explanation of the format, please check the method
read_dates.
"""
filename = os.path.join(basedir, "Dates")
self.log("Writing upload data to %s" % filename)
with open(filename, 'w', encoding='utf-8') as f:
for pkg in sorted(dates):
f.write("%s %s %d\n" % ((pkg,) + dates[pkg]))
def read_urgencies(self, basedir):
"""Read the upload urgency of the packages from the specified directory
The upload urgencies are read from the `Urgency' file within the
directory specified as `basedir' parameter. The file contains rows
with the format:
<package-name> <version> <urgency>
The method returns a dictionary where the key is the binary package
name and the value is the greatest urgency from the versions of the
package that are higher then the testing one.
"""
urgencies = {}
filename = os.path.join(basedir, "Urgency")
self.log("Loading upload urgencies from %s" % filename)
for line in open(filename, errors='surrogateescape', encoding='ascii'):
l = line.split()
if len(l) != 3: continue
# read the minimum days associated with the urgencies
urgency_old = urgencies.get(l[0], None)
mindays_old = self.MINDAYS.get(urgency_old, 1000)
mindays_new = self.MINDAYS.get(l[2], self.MINDAYS[self.options.default_urgency])
# if the new urgency is lower (so the min days are higher), do nothing
if mindays_old <= mindays_new:
continue
# if the package exists in testing and it is more recent, do nothing
tsrcv = self.sources['testing'].get(l[0], None)
if tsrcv and apt_pkg.version_compare(tsrcv[VERSION], l[1]) >= 0:
continue
# if the package doesn't exist in unstable or it is older, do nothing
usrcv = self.sources['unstable'].get(l[0], None)
if not usrcv or apt_pkg.version_compare(usrcv[VERSION], l[1]) < 0:
continue
# update the urgency for the package
urgencies[l[0]] = l[2]
return urgencies
def read_hints(self, basedir):
"""Read the hint commands from the specified directory
@ -1357,13 +1274,6 @@ class Britney(object):
excuse.addhtml("%s source package doesn't exist" % (src))
update_candidate = False
# retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing)
urgency = self.urgencies.get(src, self.options.default_urgency)
if not source_t:
if self.MINDAYS[urgency] < self.MINDAYS[self.options.default_urgency]:
excuse.addhtml("Ignoring %s urgency setting for NEW package" % (urgency))
urgency = self.options.default_urgency
# if there is a `remove' hint and the requested version is the same as the
# version in testing, then stop here and return False
for item in self.hints.search('remove', package=src):
@ -1424,30 +1334,32 @@ class Britney(object):
# permanence in unstable before updating testing; if the source package is too young,
# the check fails and we set update_candidate to False to block the update; consider
# the age-days hint, if specified for the package
if suite == 'unstable':
if src not in self.dates:
self.dates[src] = (source_u[VERSION], self.date_now)
elif self.dates[src][0] != source_u[VERSION]:
self.dates[src] = (source_u[VERSION], self.date_now)
policy_info = excuse.policy_info
policy_verdict = PolicyVerdict.PASS
for policy in self.policies:
if suite in policy.applicable_suites:
v = policy.apply_policy(policy_info, suite, src, source_t, source_u)
if v.value > policy_verdict.value:
policy_verdict = v
days_old = self.date_now - self.dates[src][1]
min_days = self.MINDAYS[urgency]
if policy_verdict.is_rejected:
update_candidate = False
for age_days_hint in [x for x in self.hints.search('age-days', package=src)
if source_u[VERSION] == x.version]:
excuse.addhtml("Overriding age needed from %d days to %d by %s" % (min_days,
int(age_days_hint.days), age_days_hint.user))
min_days = int(age_days_hint.days)
excuse.setdaysold(days_old, min_days)
if days_old < min_days:
urgent_hints = [x for x in self.hints.search('urgent', package=src)
if source_u[VERSION] == x.version]
if urgent_hints:
excuse.addhtml("Too young, but urgency pushed by %s" % (urgent_hints[0].user))
# Joggle some things into excuses
# - remove once the YAML is the canonical source for this information
if 'age' in policy_info:
age_info = policy_info['age']
age_hint = age_info.get('age-requirement-reduced', None)
age_min_req = age_info['age-requirement']
if age_hint:
new_req = age_hint['new-requirement']
who = age_hint['changed-by']
if new_req:
excuse.addhtml("Overriding age needed from %d days to %d by %s" % (
age_min_req, new_req, who))
else:
update_candidate = False
excuse.addreason("age")
excuse.addhtml("Too young, but urgency pushed by %s" % who)
excuse.setdaysold(age_info['current-age'], age_min_req)
all_binaries = self.all_binaries
@ -1560,7 +1472,7 @@ class Britney(object):
excuse.addreason("build-arch")
excuse.addreason("build-arch-%s" % arch)
if self.date_now != self.dates[src][1]:
if 'age' in policy_info and policy_info['age']['current-age']:
excuse.addhtml(text)
# if the source package has no binaries, set update_candidate to False to block the update
@ -2656,11 +2568,8 @@ class Britney(object):
write_controlfiles(self.sources, self.binaries,
'testing', self.options.testing)
# write dates
try:
self.write_dates(self.options.outputdir, self.dates)
except AttributeError:
self.write_dates(self.options.testing, self.dates)
for policy in self.policies:
policy.save_state(self)
# write HeidiResult
self.log("Writing Heidi results to %s" % self.options.heidi_output)

1
excuse.py
View File

@ -59,6 +59,7 @@ class Excuse(object):
self.oldbugs = set()
self.reason = {}
self.htmlline = []
self.policy_info = {}
def sortkey(self):
if self.daysold == None:

0
policies/__init__.py Normal file
View File

242
policies/policy.py Normal file
View File

@ -0,0 +1,242 @@
from abc import abstractmethod
from enum import Enum, unique
import apt_pkg
import os
import time
from consts import VERSION
@unique
class PolicyVerdict(Enum):
""""""
"""
The migration item passed the policy.
"""
PASS = 1
"""
The policy was completely overruled by a hint.
"""
PASS_HINTED = 2
"""
The migration item did not pass the policy, but the failure is believed
to be temporary
"""
REJECTED_TEMPORARILY = 3
"""
The migration item did not pass the policy and the failure is believed
to be uncorrectable (i.e. a hint or a new version is needed)
"""
REJECTED_PERMANENTLY = 4
@property
def is_rejected(self):
return True if self.name.startswith('REJECTED') else False
class BasePolicy(object):
def __init__(self, options, applicable_suites):
self.options = options
self.applicable_suites = applicable_suites
self.hints = None
# FIXME: use a proper logging framework
def log(self, msg, type="I"):
"""Print info messages according to verbosity level
An easy-and-simple log method which prints messages to the standard
output. The type parameter controls the urgency of the message, and
can be equal to `I' for `Information', `W' for `Warning' and `E' for
`Error'. Warnings and errors are always printed, and information is
printed only if verbose logging is enabled.
"""
if self.options.verbose or type in ("E", "W"):
print("%s: [%s] - %s" % (type, time.asctime(), msg))
def initialise(self, britney):
"""Called once to make the policy initialise any data structures
This is useful for e.g. parsing files or other "heavy do-once" work.
"""
pass
def save_state(self, britney):
"""Called once at the end of the run to make the policy save any persistent data
Note this will *not* be called for "dry-runs" as such runs should not change
the state.
"""
pass
@abstractmethod
def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist):
pass
class AgePolicy(BasePolicy):
"""Configurable Aging policy for source migrations
The AgePolicy will let packages stay in the source suite for a pre-defined
amount of days before letting migrate (based on their urgency, if any).
The AgePolicy's decision is influenced by the following:
State files:
* ${TESTING}/Urgency: File containing urgencies for source packages.
Note that urgencies are "sticky" and the most "urgent" urgency will be
used (i.e. the one with lowest age-requirements).
- This file needs to be updated externally, if the policy should take
urgencies into consideration. If empty (or not updated), the policy
will simply use the default urgency (see the "Config" section below)
- In Debian, these values are taken from the .changes file, but that is
not a requirement for Britney.
* ${TESTING}/Dates: File containing the age of all source packages.
- The policy will automatically update this file.
Config:
* DEFAULT_URGENCY: Name of the urgency used for packages without an urgency
(or for unknown urgencies). Will also be used to set the "minimum"
aging requirements for packages not in the target suite.
* MINDAYS_<URGENCY>: The age-requirements in days for packages with the
given urgency.
- Commonly used urgencies are: low, medium, high, emergency, critical
Hints:
* urgent <source>/<version>: Disregard the age requirements for a given
source/version.
* age-days X <source>/<version>: Set the age requirements for a given
source/version to X days. Note that X can exceed the highest
age-requirement normally given.
"""
def __init__(self, options, mindays):
super().__init__(options, {'unstable'})
self._min_days = mindays
if options.default_urgency not in mindays:
raise ValueError("Missing age-requirement for default urgency (MINDAYS_%s)" % options.default_urgency)
self._min_days_default = mindays[options.default_urgency]
# britney's "day" begins at 3pm
self._date_now = int(((time.time() / (60*60)) - 15) / 24)
self._dates = {}
self._urgencies = {}
def initialise(self, britney):
super().initialise(britney)
self._read_dates_file()
self._read_urgencies_file(britney)
def save_state(self, britney):
super().save_state(britney)
self._write_dates_file()
def apply_policy(self, policy_info, suite, source_name, source_data_tdist, source_data_srcdist):
# retrieve the urgency for the upload, ignoring it if this is a NEW package (not present in testing)
urgency = self._urgencies.get(source_name, self.options.default_urgency)
if 'age' not in policy_info:
policy_info['age'] = age_info = {}
else:
age_info = policy_info['age']
if urgency not in self._min_days:
age_info['unknown-urgency'] = urgency
urgency = self.options.default_urgency
if not source_data_tdist:
if self._min_days[urgency] < self._min_days_default:
age_info['urgency-reduced'] = {
'from': urgency,
'to': self.options.default_urgency,
}
urgency = self.options.default_urgency
if source_name not in self._dates:
self._dates[source_name] = (source_data_srcdist[VERSION], self._date_now)
elif self._dates[source_name][0] != source_data_srcdist[VERSION]:
self._dates[source_name] = (source_data_srcdist[VERSION], self._date_now)
days_old = self._date_now - self._dates[source_name][1]
min_days = self._min_days[urgency]
age_info['age-requirement'] = min_days
age_info['current-age'] = days_old
for age_days_hint in [x for x in self.hints.search('age-days', package=source_name)
if source_data_srcdist[VERSION] == x.version]:
new_req = int(age_days_hint.days)
age_info['age-requirement-reduced'] = {
'new-requirement': new_req,
'changed-by': age_days_hint.user
}
min_days = new_req
if days_old < min_days:
urgent_hints = [x for x in self.hints.search('urgent', package=source_name)
if source_data_srcdist[VERSION] == x.version]
if urgent_hints:
age_info['age-requirement-reduced'] = {
'new-requirement': 0,
'changed-by': urgent_hints[0].user
}
return PolicyVerdict.PASS_HINTED
else:
return PolicyVerdict.REJECTED_TEMPORARILY
return PolicyVerdict.PASS
def _read_dates_file(self):
"""Parse the dates file"""
dates = self._dates
filename = os.path.join(self.options.testing, 'Dates')
with open(filename, encoding='utf-8') as fd:
for line in fd:
# <source> <version> <date>
l = line.split()
if len(l) != 3:
continue
try:
dates[l[0]] = (l[1], int(l[2]))
except ValueError:
pass
def _read_urgencies_file(self, britney):
urgencies = self._urgencies
filename = os.path.join(self.options.testing, 'Urgency')
min_days_default = self._min_days_default
with open(filename, errors='surrogateescape', encoding='ascii') as fd:
for line in fd:
# <source> <version> <urgency>
l = line.split()
if len(l) != 3:
continue
# read the minimum days associated with the urgencies
urgency_old = urgencies.get(l[0], None)
mindays_old = self._min_days.get(urgency_old, 1000)
mindays_new = self._min_days.get(l[2], min_days_default)
# if the new urgency is lower (so the min days are higher), do nothing
if mindays_old <= mindays_new:
continue
# if the package exists in testing and it is more recent, do nothing
tsrcv = britney.sources['testing'].get(l[0], None)
if tsrcv and apt_pkg.version_compare(tsrcv[VERSION], l[1]) >= 0:
continue
# if the package doesn't exist in unstable or it is older, do nothing
usrcv = britney.sources['unstable'].get(l[0], None)
if not usrcv or apt_pkg.version_compare(usrcv[VERSION], l[1]) < 0:
continue
# update the urgency for the package
urgencies[l[0]] = l[2]
def _write_dates_file(self):
dates = self._dates
directory = self.options.testing
filename = os.path.join(directory, 'Dates')
filename_tmp = os.path.join(directory, 'Dates_new')
with open(filename_tmp, 'w', encoding='utf-8') as fd:
for pkg in sorted(dates):
version, date = dates[pkg]
fd.write("%s %s %d\n" % (pkg, version, date))
os.rename(filename_tmp, filename)