From 90e4bb6ba222fc99efeecdf7ad0743bc0e626ba7 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 3 Dec 2017 12:01:42 +0000 Subject: [PATCH 1/7] Rewrite documentation in sphinx Signed-off-by: Niels Thykier --- doc/conf.py | 172 ++++++++++++++++++ doc/contributing-to-britney.rst | 40 ++++ doc/index.rst | 23 +++ ...ified.md => short-intro-to-migrations.rst} | 158 ++++------------ doc/solutions-to-common-policy-issues.rst | 105 +++++++++++ doc/what-is-britney.rst | 13 ++ 6 files changed, 389 insertions(+), 122 deletions(-) create mode 100644 doc/conf.py create mode 100644 doc/contributing-to-britney.rst create mode 100644 doc/index.rst rename doc/{migrations-simplified.md => short-intro-to-migrations.rst} (66%) create mode 100644 doc/solutions-to-common-policy-issues.rst create mode 100644 doc/what-is-britney.rst diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..5e2f425 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# britney2 documentation build configuration file, created by +# sphinx-quickstart on Sat Dec 2 12:34:27 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +# Add .. so sphinx can find the britney2 modules. +sys.path.insert(0, os.path.abspath('..')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'britney2' +copyright = '2017, Niels Thykier and others' +author = 'Niels Thykier' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'classic' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'britney2doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'britney2.tex', 'britney2 Documentation', + 'Niels Thykier', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'britney2', 'britney2 Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'britney2', 'britney2 Documentation', + author, 'britney2', 'One line description of project.', + 'Miscellaneous'), +] + diff --git a/doc/contributing-to-britney.rst b/doc/contributing-to-britney.rst new file mode 100644 index 0000000..2bf1523 --- /dev/null +++ b/doc/contributing-to-britney.rst @@ -0,0 +1,40 @@ +Contributing to the development of britney2 +=========================================== + +If you are interested in improving britney, you can obtain the source +code via:: + + git clone https://anonscm.debian.org/git/mirror/britney2.git + # Additional tests + git clone https://anonscm.debian.org/git/collab-maint/britney2-tests.git + +You will need some packages to run britney and the test suites:: + + # Runtime dependencies + apt install python3 python3-apt python3-yaml + # Test dependencies + apt install python3-pytest libclass-accessor-perl rsync + # Documentation generator + apt install python3-sphinx + + +Britney has some basic unit tests, which are handled by py.test. It +also has some larger integration tests (from the `britney2-tests` +repo). Running the tests are done via:: + + cd britney2 + # Basic unit tests + py.test-3 + # Integration tests + rm -fr ./test-out/ + ../britney2-tests/bin/runtests ./britney.py ../britney2-tests/t ./test-out + +The `runtests` command in `britney2-tests` supports running only a +subset of the tests. Please see its `--help` output for more +information. + +Finally, there is also some heavier tests based on some snapshots of +live data from Debian. The data files for these are available in the +`live-data` submodule of the `britney2-tests` repo. They consume +quite a bit of disk space and britney will need at least 1.3GB of RAM +to process them. diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..95ddab3 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,23 @@ +.. britney2 documentation master file, created by + sphinx-quickstart on Sat Dec 2 12:34:27 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to britney2's documentation! +==================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + what-is-britney + short-intro-to-migrations + solutions-to-common-policy-issues + contributing-to-britney + +* :ref:`search` + +.. Add these later if/when needed: +.. * :ref:`genindex` +.. * :ref:`modindex` + diff --git a/doc/migrations-simplified.md b/doc/short-intro-to-migrations.rst similarity index 66% rename from doc/migrations-simplified.md rename to doc/short-intro-to-migrations.rst index 16c9229..92d3336 100644 --- a/doc/migrations-simplified.md +++ b/doc/short-intro-to-migrations.rst @@ -1,13 +1,11 @@ -#! TITLE: Britney migration documentation -#! SUBTITLE: Understanding britney's workflow - -# Migrations +A short introduction to migrations +================================== This is a technical introduction to how britney handles migrations. Being an introduction, it deliberately oversimplifies certain things at the expense of accuracy. It also covers common migration issues and how to fix -them. +them in :doc:`solutions-to-common-policy-issues`. The document is primarily aimed at contributors for distributions that want to understand the basics of @@ -21,7 +19,8 @@ abstract. Furthermore, the document assumes familiarity with Debian-based distribution practises and terminology (such as "suites" and "source package"). -## A high level overview of britney and migrations +A high level overview of britney and migrations +----------------------------------------------- The purpose of britney is to (semi-)automatically select a number of migration items from a series of source suites @@ -34,9 +33,12 @@ of the following points: 1. The migration items pass a number of policies for the target suite. Most of these policies are basically that the migration items do not regress on selected QA checks. + * An item satisfying this part is called a `valid candiate`. - 1. Installability will not regress as a result of + + 2. Installability will not regress as a result of migrating the migration items. + * An item that (also) satisfies this part will be selected for migration. @@ -48,14 +50,15 @@ issue (as it is not a regression). This only leaves the definition of a migration items. They come in several variants defined in the next section. -## Migration items +Migration items +--------------- Internally, britney groups packages into migration items based on a few rules. There are several kinds of migration items and this document will only describe the source migration item. -> A source migration item is one upload of a source package, with -> associated binary packages once built. + A source migration item is one upload of a source package, with + associated binary packages once built. Once a new version of a source package appears in the source suite, britney will create track it with a source migration item. As the @@ -72,7 +75,8 @@ As implied earlier, there are several other migration types. But they are not covered in this document. They deal with cases like removals, rebuilds of existing binaries, etc. -## Migration phase 1: Policies / Excuses +Migration phase 1: Policies / Excuses +------------------------------------- To begin with, britney will apply a number of policies to all migration items. Each policy will rate each migration @@ -100,19 +104,25 @@ data that britney has access to. This mean that without any change to the items themselves: 1. Items that passed originally may fail in a later britney run. - 1. Likewise, items may go from a "permanent failure" to a pass. + + 2. Likewise, items may go from a "permanent failure" to a pass. This can be seen in the following example case: 1. A new version of package is uploaded. + * Britney processes the package and concludes that there no blocking bugs, so the package passes the bug policy. - 1. Then before it migrates, someone files a blocking bug against + + 2. Then before it migrates, someone files a blocking bug against the new version. + * Britney reprocesses the package and now concludes it has a regression in the bug policy (i.e. the policy verdict goes from "pass" to "permanent fail"). - 1. The bug is examined and it is determined that the bug also affects the + + 3. The bug is examined and it is determined that the bug also affects the version in the target suite. The bug tracker is updated to reflect this. + * Britney reprocesses the package again and now concludes there is a blocking bug, but it is not a regression (since it also affects the target suite). This means the policy verdict now go from "fail" to "pass". @@ -131,7 +141,8 @@ a hinted migration generally implies that the problem will not prevent future migrations for newer versions of the same source package (assuming that the problem is deterministic). -## Migration phase 2: Installability regression testing +Migration phase 2: Installability regression testing +---------------------------------------------------- For the migration items that pass the previous phase, britney will do a test migration to see if anything becomes uninstallable. @@ -143,11 +154,12 @@ problems here, the britney log file has to be examined. This requires a bit more technical insight as it has not been polished as much as the excuses. -### Confirming a migration +Confirming a migration +^^^^^^^^^^^^^^^^^^^^^^ To start with; if a migration is accepted and "committed" (i.e. it will not be rolled back), britney will include in a line starting with `final:` like -in this example: +in this example:: Apparently successful final: -cwltool,-libtest-redisserver-perl,-pinfo,-webdis,hol88 @@ -173,12 +185,13 @@ Reminder: Migration items generally use the name of the source package. There are exceptions to that "rule" (but they are not common cases covered by this document). -### Debugging failed migration attempts +Debugging failed migration attempts +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Start by confirming that the migration item was not accepted (as described in the above section). If the migration item does not appear on a `final:` line, then we need to debug the actual migration attempts. Migration attempts look -something like this: +something like this:: trying: -webdis accepted: -webdis @@ -214,8 +227,9 @@ later attempt. It is not always immediately obvious, which attempt is better for debugging. When in doubt, it is *usually* easiest to look at the attempt with the least amount of new uninstallable packages. -In the libaws example, a total of 4 binary packages become uninstallable on the -architecture `arm64`. Here is the output again with this information high lighted: +In the libaws example, a total of 4 binary packages become +uninstallable on the architecture `arm64`. Here is the output again +with this information high lighted:: migration item(s) being attemped vvvvvv @@ -230,7 +244,7 @@ architecture `arm64`. Here is the output again with this information high light Please note that britney is lazy and will often reject an item after proving that there is a regression on a single architecture. So in the above example, we are not actually sure whether this problem is architecture specific. For -`easy`-hints, the information is presented slightly different. +`easy`-hints, the information is presented slightly different:: Trying easy from autohinter: asis/2017-1 dh-ada-library/6.12 [...] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -255,103 +269,3 @@ things break. If there are few broken packages, it is often a question of looking for `Breaks`-relations or `Depends`-relations with upper bounds on versions / on old packages being removed. Alternatively, there are also tools like `dose-debcheck`, which attempts to analyse and explain problems like this. - -# Common issues with policy decisions - -## Britney complains about a fixed bug in the source suite (bug policy) - -All decisions about bugs are related to data set extracted -from the bug tracker. If britney says that the new version -introduces a bug, then it is because the data set from the bug -tracker lists that bug for *a* version in the source suite and -without it appearing for the version(s) in the target suite. - -Please note that these data sets do not include versions, so -britney is unable to tell exactly which versions are affected. -The only thing, it can tell, is what suite the bug affects. - -There is a number of common cases, where this is observed: - - * The metadata on the bug is wrong. A known example is the - Debian BTS, where if a bug has a `fixed` version equal to - a `found` version, the bug is considered unfixed. - - * The bug is fixed, but the old version is still around in - the source suite. In this case, britney will generally - mention a "missing build" or "old binaries". - -If the metadata is wrong, the solution is to fix it in the bug -tracker and wait until britney receives a new data set. In -the other case, the recommendation is to see the sections on -"missing builds" and "old binaries" below. As long as they -are present, the package may be blocked by bugs in the older -versions of the binaries. - -## Britney complains about "missing builds" - -A "missing build" happens when britney detects that the binaries -for a given architecture are missing or is not up to date. This -is detected by checking the "Packages" files in the archive, so -britney have no knowledge of *why* the build is missing. -Accordingly, this kind of issue is flagged as a "possibly permanent" -issue. - -If the omission is deliberate (e.g. the new version no longer -supports that architecture), then please have the old binaries -for that architecture removed from the *source* suite. Once -those are removed, britney will no longer see that as a problem. - -Otherwise, please check the build services for any issues with -building or uploading the package to the archive. - -**Common misconceptions**: If the architecture is no longer -supported, the removal of the old binaries should happen in -the *source* suite (e.g. Debian unstable). However, many -people mistakenly request a removal from the *target* suite -(e.g. Debian testing). Unfortunately, this is not the proper -solution (and, britney does not support architecture -specific removals so it may be difficult to do anyhow). - -## Britney complains about "old binaries" - -Depending on the configuration of the britney instance, this may -or may not be a blocker. If the distribution has chosen to enable -the "ignore_cruft" option, this is merely a warning/note. That -said, even in this mode it can block a package from migration. - -This appears when britney detects that there are older versions of -the binary packages around, which was built by (an older version of) -the same source package. - -This is common with distributions where their archive management -software is capable of keeping old binaries as long as something -depends on them (e.g. DAK as used by Debian). Therefore, the -most common solution is to ensure all reverse dependencies are -updated to use the new binaries and then have the old ones -removed (the latter commonly known as "decrufting"). Technically, -this is also solvable by "decrufting" without updating/rebuilding -other packages. Though whether this is an acceptable practise -depends on the distribution. - -Alternatively, if the distribution uses the "ignore_cruft" option, -this (in itself) is not a blocker. However, it commonly triggers -non-obvious issues: - - * If the bugs policy is enabled, an bug in the old binaries that - is fixed in the new version will still be a blocker. Here, the - best solution is to get rid of the old binaries. - - (Note: the bugs data is not versioned so britney cannot tell which - versions the bug applies to. Just which suite they affect) - - * Even if the migration item is a valid candidate (i.e. all policy - checked have passed), it may cause installability regressions as - britney will also attempt to keep the old binaries around as long - as they are used. The most often cause of this when the old - binaries are not co-installable with the new ones. - - (Note: Britney generally only works with the highest version of a - given binary. If you have libfoo1 depends on libfoo-data v1 and - then libfoo2 depends on libfoo-data v2, then libfoo1 will become - uninstallable as libfoo-data v2 will "shadow" libfoo-data v1) - diff --git a/doc/solutions-to-common-policy-issues.rst b/doc/solutions-to-common-policy-issues.rst new file mode 100644 index 0000000..ec1b43a --- /dev/null +++ b/doc/solutions-to-common-policy-issues.rst @@ -0,0 +1,105 @@ +Solutions to common policy decisions +==================================== + +.. contents:: + +Britney complains about a fixed bug in the source suite (bug policy) +-------------------------------------------------------------------- + +All decisions about bugs are related to data set extracted +from the bug tracker. If britney says that the new version +introduces a bug, then it is because the data set from the bug +tracker lists that bug for *a* version in the source suite and +without it appearing for the version(s) in the target suite. + +Please note that these data sets do not include versions, so +britney is unable to tell exactly which versions are affected. +The only thing, it can tell, is what suite the bug affects. + +There is a number of common cases, where this is observed: + + * The metadata on the bug is wrong. A known example is the + Debian BTS, where if a bug has a `fixed` version equal to + a `found` version, the bug is considered unfixed. + + * The bug is fixed, but the old version is still around in + the source suite. In this case, britney will generally + mention a "missing build" or "old binaries". + +If the metadata is wrong, the solution is to fix it in the bug +tracker and wait until britney receives a new data set. In +the other case, the recommendation is to see the sections on +"missing builds" and "old binaries" below. As long as they +are present, the package may be blocked by bugs in the older +versions of the binaries. + +Britney complains about "missing builds" +---------------------------------------- + +A "missing build" happens when britney detects that the binaries +for a given architecture are missing or is not up to date. This +is detected by checking the "Packages" files in the archive, so +britney have no knowledge of *why* the build is missing. +Accordingly, this kind of issue is flagged as a "possibly permanent" +issue. + +If the omission is deliberate (e.g. the new version no longer +supports that architecture), then please have the old binaries +for that architecture removed from the *source* suite. Once +those are removed, britney will no longer see that as a problem. + +Otherwise, please check the build services for any issues with +building or uploading the package to the archive. + +**Common misconceptions**: If the architecture is no longer +supported, the removal of the old binaries should happen in +the *source* suite (e.g. Debian unstable). However, many +people mistakenly request a removal from the *target* suite +(e.g. Debian testing). Unfortunately, this is not the proper +solution (and, britney does not support architecture +specific removals so it may be difficult to do anyhow). + +Britney complains about "old binaries" +-------------------------------------- + +Depending on the configuration of the britney instance, this may +or may not be a blocker. If the distribution has chosen to enable +the "ignore_cruft" option, this is merely a warning/note. That +said, even in this mode it can block a package from migration. + +This appears when britney detects that there are older versions of +the binary packages around, which was built by (an older version of) +the same source package. + +This is common with distributions where their archive management +software is capable of keeping old binaries as long as something +depends on them (e.g. DAK as used by Debian). Therefore, the +most common solution is to ensure all reverse dependencies are +updated to use the new binaries and then have the old ones +removed (the latter commonly known as "decrufting"). Technically, +this is also solvable by "decrufting" without updating/rebuilding +other packages. Though whether this is an acceptable practise +depends on the distribution. + +Alternatively, if the distribution uses the "ignore_cruft" option, +this (in itself) is not a blocker. However, it commonly triggers +non-obvious issues: + + * If the bugs policy is enabled, an bug in the old binaries that + is fixed in the new version will still be a blocker. Here, the + best solution is to get rid of the old binaries. + + * Note: the bugs data is not versioned so britney cannot tell which + versions the bug applies to. Just which suite they affect. + + * Even if the migration item is a valid candidate (i.e. all policy + checked have passed), it may cause installability regressions as + britney will also attempt to keep the old binaries around as long + as they are used. The most often cause of this when the old + binaries are not co-installable with the new ones. + + * Note: Britney generally only works with the highest version of a + given binary. If you have libfoo1 depends on libfoo-data v1 and + then libfoo2 depends on libfoo-data v2, then libfoo1 will become + uninstallable as libfoo-data v2 will "shadow" libfoo-data v1. + diff --git a/doc/what-is-britney.rst b/doc/what-is-britney.rst new file mode 100644 index 0000000..28ad0ab --- /dev/null +++ b/doc/what-is-britney.rst @@ -0,0 +1,13 @@ +What is britney and britney2 +============================ + +Britney2 is a program that can be used to `migrate` debian-based +packages from one suite to another. It is the tool used by Debian and +Ubuntu to choose which packages are ready for wider testing. + +Britney2 is the reimplementation and replacement of the original +britney program. Through out this documentation, any mention of +britney generally refers to the britney2 implementation. When there +is a need to reference the original britney implementation, this +documentation will use britney1. + From 0f41b1b6df564e1ed20c270aa86b49ce1575c2b3 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 3 Dec 2017 12:56:24 +0000 Subject: [PATCH 2/7] Rewrite hints as restructured text Signed-off-by: Niels Thykier --- doc/{hints.md => hints.rst} | 135 ++++++++++++++++++------------------ doc/index.rst | 2 + doc/setting-up-britney.rst | 77 ++++++++++++++++++++ 3 files changed, 146 insertions(+), 68 deletions(-) rename doc/{hints.md => hints.rst} (77%) create mode 100644 doc/setting-up-britney.rst diff --git a/doc/hints.md b/doc/hints.rst similarity index 77% rename from doc/hints.md rename to doc/hints.rst index 42f6aae..359175c 100644 --- a/doc/hints.md +++ b/doc/hints.rst @@ -1,50 +1,30 @@ -#! TITLE: Britney hints documentation -#! SUBTITLE: How to override the britney policies +Hints +===== -# Britney hints +This document describes the `britney hints` and the format of the hint +file. All hints are basically small instructions to britney. The +vast majority of them involve overriding a quality gating policy in +britney. However, there are a few hints that assist britney into +finding solutions that it cannot compute itself. -This document describes the "britney hints". These are basically -small instructions to britney. The vast majority of them involve -overriding a quality gating policy in britney. However, there are a -few hints that assist britney into finding solutions that it cannot -compute itself. - - -## Related configuration - -The configuration `HINTSDIR` determines which directory contains the -so called "hints files". This is complimented with the `HINTS_` -configurations that defines a "hint file" and the related hint -permissions for it. - -For each `HINTS_` configuration, britney will attempt to read -`/`. Note that it lowercases `` when looking -for the file. - - -Configuration example: - - HINTSDIR = /etc/britney2/hints - HINTS_ANNA = ALL - HINTS_JOHN = STANDARD - HINTS_FREEZE = block block-all block-udeb - HINTS_AUTO-REMOVALS = remove +There are the following type of hints: -There are no fixed rules for how to use hints files. Though usually, -each person with permissions to give hints to britney will have their -own hint file along with write permissions for that file. It can also -make sense to create hint files for "roles". Like in the above -example there are two human hints (anna and john) plus two non-human -hinters (freeze and auto-removals). + * Policy overrides + * Migration selections (with or without overrides) + * Other +Please see :doc:`setting-up-britney` for how to configure hint files +and for how to limit which hints are allowed in a given hint file. -## Format of the hint file +Format of the hint file +----------------------- -The hint file is a plain text file with line-based content. Empty and -whitespace-only lines are ignored. If the first (non-whitespace) -character is a `#`, then britney considers it a comment and ignores -it. However, other tooling may interpret these comments (as is the -case for e.g. some parts of the Debian infrastructure). +All hints are read from hint files. The hint file is a plain text file +with line-based content. Empty and whitespace-only lines are ignored. +If the first (non-whitespace) character is a `#`, then britney +considers it a comment and ignores it. However, other tooling may +interpret these comments (as is the case for e.g. some parts of the +Debian infrastructure). The remaining lines are considered space-separated lists, where the first element must be a known hint. The remaining elements will be @@ -65,16 +45,8 @@ generally refers to the removal of said item rather than the migration of the hint. -# Hints - -There are the following type of hints: - - * Policy overrides - * Migration selections (with or without overrides) - * Other - - -## Policy override hints +Policy override hints +--------------------- The policy override hints are used to disable or tweak various policies in britney. Their effects are generally very precise ways of @@ -84,7 +56,9 @@ Some of these items are built-in while others are related to specific policies. In the latter case, they are only valid if the given policy is enabled (as the policy registers them when it is enabled). -### block-all `` + +block-all `` +^^^^^^^^^^^^^^^^^^ Usually used during freezes to prevent migrations by default. @@ -92,16 +66,19 @@ The `` can be one of: * `source`: Blocks all source migrations. This is a superset of `new-source`. + * `new-source`: Block source migrations if the given source is not - already in the target suite. (Side-effect: Removed packages will - not re-enter the taget suite automatically). + already in the target suite. (Side-effect: Removed packages will + not re-enter the taget suite automatically). All variants of these can be overruled by a valid `unblock`-hint. Note that this does not and cannot restrict architecture specific migrations (e.g. binNMUs or first time builds for architectures). -### block ``, block-udeb `` + +block ``, block-udeb `` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Prevent the items in the `` from migrating until the hint is removed or overruled by the equivalent unblock hint (or a @@ -117,7 +94,9 @@ release cycle. Note that this does not and cannot restrict architecture specific migrations (e.g. binNMUs or first time builds for architectures). -### unblock ``, unblock-udeb `` + +unblock ``, unblock-udeb `` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Enable the items in `` to migrate by overriding `block`-, `block-all`- or `block-udeb`-hints. The `unblock`-hint (often under @@ -133,7 +112,9 @@ release cycle. The two types of block hint must be paired with their corresponding unblock hint - i.e. an `unblock-udeb` does not override a `block`. -### approve `` + +approve `` +^^^^^^^^^^^^^^^^^^^^^^^ A synonym of `unblock`. The variant is generally used for approving migrations from suites that require approvals. @@ -142,7 +123,9 @@ Aside from the tab-completion in the hint testing interface, which will give different suggestions to `approve` and `unblock`, the rest of britney will consider them identical. -### age-days `` `` + +age-days `` `` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set the length of time which the listed action items must have been in unstable before they can be considered candidates. This may be used @@ -154,7 +137,9 @@ whichever is encountered first during parsing overrides the others. Provided by the `age` policy. -### urgent `` + +urgent `` +^^^^^^^^^^^^^^^^^^^^^^ Approximately equivalent to `age-days 0 `, with the distinction that an "urgent" hint overrides any "age-days" hint for @@ -162,7 +147,9 @@ the same action item. Provided by the `age` policy. -### ignore-rc-bugs `` `` + +ignore-rc-bugs `` `` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The `` argument is a comma separated list of bugs that affect the items in ``. Britney will ignore these bugs @@ -175,7 +162,8 @@ migration item. Provided by the `bugs` policy -### ignore-piuparts `` +ignore-piuparts `` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The items in `` will not be blocked by regressions in results from piuparts tests. All items in `` must be @@ -183,7 +171,9 @@ versioned items. Provided by the `piuparts` policy -### force `` + +force `` +^^^^^^^^^^^^^^^^^^^^^ Override all policies that claim the items in `` have regressions or are otherwise not ready to migrate. All items in the @@ -194,7 +184,9 @@ This hint does not guarantee that they will migrate. To ensure that, you will have to combine it with a `force-hint`. However, please read the warning in the documentation for `force-hint` before you do this. -## Migration selection hints + +Migration selection hints +------------------------- All migration selection hints work on an "action list". This consists of at least 1 or more of the following (in any combination): @@ -207,7 +199,9 @@ All elements in the action list must be valid at the time the hint is attempted. Notably, if one action has already been completed, the entire hint is rejected as invalid. -### easy `` + +easy `` +^^^^^^^^^^^^^^^^^^^^ Perform all the migrations and removals denoted by `` as if it were a single migration group. If the end result is equal or better @@ -221,7 +215,8 @@ Note that for `easy` the `` must have at least two elements. There is no use-case where a single element for easy will make sense (as britney always tries those). -### hint `` +hint `` +^^^^^^^^^^^^^^^^^^^^ Perform all the migrations and removals denoted by `` as if it were a single migration group. After that, process all remaining @@ -246,7 +241,9 @@ undesireable changes to the target suite. In practise, this is rather rare but the hinter is letting britney decide what "repairs" the situation. -### force-hint `` + +force-hint `` +^^^^^^^^^^^^^^^^^^^^^^^^^^ The provided `` is migrated as-is regardless of what is broken by said migration. This often needs to be paired with a @@ -260,11 +257,13 @@ desirable than the resulting breakage. change can have long lasting undesireable consequences on the end result. -## Other hints +Other hints +----------- This section cover hints that have no other grouping. -### remove `` +remove `` +^^^^^^^^^^^^^^^^^^^^^^ Britney should attempt to remove all items in the `` from the target suite. The `` must consist entirely of diff --git a/doc/index.rst b/doc/index.rst index 95ddab3..56553cd 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,6 +14,8 @@ Welcome to britney2's documentation! short-intro-to-migrations solutions-to-common-policy-issues contributing-to-britney + hints + setting-up-britney * :ref:`search` diff --git a/doc/setting-up-britney.rst b/doc/setting-up-britney.rst new file mode 100644 index 0000000..b0aa43d --- /dev/null +++ b/doc/setting-up-britney.rst @@ -0,0 +1,77 @@ +How to setup britney +==================== + +This document describes how to install, configure and run britney in +your infrastructure. + +Installing britney +------------------ + +TODO + +Configuring britney +------------------- + +TODO + +hints - Configuring who can provide which hints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Britney reads all hints from a set of `hints` files. These files must +be placed in the directory denoted by the `HINTSDIR` configuration. +This is complimented with the `HINTS_` configurations that +defines a "hint file" and the related hint permissions for it. + +For each `HINTS_` configuration, britney will attempt to read +`/`. Note that it lowercases `` when looking +for the file. + + +Configuration example:: + + HINTSDIR = /etc/britney2/hints + HINTS_ANNA = ALL + HINTS_JOHN = STANDARD + HINTS_FREEZE = block block-all block-udeb + HINTS_AUTO-REMOVALS = remove + +In the above example, we have defined 4 hints files named `anna`, +`john`, `freeze` and `auto-removals`. These must be placed in +`/etc/britney2/hints` and be readable by britney. Furthermore, they +must be writable by (only) the people that are allowed to use the +particular hints file (apply `chown`, `chmod` and `setfacl` as +neccesary). + +The values on the right hand side of the `=` decides which hints are +permitted in the files. With the above definitions: + + * The file `anna` may use any known hint (including potentially + dangerous ones like `force` and `force-hint`) + + * The file `john` may use most of the known hints. The STANDARD + includes a lot of hints for overriding most policies when it + can be done without (additional) side-effects. However, it + excludes `force` and `force-hint` as they can cause unintional + results. + + * The file `freeze` can use any of the hints `block`, `block-all` + and `block-udeb`. + + * The file `auto-removals` can only use the hint called `remove`. + +There are no fixed rules for how to use hints files. Though usually, +each person with permissions to give hints to britney will have their +own hint file along with write permissions for that file. It can also +make sense to create hint files for "roles". Like in the above +example there are two human hinters (`anna` and `john`) plus two +non-human hinters (`freeze` and `auto-removals`). + +Please see :doc:`hints` for which hints are available and what they +can do. + + +Using the results from britney +------------------------------ + +TODO + From c11d329d77990f86b8c00e74f5d5b9cd9707ce14 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 3 Dec 2017 13:00:41 +0000 Subject: [PATCH 3/7] Add Makefile to generate documentation Signed-off-by: Niels Thykier --- Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..839f2d2 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = britney2 +SOURCEDIR = doc +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +docs: html + +.PHONY: help Makefile docs + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) From 20f709e1578138a41fca435bfbb127ab73d7356f Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Tue, 5 Dec 2017 06:31:23 +0000 Subject: [PATCH 4/7] doc: Wording/typos Signed-off-by: Niels Thykier --- doc/setting-up-britney.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/setting-up-britney.rst b/doc/setting-up-britney.rst index b0aa43d..cdb65f8 100644 --- a/doc/setting-up-britney.rst +++ b/doc/setting-up-britney.rst @@ -48,10 +48,10 @@ permitted in the files. With the above definitions: * The file `anna` may use any known hint (including potentially dangerous ones like `force` and `force-hint`) - * The file `john` may use most of the known hints. The STANDARD + * The file `john` may use most of the known hints. The set called STANDARD includes a lot of hints for overriding most policies when it can be done without (additional) side-effects. However, it - excludes `force` and `force-hint` as they can cause unintional + excludes `force` and `force-hint` as they can cause unintentional results. * The file `freeze` can use any of the hints `block`, `block-all` From 94a3136f643c3771c412fd63d673d5a974b5cb83 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Sat, 6 Jan 2018 11:28:21 +0800 Subject: [PATCH 5/7] Drop duplicate 'has new bugs' excuses item The other item includes bug numbers so this one is less useful. Closes: https://bugs.debian.org/886509 Signed-off-by: Niels Thykier --- britney2/policies/policy.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/britney2/policies/policy.py b/britney2/policies/policy.py index e588351..adcc938 100644 --- a/britney2/policies/policy.py +++ b/britney2/policies/policy.py @@ -476,9 +476,6 @@ class RCBugPolicy(BasePolicy): old_bugs = rcbugs_info['unique-target-bugs'] excuse.setbugs(old_bugs, new_bugs) if new_bugs: - excuse.addhtml("%s has new bugs!" % (source_name, quote(source_name))) excuse.addhtml("Updating %s introduces new bugs: %s" % (source_name, ", ".join( ["#%s" % (quote(a), a) for a in new_bugs]))) From 24bd315c202e459a28569cac6d1afc034fd3ee9d Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 7 Jan 2018 12:19:46 +0000 Subject: [PATCH 6/7] Update .gitignore Signed-off-by: Niels Thykier --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 1aa7995..2ff1b50 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ *.so *.pyc +.cache +.idea +_build + /lib/aptvercmp /lib/checklib /lib/freelist From bc0a5f6e2bc701b398979e4e836818d7826b2cd3 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 7 Jan 2018 13:16:06 +0000 Subject: [PATCH 7/7] britney: Split "ood" and "unsat_deps" loops The "Out of date" binaries loop has gotten too complex to also handle the "unsatisifiable dependency" check. Concretely, we failed to generate proper excuses for arch:all packages due to this. Separate the two loops to restore the arch:all check. Signed-off-by: Niels Thykier --- britney.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/britney.py b/britney.py index 4263c45..65146da 100755 --- a/britney.py +++ b/britney.py @@ -1368,9 +1368,29 @@ class Britney(object): excuse.addreason("block") excuse.policy_verdict = PolicyVerdict.REJECTED_NEEDS_APPROVAL + all_binaries = self.all_binaries + for pkg_id in source_u.binaries: + is_valid = self.excuse_unsat_deps(pkg_id.package_name, src, pkg_id.architecture, suite, excuse) + if is_valid: + continue + + binary_u = all_binaries[pkg_id] + # There is an issue with the package. If it is arch:any, then excuse_unsat_deps will have + # handled everything for us correctly. However, arch:all have some special-casing IRT + # nobreakall that we deal with ourselves here. + if binary_u.architecture == 'all' and pkg_id.architecture in self.options.nobreakall_arches: + inst_tester = self._inst_tester + # We sometimes forgive uninstallable arch:all packages on nobreakall architectures + # (e.g. we sometimes force-hint in arch:all packages that are only installable on + # on a subset of all nobreak architectures). + # This forgivness is only done if the package is already in testing AND it is broken + # in testing on this architecture already. Anything else would be a regression + if inst_tester.any_of_these_are_in_testing({pkg_id}) and not inst_tester.is_installable(pkg_id): + # It is a regression. + excuse.policy_verdict = PolicyVerdict.REJECTED_PERMANENTLY + # at this point, we check the status of the builds on all the supported architectures # to catch the out-of-date ones - all_binaries = self.all_binaries archs_to_consider = list(self.options.architectures) archs_to_consider.append('all') for arch in archs_to_consider: @@ -1404,20 +1424,6 @@ class Britney(object): else: uptodatebins = True - # if the package is architecture-dependent or the current arch is `nobreakall' - # find unsatisfied dependencies for the binary package - if binary_u.architecture != 'all' or arch in self.options.nobreakall_arches: - is_valid = self.excuse_unsat_deps(pkg, src, arch, suite, excuse) - inst_tester = self._inst_tester - if not is_valid and inst_tester.any_of_these_are_in_testing({binary_u.pkg_id}) \ - and not inst_tester.is_installable(binary_u.pkg_id): - # Forgive uninstallable packages only when - # they are already broken in testing ideally - # we would not need to be forgiving at - # all. However, due to how arch:all packages - # are handled, we do run into occasionally. - excuse.policy_verdict = PolicyVerdict.REJECTED_PERMANENTLY - # if there are out-of-date packages, warn about them in the excuse and set excuse.is_valid # to False to block the update; if the architecture where the package is out-of-date is # in the `outofsync_arches' list, then do not block the update