Merge branch 'master' into autopkgtest

.gitignore

@@ -2,6 +2,10 @@
 *.so
 *.pyc
 
+.cache
+.idea
+_build
+
 /lib/aptvercmp
 /lib/checklib
 /lib/freelist

Makefile

@@ -0,0 +1,22 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = britney2
+SOURCEDIR     = doc
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+docs: html
+
+.PHONY: help Makefile docs
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

britney.py

@@ -1386,9 +1386,29 @@ class Britney(object):
                     excuse.addreason("block")
                 excuse.policy_verdict = PolicyVerdict.REJECTED_NEEDS_APPROVAL
 
+        all_binaries = self.all_binaries
+        for pkg_id in source_u.binaries:
+            is_valid = self.excuse_unsat_deps(pkg_id.package_name, src, pkg_id.architecture, suite, excuse)
+            if is_valid:
+                continue
+
+            binary_u = all_binaries[pkg_id]
+            # There is an issue with the package.  If it is arch:any, then excuse_unsat_deps will have
+            # handled everything for us correctly.  However, arch:all have some special-casing IRT
+            # nobreakall that we deal with ourselves here.
+            if binary_u.architecture == 'all' and pkg_id.architecture in self.options.nobreakall_arches:
+                inst_tester = self._inst_tester
+                # We sometimes forgive uninstallable arch:all packages on nobreakall architectures
+                # (e.g. we sometimes force-hint in arch:all packages that are only installable on
+                #  on a subset of all nobreak architectures).
+                # This forgivness is only done if the package is already in testing AND it is broken
+                # in testing on this architecture already.  Anything else would be a regression
+                if inst_tester.any_of_these_are_in_testing({pkg_id}) and not inst_tester.is_installable(pkg_id):
+                    # It is a regression.
+                    excuse.policy_verdict = PolicyVerdict.REJECTED_PERMANENTLY
+
         # at this point, we check the status of the builds on all the supported architectures
         # to catch the out-of-date ones
-        all_binaries = self.all_binaries
         archs_to_consider = list(self.options.architectures)
         archs_to_consider.append('all')
         for arch in archs_to_consider:
@@ -1422,20 +1442,6 @@ class Britney(object):
                 else:
                     uptodatebins = True
 
-                # if the package is architecture-dependent or the current arch is `nobreakall'
-                # find unsatisfied dependencies for the binary package
-                if binary_u.architecture != 'all' or arch in self.options.nobreakall_arches:
-                    is_valid = self.excuse_unsat_deps(pkg, src, arch, suite, excuse)
-                    inst_tester = self._inst_tester
-                    if not is_valid and inst_tester.any_of_these_are_in_testing({binary_u.pkg_id}) \
-                            and not inst_tester.is_installable(binary_u.pkg_id):
-                        # Forgive uninstallable packages only when
-                        # they are already broken in testing ideally
-                        # we would not need to be forgiving at
-                        # all. However, due to how arch:all packages
-                        # are handled, we do run into occasionally.
-                        excuse.policy_verdict = PolicyVerdict.REJECTED_PERMANENTLY
-
             # if there are out-of-date packages, warn about them in the excuse and set excuse.is_valid
             # to False to block the update; if the architecture where the package is out-of-date is
             # in the `outofsync_arches' list, then do not block the update

britney2/policies/policy.py

@@ -506,9 +506,6 @@ class RCBugPolicy(BasePolicy):
         old_bugs = rcbugs_info['unique-target-bugs']
         excuse.setbugs(old_bugs, new_bugs)
         if new_bugs:
-            excuse.addhtml("%s <a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?" \
-                           "src=%s&sev-inc=critical&sev-inc=grave&sev-inc=serious\" " \
-                           "target=\"_blank\">has new bugs</a>!" % (source_name, quote(source_name)))
             excuse.addhtml("Updating %s introduces new bugs: %s" % (source_name, ", ".join(
                 ["<a href=\"https://bugs.debian.org/%s\">#%s</a>" % (quote(a), a) for a in new_bugs])))
 

doc/conf.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# britney2 documentation build configuration file, created by
+# sphinx-quickstart on Sat Dec  2 12:34:27 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+# Add .. so sphinx can find the britney2 modules.
+sys.path.insert(0, os.path.abspath('..'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'britney2'
+copyright = '2017, Niels Thykier and others'
+author = 'Niels Thykier'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = ''
+# The full version, including alpha/beta/rc tags.
+release = ''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'classic'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+    '**': [
+        'relations.html',  # needs 'show_related': True theme option to display
+        'searchbox.html',
+    ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'britney2doc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'britney2.tex', 'britney2 Documentation',
+     'Niels Thykier', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'britney2', 'britney2 Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'britney2', 'britney2 Documentation',
+     author, 'britney2', 'One line description of project.',
+     'Miscellaneous'),
+]
+

doc/contributing-to-britney.rst

@@ -0,0 +1,40 @@
+Contributing to the development of britney2
+===========================================
+
+If you are interested in improving britney, you can obtain the source
+code via::
+
+  git clone https://anonscm.debian.org/git/mirror/britney2.git
+  # Additional tests
+  git clone https://anonscm.debian.org/git/collab-maint/britney2-tests.git
+
+You will need some packages to run britney and the test suites::
+
+  # Runtime dependencies
+  apt install python3 python3-apt python3-yaml
+  # Test dependencies
+  apt install python3-pytest libclass-accessor-perl rsync 
+  # Documentation generator
+  apt install python3-sphinx
+
+
+Britney has some basic unit tests, which are handled by py.test.  It
+also has some larger integration tests (from the `britney2-tests`
+repo).  Running the tests are done via::
+
+  cd britney2
+  # Basic unit tests
+  py.test-3
+  # Integration tests
+  rm -fr ./test-out/
+  ../britney2-tests/bin/runtests ./britney.py ../britney2-tests/t ./test-out
+
+The `runtests` command in `britney2-tests` supports running only a
+subset of the tests.  Please see its `--help` output for more
+information.
+
+Finally, there is also some heavier tests based on some snapshots of
+live data from Debian.  The data files for these are available in the
+`live-data` submodule of the `britney2-tests` repo.  They consume
+quite a bit of disk space and britney will need at least 1.3GB of RAM
+to process them.

doc/hints.rst

@@ -1,50 +1,30 @@
-#! TITLE: Britney hints documentation
+Hints
-#! SUBTITLE: How to override the britney policies
+=====
 
-# Britney hints
+This document describes the `britney hints` and the format of the hint
+file.  All hints are basically small instructions to britney.  The
+vast majority of them involve overriding a quality gating policy in
+britney.  However, there are a few hints that assist britney into
+finding solutions that it cannot compute itself.
 
-This document describes the "britney hints".  These are basically
+There are the following type of hints:
-small instructions to britney.  The vast majority of them involve
-overriding a quality gating policy in britney.  However, there are a
-few hints that assist britney into finding solutions that it cannot
-compute itself.
 
+ * Policy overrides
+ * Migration selections (with or without overrides)
+ * Other
 
-## Related configuration
+Please see :doc:`setting-up-britney` for how to configure hint files
+and for how to limit which hints are allowed in a given hint file.
 
-The configuration `HINTSDIR` determines which directory contains the
+Format of the hint file
-so called "hints files".  This is complimented with the `HINTS_<NAME>`
+-----------------------
-configurations that defines a "hint file" and the related hint
-permissions for it.
 
-For each `HINTS_<NAME>` configuration, britney will attempt to read
+All hints are read from hint files. The hint file is a plain text file
-`<HINTSDIR>/<name>`.  Note that it lowercases `<NAME>` when looking
+with line-based content.  Empty and whitespace-only lines are ignored.
-for the file.
+If the first (non-whitespace) character is a `#`, then britney
-
+considers it a comment and ignores it.  However, other tooling may
-
+interpret these comments (as is the case for e.g. some parts of the
-Configuration example:
+Debian infrastructure).
-
-    HINTSDIR = /etc/britney2/hints
-    HINTS_ANNA = ALL
-    HINTS_JOHN = STANDARD
-    HINTS_FREEZE = block block-all block-udeb
-    HINTS_AUTO-REMOVALS = remove
-
-There are no fixed rules for how to use hints files.  Though usually,
-each person with permissions to give hints to britney will have their
-own hint file along with write permissions for that file.  It can also
-make sense to create hint files for "roles".  Like in the above
-example there are two human hints (anna and john) plus two non-human
-hinters (freeze and auto-removals).
-
-
-## Format of the hint file
-
-The hint file is a plain text file with line-based content.  Empty and
-whitespace-only lines are ignored.  If the first (non-whitespace)
-character is a `#`, then britney considers it a comment and ignores
-it.  However, other tooling may interpret these comments (as is the
-case for e.g. some parts of the Debian infrastructure).
 
 The remaining lines are considered space-separated lists, where the
 first element must be a known hint.  The remaining elements will be
@@ -65,16 +45,8 @@ generally refers to the removal of said item rather than the migration
 of the hint.
 
 
-# Hints
+Policy override hints
-
+---------------------
-There are the following type of hints:
-
- * Policy overrides
- * Migration selections (with or without overrides)
- * Other
-
-
-## Policy override hints
 
 The policy override hints are used to disable or tweak various
 policies in britney.  Their effects are generally very precise ways of
@@ -84,7 +56,9 @@ Some of these items are built-in while others are related to specific
 policies.  In the latter case, they are only valid if the given policy
 is enabled (as the policy registers them when it is enabled).
 
-### block-all `<type>`
+
+block-all `<type>`
+^^^^^^^^^^^^^^^^^^
 
 Usually used during freezes to prevent migrations by default.
 
@@ -92,16 +66,19 @@ The `<type>` can be one of:
 
  * `source`: Blocks all source migrations.  This is a superset of
    `new-source`.
+
  * `new-source`: Block source migrations if the given source is not
-    already in the target suite.  (Side-effect: Removed packages will
+   already in the target suite.  (Side-effect: Removed packages will
-    not re-enter the taget suite automatically).
+   not re-enter the taget suite automatically).
 
 All variants of these can be overruled by a valid `unblock`-hint.
 
 Note that this does not and cannot restrict architecture specific
 migrations (e.g. binNMUs or first time builds for architectures).
 
-### block `<action list>`, block-udeb `<action list>`
+
+block `<action list>`, block-udeb `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Prevent the items in the `<action list>` from migrating until the hint
 is removed or overruled by the equivalent unblock hint (or a
@@ -117,7 +94,9 @@ release cycle.
 Note that this does not and cannot restrict architecture specific
 migrations (e.g. binNMUs or first time builds for architectures).
 
-### unblock `<action list>`, unblock-udeb `<action list>`
+
+unblock `<action list>`, unblock-udeb `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Enable the items in `<action list>` to migrate by overriding `block`-,
 `block-all`- or `block-udeb`-hints.  The `unblock`-hint (often under
@@ -133,7 +112,9 @@ release cycle.
 The two types of block hint must be paired with their corresponding
 unblock hint - i.e. an `unblock-udeb` does not override a `block`.
 
-### approve `<action list>`
+
+approve `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^
 
 A synonym of `unblock`.  The variant is generally used for approving
 migrations from suites that require approvals.
@@ -142,7 +123,9 @@ Aside from the tab-completion in the hint testing interface, which
 will give different suggestions to `approve` and `unblock`, the rest
 of britney will consider them identical.
 
-### age-days `<days>` `<action list>`
+
+age-days `<days>` `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Set the length of time which the listed action items must have been in
 unstable before they can be considered candidates.  This may be used
@@ -154,7 +137,9 @@ whichever is encountered first during parsing overrides the others.
 
 Provided by the `age` policy.
 
-### urgent `<action list>`
+
+urgent `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^
 
 Approximately equivalent to `age-days 0 <action list>`, with the
 distinction that an "urgent" hint overrides any "age-days" hint for
@@ -162,7 +147,9 @@ the same action item.
 
 Provided by the `age` policy.
 
-### ignore-rc-bugs `<bugs>` `<action list>`
+
+ignore-rc-bugs `<bugs>` `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The `<bugs>` argument is a comma separated list <bugs> of bugs that
 affect the items in `<action list>`.  Britney will ignore these bugs
@@ -175,7 +162,8 @@ migration item.
 
 Provided by the `bugs` policy
 
-### ignore-piuparts `<action list>`
+ignore-piuparts `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The items in `<action list>` will not be blocked by regressions in
 results from piuparts tests.  All items in `<action list>` must be
@@ -183,7 +171,9 @@ versioned items.
 
 Provided by the `piuparts` policy
 
-### force `<action list>`
+
+force `<action list>`
+^^^^^^^^^^^^^^^^^^^^^
 
 Override all policies that claim the items in `<action list>` have
 regressions or are otherwise not ready to migrate.  All items in the
@@ -194,7 +184,9 @@ This hint does not guarantee that they will migrate.  To ensure that,
 you will have to combine it with a `force-hint`.  However, please read
 the warning in the documentation for `force-hint` before you do this.
 
-## Migration selection hints
+
+Migration selection hints
+-------------------------
 
 All migration selection hints work on an "action list".  This consists
 of at least 1 or more of the following (in any combination):
@@ -207,7 +199,9 @@ All elements in the action list must be valid at the time the hint is
 attempted.  Notably, if one action has already been completed, the
 entire hint is rejected as invalid.
 
-### easy `<action list>`
+
+easy `<action list>`
+^^^^^^^^^^^^^^^^^^^^
 
 Perform all the migrations and removals denoted by `<action list>` as if
 it were a single migration group.  If the end result is equal or better
@@ -221,7 +215,8 @@ Note that for `easy` the `<action list>` must have at least two
 elements.  There is no use-case where a single element for easy will
 make sense (as britney always tries those).
 
-### hint `<action list>`
+hint `<action list>`
+^^^^^^^^^^^^^^^^^^^^
 
 Perform all the migrations and removals denoted by `<action list>` as if
 it were a single migration group.  After that, process all remaining
@@ -246,7 +241,9 @@ undesireable changes to the target suite.  In practise, this is rather
 rare but the hinter is letting britney decide what "repairs" the
 situation.
 
-### force-hint `<action list>`
+
+force-hint `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The provided `<action list>` is migrated as-is regardless of what is
 broken by said migration.  This often needs to be paired with a
@@ -260,11 +257,13 @@ desirable than the resulting breakage.
 change can have long lasting undesireable consequences on the end
 result.
 
-## Other hints
+Other hints
+-----------
 
 This section cover hints that have no other grouping.
 
-### remove `<action list>`
+remove `<action list>`
+^^^^^^^^^^^^^^^^^^^^^^
 
 Britney should attempt to remove all items in the `<action list>` from
 the target suite.  The `<action list>` must consist entirely of

doc/index.rst

@@ -0,0 +1,25 @@
+.. britney2 documentation master file, created by
+   sphinx-quickstart on Sat Dec  2 12:34:27 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to britney2's documentation!
+====================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   what-is-britney
+   short-intro-to-migrations
+   solutions-to-common-policy-issues
+   contributing-to-britney
+   hints
+   setting-up-britney
+
+* :ref:`search`
+
+.. Add these later if/when needed:
+..   * :ref:`genindex`
+..   * :ref:`modindex`
+

doc/setting-up-britney.rst

@@ -0,0 +1,77 @@
+How to setup britney
+====================
+
+This document describes how to install, configure and run britney in
+your infrastructure.
+
+Installing britney
+------------------
+
+TODO
+
+Configuring britney
+-------------------
+
+TODO
+
+hints - Configuring who can provide which hints
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Britney reads all hints from a set of `hints` files.  These files must
+be placed in the directory denoted by the `HINTSDIR` configuration.
+This is complimented with the `HINTS_<NAME>` configurations that
+defines a "hint file" and the related hint permissions for it.
+
+For each `HINTS_<NAME>` configuration, britney will attempt to read
+`<HINTSDIR>/<name>`.  Note that it lowercases `<NAME>` when looking
+for the file.
+
+
+Configuration example::
+
+    HINTSDIR = /etc/britney2/hints
+    HINTS_ANNA = ALL
+    HINTS_JOHN = STANDARD
+    HINTS_FREEZE = block block-all block-udeb
+    HINTS_AUTO-REMOVALS = remove
+
+In the above example, we have defined 4 hints files named `anna`,
+`john`, `freeze` and `auto-removals`.  These must be placed in
+`/etc/britney2/hints` and be readable by britney.  Furthermore, they
+must be writable by (only) the people that are allowed to use the
+particular hints file (apply `chown`, `chmod` and `setfacl` as
+neccesary).
+
+The values on the right hand side of the `=` decides which hints are
+permitted in the files.  With the above definitions:
+
+ * The file `anna` may use any known hint (including potentially
+   dangerous ones like `force` and `force-hint`)
+
+ * The file `john` may use most of the known hints.  The set called STANDARD
+   includes a lot of hints for overriding most policies when it
+   can be done without (additional) side-effects.  However, it
+   excludes `force` and `force-hint` as they can cause unintentional
+   results.
+
+ * The file `freeze` can use any of the hints `block`, `block-all`
+   and `block-udeb`.
+
+ * The file `auto-removals` can only use the hint called `remove`.
+
+There are no fixed rules for how to use hints files.  Though usually,
+each person with permissions to give hints to britney will have their
+own hint file along with write permissions for that file.  It can also
+make sense to create hint files for "roles".  Like in the above
+example there are two human hinters (`anna` and `john`) plus two
+non-human hinters (`freeze` and `auto-removals`).
+
+Please see :doc:`hints` for which hints are available and what they
+can do.
+
+
+Using the results from britney
+------------------------------
+
+TODO
+

doc/short-intro-to-migrations.rst

@@ -1,13 +1,11 @@
-#! TITLE: Britney migration documentation
+A short introduction to migrations
-#! SUBTITLE: Understanding britney's workflow
+==================================
-
-# Migrations
 
 This is a technical introduction to how britney
 handles migrations.  Being an introduction, it deliberately
 oversimplifies certain things at the expense of accuracy.
 It also covers common migration issues and how to fix
-them.
+them in :doc:`solutions-to-common-policy-issues`.
 
 The document is primarily aimed at contributors for
 distributions that want to understand the basics of
@@ -21,7 +19,8 @@ abstract.  Furthermore, the document assumes familiarity with
 Debian-based distribution practises and terminology (such as
 "suites" and "source package").
 
-## A high level overview of britney and migrations
+A high level overview of britney and migrations
+-----------------------------------------------
 
 The purpose of britney is to (semi-)automatically select
 a number of migration items from a series of source suites
@@ -34,9 +33,12 @@ of the following points:
  1. The migration items pass a number of policies for the target
     suite.  Most of these policies are basically that the
     migration items do not regress on selected QA checks.
+    
     * An item satisfying this part is called a `valid candiate`.
- 1. Installability will not regress as a result of
+
+ 2. Installability will not regress as a result of
     migrating the migration items.
+
     * An item that (also) satisfies this part will be selected
       for migration.
 
@@ -48,14 +50,15 @@ issue (as it is not a regression).
 This only leaves the definition of a migration items.  They come
 in several variants defined in the next section.
 
-## Migration items
+Migration items
+---------------
 
 Internally, britney groups packages into migration items based on a
 few rules.  There are several kinds of migration items and this
 document will only describe the source migration item.
 
->  A source migration item is one upload of a source package, with
+   A source migration item is one upload of a source package, with
->  associated binary packages once built.
+   associated binary packages once built.
 
 Once a new version of a source package appears in the source suite,
 britney will create track it with a source migration item.  As the
@@ -72,7 +75,8 @@ As implied earlier, there are several other migration types.  But they
 are not covered in this document.  They deal with cases like removals,
 rebuilds of existing binaries, etc.
 
-## Migration phase 1: Policies / Excuses
+Migration phase 1: Policies / Excuses
+-------------------------------------
 
 To begin with, britney will apply a number of policies to
 all migration items.  Each policy will rate each migration
@@ -100,19 +104,25 @@ data that britney has access to.  This mean that without any change
 to the items themselves:
 
  1. Items that passed originally may fail in a later britney run.
- 1. Likewise, items may go from a "permanent failure" to a pass.
+
+ 2. Likewise, items may go from a "permanent failure" to a pass.
 
 This can be seen in the following example case:
 
  1. A new version of package is uploaded.
+
     * Britney processes the package and concludes that there no blocking bugs,
       so the package passes the bug policy.
- 1. Then before it migrates, someone files a blocking bug against
+
+ 2. Then before it migrates, someone files a blocking bug against
     the new version.
+
     * Britney reprocesses the package and now concludes it has a regression in
       the bug policy (i.e. the policy verdict goes from "pass" to "permanent fail").
- 1. The bug is examined and it is determined that the bug also affects the
+
+ 3. The bug is examined and it is determined that the bug also affects the
     version in the target suite.  The bug tracker is updated to reflect this.
+
     * Britney reprocesses the package again and now concludes there is a blocking
       bug, but it is not a regression (since it also affects the target suite).
       This means the policy verdict now go from "fail" to "pass".
@@ -131,7 +141,8 @@ a hinted migration generally implies that the problem will not
 prevent future migrations for newer versions of the same source
 package (assuming that the problem is deterministic).
 
-## Migration phase 2: Installability regression testing
+Migration phase 2: Installability regression testing
+----------------------------------------------------
 
 For the migration items that pass the previous phase, britney
 will do a test migration to see if anything becomes uninstallable.
@@ -143,11 +154,12 @@ problems here, the britney log file has to be examined.  This requires
 a bit more technical insight as it has not been polished as much as
 the excuses.
 
-### Confirming a migration
+Confirming a migration
+^^^^^^^^^^^^^^^^^^^^^^
 
 To start with; if a migration is accepted and "committed" (i.e. it will not
 be rolled back), britney will include in a line starting with `final:` like
-in this example:
+in this example::
 
     Apparently successful
     final: -cwltool,-libtest-redisserver-perl,-pinfo,-webdis,hol88
@@ -173,12 +185,13 @@ Reminder: Migration items generally use the name of the source package.  There
 are exceptions to that "rule" (but they are not common cases covered by this
 document).
 
-### Debugging failed migration attempts
+Debugging failed migration attempts
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Start by confirming that the migration item was not accepted (as described
 in the above section).  If the migration item does not appear on a `final:` line,
 then we need to debug the actual migration attempts.  Migration attempts look
-something like this:
+something like this::
 
     trying: -webdis
     accepted: -webdis
@@ -214,8 +227,9 @@ later attempt.  It is not always immediately obvious, which attempt is better
 for debugging.  When in doubt, it is *usually* easiest to look at the attempt
 with the least amount of new uninstallable packages.
 
-In the libaws example, a total of 4 binary packages become uninstallable on the
+In the libaws example, a total of 4 binary packages become
-architecture `arm64`.  Here is the output again with this information high lighted:
+uninstallable on the architecture `arm64`.  Here is the output again
+with this information high lighted::
 
     migration item(s) being attemped
             vvvvvv
@@ -230,7 +244,7 @@ architecture `arm64`.  Here is the output again with this information high light
 Please note that britney is lazy and will often reject an item after proving
 that there is a regression on a single architecture.  So in the above example,
 we are not actually sure whether this problem is architecture specific.  For
-`easy`-hints, the information is presented slightly different.
+`easy`-hints, the information is presented slightly different::
 
     Trying easy from autohinter: asis/2017-1 dh-ada-library/6.12 [...]
                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -255,103 +269,3 @@ things break.  If there are few broken packages, it is often a question of
 looking for `Breaks`-relations or `Depends`-relations with upper bounds on
 versions / on old packages being removed.  Alternatively, there are also tools
 like `dose-debcheck`, which attempts to analyse and explain problems like this.
-
-# Common issues with policy decisions
-
-## Britney complains about a fixed bug in the source suite (bug policy)
-
-All decisions about bugs are related to data set extracted
-from the bug tracker.  If britney says that the new version
-introduces a bug, then it is because the data set from the bug
-tracker lists that bug for *a* version in the source suite and
-without it appearing for the version(s) in the target suite.
-
-Please note that these data sets do not include versions, so
-britney is unable to tell exactly which versions are affected.
-The only thing, it can tell, is what suite the bug affects.
-
-There is a number of common cases, where this is observed:
-
- * The metadata on the bug is wrong.  A known example is the
-   Debian BTS, where if a bug has a `fixed` version equal to
-   a `found` version, the bug is considered unfixed.
-
- * The bug is fixed, but the old version is still around in
-   the source suite.  In this case, britney will generally
-   mention a "missing build" or "old binaries".
-
-If the metadata is wrong, the solution is to fix it in the bug
-tracker and wait until britney receives a new data set.  In
-the other case, the recommendation is to see the sections on
-"missing builds" and "old binaries" below.  As long as they
-are present, the package may be blocked by bugs in the older
-versions of the binaries.
-
-## Britney complains about "missing builds"
-
-A "missing build" happens when britney detects that the binaries
-for a given architecture are missing or is not up to date.  This
-is detected by checking the "Packages" files in the archive, so
-britney have no knowledge of *why* the build is missing.
-Accordingly, this kind of issue is flagged as a "possibly permanent"
-issue.
-
-If the omission is deliberate (e.g. the new version no longer
-supports that architecture), then please have the old binaries
-for that architecture removed from the *source* suite.  Once
-those are removed, britney will no longer see that as a problem.
-
-Otherwise, please check the build services for any issues with
-building or uploading the package to the archive.
-
-**Common misconceptions**: If the architecture is no longer
-supported, the removal of the old binaries should happen in
-the *source* suite (e.g. Debian unstable).  However, many
-people mistakenly request a removal from the *target* suite
-(e.g. Debian testing).  Unfortunately, this is not the proper
-solution (and, britney does not support architecture
-specific removals so it may be difficult to do anyhow).
-
-## Britney complains about "old binaries"
-
-Depending on the configuration of the britney instance, this may
-or may not be a blocker.  If the distribution has chosen to enable
-the "ignore_cruft" option, this is merely a warning/note.  That
-said, even in this mode it can block a package from migration.
-
-This appears when britney detects that there are older versions of
-the binary packages around, which was built by (an older version of)
-the same source package.
-
-This is common with distributions where their archive management
-software is capable of keeping old binaries as long as something
-depends on them (e.g. DAK as used by Debian).  Therefore, the
-most common solution is to ensure all reverse dependencies are
-updated to use the new binaries and then have the old ones
-removed (the latter commonly known as "decrufting").  Technically,
-this is also solvable by "decrufting" without updating/rebuilding
-other packages.  Though whether this is an acceptable practise
-depends on the distribution.
-
-Alternatively, if the distribution uses the "ignore_cruft" option,
-this (in itself) is not a blocker.  However, it commonly triggers
-non-obvious issues:
-
- * If the bugs policy is enabled, an bug in the old binaries that
-   is fixed in the new version will still be a blocker.  Here, the
-   best solution is to get rid of the old binaries.
-   
-   (Note: the bugs data is not versioned so britney cannot tell which
-    versions the bug applies to.  Just which suite they affect)
-
- * Even if the migration item is a valid candidate (i.e. all policy
-   checked have passed), it may cause installability regressions as
-   britney will also attempt to keep the old binaries around as long
-   as they are used.  The most often cause of this when the old
-   binaries are not co-installable with the new ones.
-   
-   (Note: Britney generally only works with the highest version of a
-    given binary.  If you have libfoo1 depends on libfoo-data v1 and
-    then libfoo2 depends on libfoo-data v2, then libfoo1 will become
-    uninstallable as libfoo-data v2 will "shadow" libfoo-data v1)
-

doc/solutions-to-common-policy-issues.rst

@@ -0,0 +1,105 @@
+Solutions to common policy decisions
+====================================
+
+.. contents::
+
+Britney complains about a fixed bug in the source suite (bug policy)
+--------------------------------------------------------------------
+
+All decisions about bugs are related to data set extracted
+from the bug tracker.  If britney says that the new version
+introduces a bug, then it is because the data set from the bug
+tracker lists that bug for *a* version in the source suite and
+without it appearing for the version(s) in the target suite.
+
+Please note that these data sets do not include versions, so
+britney is unable to tell exactly which versions are affected.
+The only thing, it can tell, is what suite the bug affects.
+
+There is a number of common cases, where this is observed:
+
+ * The metadata on the bug is wrong.  A known example is the
+   Debian BTS, where if a bug has a `fixed` version equal to
+   a `found` version, the bug is considered unfixed.
+
+ * The bug is fixed, but the old version is still around in
+   the source suite.  In this case, britney will generally
+   mention a "missing build" or "old binaries".
+
+If the metadata is wrong, the solution is to fix it in the bug
+tracker and wait until britney receives a new data set.  In
+the other case, the recommendation is to see the sections on
+"missing builds" and "old binaries" below.  As long as they
+are present, the package may be blocked by bugs in the older
+versions of the binaries.
+
+Britney complains about "missing builds"
+----------------------------------------
+
+A "missing build" happens when britney detects that the binaries
+for a given architecture are missing or is not up to date.  This
+is detected by checking the "Packages" files in the archive, so
+britney have no knowledge of *why* the build is missing.
+Accordingly, this kind of issue is flagged as a "possibly permanent"
+issue.
+
+If the omission is deliberate (e.g. the new version no longer
+supports that architecture), then please have the old binaries
+for that architecture removed from the *source* suite.  Once
+those are removed, britney will no longer see that as a problem.
+
+Otherwise, please check the build services for any issues with
+building or uploading the package to the archive.
+
+**Common misconceptions**: If the architecture is no longer
+supported, the removal of the old binaries should happen in
+the *source* suite (e.g. Debian unstable).  However, many
+people mistakenly request a removal from the *target* suite
+(e.g. Debian testing).  Unfortunately, this is not the proper
+solution (and, britney does not support architecture
+specific removals so it may be difficult to do anyhow).
+
+Britney complains about "old binaries"
+--------------------------------------
+
+Depending on the configuration of the britney instance, this may
+or may not be a blocker.  If the distribution has chosen to enable
+the "ignore_cruft" option, this is merely a warning/note.  That
+said, even in this mode it can block a package from migration.
+
+This appears when britney detects that there are older versions of
+the binary packages around, which was built by (an older version of)
+the same source package.
+
+This is common with distributions where their archive management
+software is capable of keeping old binaries as long as something
+depends on them (e.g. DAK as used by Debian).  Therefore, the
+most common solution is to ensure all reverse dependencies are
+updated to use the new binaries and then have the old ones
+removed (the latter commonly known as "decrufting").  Technically,
+this is also solvable by "decrufting" without updating/rebuilding
+other packages.  Though whether this is an acceptable practise
+depends on the distribution.
+
+Alternatively, if the distribution uses the "ignore_cruft" option,
+this (in itself) is not a blocker.  However, it commonly triggers
+non-obvious issues:
+
+ * If the bugs policy is enabled, an bug in the old binaries that
+   is fixed in the new version will still be a blocker.  Here, the
+   best solution is to get rid of the old binaries.
+   
+   * Note: the bugs data is not versioned so britney cannot tell which
+     versions the bug applies to.  Just which suite they affect.
+
+ * Even if the migration item is a valid candidate (i.e. all policy
+   checked have passed), it may cause installability regressions as
+   britney will also attempt to keep the old binaries around as long
+   as they are used.  The most often cause of this when the old
+   binaries are not co-installable with the new ones.
+   
+   * Note: Britney generally only works with the highest version of a
+     given binary.  If you have libfoo1 depends on libfoo-data v1 and
+     then libfoo2 depends on libfoo-data v2, then libfoo1 will become
+     uninstallable as libfoo-data v2 will "shadow" libfoo-data v1.
+

doc/what-is-britney.rst

@@ -0,0 +1,13 @@
+What is britney and britney2
+============================
+
+Britney2 is a program that can be used to `migrate` debian-based
+packages from one suite to another.  It is the tool used by Debian and
+Ubuntu to choose which packages are ready for wider testing.
+
+Britney2 is the reimplementation and replacement of the original
+britney program.  Through out this documentation, any mention of
+britney generally refers to the britney2 implementation.  When there
+is a need to reference the original britney implementation, this
+documentation will use britney1.
+