Rewrite documentation in sphinx

Signed-off-by: Niels Thykier <niels@thykier.net>

doc/conf.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# britney2 documentation build configuration file, created by
+# sphinx-quickstart on Sat Dec  2 12:34:27 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+# Add .. so sphinx can find the britney2 modules.
+sys.path.insert(0, os.path.abspath('..'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'britney2'
+copyright = '2017, Niels Thykier and others'
+author = 'Niels Thykier'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = ''
+# The full version, including alpha/beta/rc tags.
+release = ''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'classic'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+    '**': [
+        'relations.html',  # needs 'show_related': True theme option to display
+        'searchbox.html',
+    ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'britney2doc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'britney2.tex', 'britney2 Documentation',
+     'Niels Thykier', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'britney2', 'britney2 Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'britney2', 'britney2 Documentation',
+     author, 'britney2', 'One line description of project.',
+     'Miscellaneous'),
+]
+

doc/contributing-to-britney.rst

@@ -0,0 +1,40 @@
+Contributing to the development of britney2
+===========================================
+
+If you are interested in improving britney, you can obtain the source
+code via::
+
+  git clone https://anonscm.debian.org/git/mirror/britney2.git
+  # Additional tests
+  git clone https://anonscm.debian.org/git/collab-maint/britney2-tests.git
+
+You will need some packages to run britney and the test suites::
+
+  # Runtime dependencies
+  apt install python3 python3-apt python3-yaml
+  # Test dependencies
+  apt install python3-pytest libclass-accessor-perl rsync 
+  # Documentation generator
+  apt install python3-sphinx
+
+
+Britney has some basic unit tests, which are handled by py.test.  It
+also has some larger integration tests (from the `britney2-tests`
+repo).  Running the tests are done via::
+
+  cd britney2
+  # Basic unit tests
+  py.test-3
+  # Integration tests
+  rm -fr ./test-out/
+  ../britney2-tests/bin/runtests ./britney.py ../britney2-tests/t ./test-out
+
+The `runtests` command in `britney2-tests` supports running only a
+subset of the tests.  Please see its `--help` output for more
+information.
+
+Finally, there is also some heavier tests based on some snapshots of
+live data from Debian.  The data files for these are available in the
+`live-data` submodule of the `britney2-tests` repo.  They consume
+quite a bit of disk space and britney will need at least 1.3GB of RAM
+to process them.

doc/index.rst

@@ -0,0 +1,23 @@
+.. britney2 documentation master file, created by
+   sphinx-quickstart on Sat Dec  2 12:34:27 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to britney2's documentation!
+====================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   what-is-britney
+   short-intro-to-migrations
+   solutions-to-common-policy-issues
+   contributing-to-britney
+
+* :ref:`search`
+
+.. Add these later if/when needed:
+..   * :ref:`genindex`
+..   * :ref:`modindex`
+

doc/short-intro-to-migrations.rst

@@ -1,13 +1,11 @@
-#! TITLE: Britney migration documentation
+A short introduction to migrations
-#! SUBTITLE: Understanding britney's workflow
+==================================
-
-# Migrations
 
 This is a technical introduction to how britney
 handles migrations.  Being an introduction, it deliberately
 oversimplifies certain things at the expense of accuracy.
 It also covers common migration issues and how to fix
-them.
+them in :doc:`solutions-to-common-policy-issues`.
 
 The document is primarily aimed at contributors for
 distributions that want to understand the basics of
@@ -21,7 +19,8 @@ abstract.  Furthermore, the document assumes familiarity with
 Debian-based distribution practises and terminology (such as
 "suites" and "source package").
 
-## A high level overview of britney and migrations
+A high level overview of britney and migrations
+-----------------------------------------------
 
 The purpose of britney is to (semi-)automatically select
 a number of migration items from a series of source suites
@@ -34,9 +33,12 @@ of the following points:
  1. The migration items pass a number of policies for the target
     suite.  Most of these policies are basically that the
     migration items do not regress on selected QA checks.
+    
     * An item satisfying this part is called a `valid candiate`.
- 1. Installability will not regress as a result of
+
+ 2. Installability will not regress as a result of
     migrating the migration items.
+
     * An item that (also) satisfies this part will be selected
       for migration.
 
@@ -48,14 +50,15 @@ issue (as it is not a regression).
 This only leaves the definition of a migration items.  They come
 in several variants defined in the next section.
 
-## Migration items
+Migration items
+---------------
 
 Internally, britney groups packages into migration items based on a
 few rules.  There are several kinds of migration items and this
 document will only describe the source migration item.
 
->  A source migration item is one upload of a source package, with
+   A source migration item is one upload of a source package, with
->  associated binary packages once built.
+   associated binary packages once built.
 
 Once a new version of a source package appears in the source suite,
 britney will create track it with a source migration item.  As the
@@ -72,7 +75,8 @@ As implied earlier, there are several other migration types.  But they
 are not covered in this document.  They deal with cases like removals,
 rebuilds of existing binaries, etc.
 
-## Migration phase 1: Policies / Excuses
+Migration phase 1: Policies / Excuses
+-------------------------------------
 
 To begin with, britney will apply a number of policies to
 all migration items.  Each policy will rate each migration
@@ -100,19 +104,25 @@ data that britney has access to.  This mean that without any change
 to the items themselves:
 
  1. Items that passed originally may fail in a later britney run.
- 1. Likewise, items may go from a "permanent failure" to a pass.
+
+ 2. Likewise, items may go from a "permanent failure" to a pass.
 
 This can be seen in the following example case:
 
  1. A new version of package is uploaded.
+
     * Britney processes the package and concludes that there no blocking bugs,
       so the package passes the bug policy.
- 1. Then before it migrates, someone files a blocking bug against
+
+ 2. Then before it migrates, someone files a blocking bug against
     the new version.
+
     * Britney reprocesses the package and now concludes it has a regression in
       the bug policy (i.e. the policy verdict goes from "pass" to "permanent fail").
- 1. The bug is examined and it is determined that the bug also affects the
+
+ 3. The bug is examined and it is determined that the bug also affects the
     version in the target suite.  The bug tracker is updated to reflect this.
+
     * Britney reprocesses the package again and now concludes there is a blocking
       bug, but it is not a regression (since it also affects the target suite).
       This means the policy verdict now go from "fail" to "pass".
@@ -131,7 +141,8 @@ a hinted migration generally implies that the problem will not
 prevent future migrations for newer versions of the same source
 package (assuming that the problem is deterministic).
 
-## Migration phase 2: Installability regression testing
+Migration phase 2: Installability regression testing
+----------------------------------------------------
 
 For the migration items that pass the previous phase, britney
 will do a test migration to see if anything becomes uninstallable.
@@ -143,11 +154,12 @@ problems here, the britney log file has to be examined.  This requires
 a bit more technical insight as it has not been polished as much as
 the excuses.
 
-### Confirming a migration
+Confirming a migration
+^^^^^^^^^^^^^^^^^^^^^^
 
 To start with; if a migration is accepted and "committed" (i.e. it will not
 be rolled back), britney will include in a line starting with `final:` like
-in this example:
+in this example::
 
     Apparently successful
     final: -cwltool,-libtest-redisserver-perl,-pinfo,-webdis,hol88
@@ -173,12 +185,13 @@ Reminder: Migration items generally use the name of the source package.  There
 are exceptions to that "rule" (but they are not common cases covered by this
 document).
 
-### Debugging failed migration attempts
+Debugging failed migration attempts
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Start by confirming that the migration item was not accepted (as described
 in the above section).  If the migration item does not appear on a `final:` line,
 then we need to debug the actual migration attempts.  Migration attempts look
-something like this:
+something like this::
 
     trying: -webdis
     accepted: -webdis
@@ -214,8 +227,9 @@ later attempt.  It is not always immediately obvious, which attempt is better
 for debugging.  When in doubt, it is *usually* easiest to look at the attempt
 with the least amount of new uninstallable packages.
 
-In the libaws example, a total of 4 binary packages become uninstallable on the
+In the libaws example, a total of 4 binary packages become
-architecture `arm64`.  Here is the output again with this information high lighted:
+uninstallable on the architecture `arm64`.  Here is the output again
+with this information high lighted::
 
     migration item(s) being attemped
             vvvvvv
@@ -230,7 +244,7 @@ architecture `arm64`.  Here is the output again with this information high light
 Please note that britney is lazy and will often reject an item after proving
 that there is a regression on a single architecture.  So in the above example,
 we are not actually sure whether this problem is architecture specific.  For
-`easy`-hints, the information is presented slightly different.
+`easy`-hints, the information is presented slightly different::
 
     Trying easy from autohinter: asis/2017-1 dh-ada-library/6.12 [...]
                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -255,103 +269,3 @@ things break.  If there are few broken packages, it is often a question of
 looking for `Breaks`-relations or `Depends`-relations with upper bounds on
 versions / on old packages being removed.  Alternatively, there are also tools
 like `dose-debcheck`, which attempts to analyse and explain problems like this.
-
-# Common issues with policy decisions
-
-## Britney complains about a fixed bug in the source suite (bug policy)
-
-All decisions about bugs are related to data set extracted
-from the bug tracker.  If britney says that the new version
-introduces a bug, then it is because the data set from the bug
-tracker lists that bug for *a* version in the source suite and
-without it appearing for the version(s) in the target suite.
-
-Please note that these data sets do not include versions, so
-britney is unable to tell exactly which versions are affected.
-The only thing, it can tell, is what suite the bug affects.
-
-There is a number of common cases, where this is observed:
-
- * The metadata on the bug is wrong.  A known example is the
-   Debian BTS, where if a bug has a `fixed` version equal to
-   a `found` version, the bug is considered unfixed.
-
- * The bug is fixed, but the old version is still around in
-   the source suite.  In this case, britney will generally
-   mention a "missing build" or "old binaries".
-
-If the metadata is wrong, the solution is to fix it in the bug
-tracker and wait until britney receives a new data set.  In
-the other case, the recommendation is to see the sections on
-"missing builds" and "old binaries" below.  As long as they
-are present, the package may be blocked by bugs in the older
-versions of the binaries.
-
-## Britney complains about "missing builds"
-
-A "missing build" happens when britney detects that the binaries
-for a given architecture are missing or is not up to date.  This
-is detected by checking the "Packages" files in the archive, so
-britney have no knowledge of *why* the build is missing.
-Accordingly, this kind of issue is flagged as a "possibly permanent"
-issue.
-
-If the omission is deliberate (e.g. the new version no longer
-supports that architecture), then please have the old binaries
-for that architecture removed from the *source* suite.  Once
-those are removed, britney will no longer see that as a problem.
-
-Otherwise, please check the build services for any issues with
-building or uploading the package to the archive.
-
-**Common misconceptions**: If the architecture is no longer
-supported, the removal of the old binaries should happen in
-the *source* suite (e.g. Debian unstable).  However, many
-people mistakenly request a removal from the *target* suite
-(e.g. Debian testing).  Unfortunately, this is not the proper
-solution (and, britney does not support architecture
-specific removals so it may be difficult to do anyhow).
-
-## Britney complains about "old binaries"
-
-Depending on the configuration of the britney instance, this may
-or may not be a blocker.  If the distribution has chosen to enable
-the "ignore_cruft" option, this is merely a warning/note.  That
-said, even in this mode it can block a package from migration.
-
-This appears when britney detects that there are older versions of
-the binary packages around, which was built by (an older version of)
-the same source package.
-
-This is common with distributions where their archive management
-software is capable of keeping old binaries as long as something
-depends on them (e.g. DAK as used by Debian).  Therefore, the
-most common solution is to ensure all reverse dependencies are
-updated to use the new binaries and then have the old ones
-removed (the latter commonly known as "decrufting").  Technically,
-this is also solvable by "decrufting" without updating/rebuilding
-other packages.  Though whether this is an acceptable practise
-depends on the distribution.
-
-Alternatively, if the distribution uses the "ignore_cruft" option,
-this (in itself) is not a blocker.  However, it commonly triggers
-non-obvious issues:
-
- * If the bugs policy is enabled, an bug in the old binaries that
-   is fixed in the new version will still be a blocker.  Here, the
-   best solution is to get rid of the old binaries.
-   
-   (Note: the bugs data is not versioned so britney cannot tell which
-    versions the bug applies to.  Just which suite they affect)
-
- * Even if the migration item is a valid candidate (i.e. all policy
-   checked have passed), it may cause installability regressions as
-   britney will also attempt to keep the old binaries around as long
-   as they are used.  The most often cause of this when the old
-   binaries are not co-installable with the new ones.
-   
-   (Note: Britney generally only works with the highest version of a
-    given binary.  If you have libfoo1 depends on libfoo-data v1 and
-    then libfoo2 depends on libfoo-data v2, then libfoo1 will become
-    uninstallable as libfoo-data v2 will "shadow" libfoo-data v1)
-

doc/solutions-to-common-policy-issues.rst

@@ -0,0 +1,105 @@
+Solutions to common policy decisions
+====================================
+
+.. contents::
+
+Britney complains about a fixed bug in the source suite (bug policy)
+--------------------------------------------------------------------
+
+All decisions about bugs are related to data set extracted
+from the bug tracker.  If britney says that the new version
+introduces a bug, then it is because the data set from the bug
+tracker lists that bug for *a* version in the source suite and
+without it appearing for the version(s) in the target suite.
+
+Please note that these data sets do not include versions, so
+britney is unable to tell exactly which versions are affected.
+The only thing, it can tell, is what suite the bug affects.
+
+There is a number of common cases, where this is observed:
+
+ * The metadata on the bug is wrong.  A known example is the
+   Debian BTS, where if a bug has a `fixed` version equal to
+   a `found` version, the bug is considered unfixed.
+
+ * The bug is fixed, but the old version is still around in
+   the source suite.  In this case, britney will generally
+   mention a "missing build" or "old binaries".
+
+If the metadata is wrong, the solution is to fix it in the bug
+tracker and wait until britney receives a new data set.  In
+the other case, the recommendation is to see the sections on
+"missing builds" and "old binaries" below.  As long as they
+are present, the package may be blocked by bugs in the older
+versions of the binaries.
+
+Britney complains about "missing builds"
+----------------------------------------
+
+A "missing build" happens when britney detects that the binaries
+for a given architecture are missing or is not up to date.  This
+is detected by checking the "Packages" files in the archive, so
+britney have no knowledge of *why* the build is missing.
+Accordingly, this kind of issue is flagged as a "possibly permanent"
+issue.
+
+If the omission is deliberate (e.g. the new version no longer
+supports that architecture), then please have the old binaries
+for that architecture removed from the *source* suite.  Once
+those are removed, britney will no longer see that as a problem.
+
+Otherwise, please check the build services for any issues with
+building or uploading the package to the archive.
+
+**Common misconceptions**: If the architecture is no longer
+supported, the removal of the old binaries should happen in
+the *source* suite (e.g. Debian unstable).  However, many
+people mistakenly request a removal from the *target* suite
+(e.g. Debian testing).  Unfortunately, this is not the proper
+solution (and, britney does not support architecture
+specific removals so it may be difficult to do anyhow).
+
+Britney complains about "old binaries"
+--------------------------------------
+
+Depending on the configuration of the britney instance, this may
+or may not be a blocker.  If the distribution has chosen to enable
+the "ignore_cruft" option, this is merely a warning/note.  That
+said, even in this mode it can block a package from migration.
+
+This appears when britney detects that there are older versions of
+the binary packages around, which was built by (an older version of)
+the same source package.
+
+This is common with distributions where their archive management
+software is capable of keeping old binaries as long as something
+depends on them (e.g. DAK as used by Debian).  Therefore, the
+most common solution is to ensure all reverse dependencies are
+updated to use the new binaries and then have the old ones
+removed (the latter commonly known as "decrufting").  Technically,
+this is also solvable by "decrufting" without updating/rebuilding
+other packages.  Though whether this is an acceptable practise
+depends on the distribution.
+
+Alternatively, if the distribution uses the "ignore_cruft" option,
+this (in itself) is not a blocker.  However, it commonly triggers
+non-obvious issues:
+
+ * If the bugs policy is enabled, an bug in the old binaries that
+   is fixed in the new version will still be a blocker.  Here, the
+   best solution is to get rid of the old binaries.
+   
+   * Note: the bugs data is not versioned so britney cannot tell which
+     versions the bug applies to.  Just which suite they affect.
+
+ * Even if the migration item is a valid candidate (i.e. all policy
+   checked have passed), it may cause installability regressions as
+   britney will also attempt to keep the old binaries around as long
+   as they are used.  The most often cause of this when the old
+   binaries are not co-installable with the new ones.
+   
+   * Note: Britney generally only works with the highest version of a
+     given binary.  If you have libfoo1 depends on libfoo-data v1 and
+     then libfoo2 depends on libfoo-data v2, then libfoo1 will become
+     uninstallable as libfoo-data v2 will "shadow" libfoo-data v1.
+

doc/what-is-britney.rst

@@ -0,0 +1,13 @@
+What is britney and britney2
+============================
+
+Britney2 is a program that can be used to `migrate` debian-based
+packages from one suite to another.  It is the tool used by Debian and
+Ubuntu to choose which packages are ready for wider testing.
+
+Britney2 is the reimplementation and replacement of the original
+britney program.  Through out this documentation, any mention of
+britney generally refers to the britney2 implementation.  When there
+is a need to reference the original britney implementation, this
+documentation will use britney1.
+