From 90e4bb6ba222fc99efeecdf7ad0743bc0e626ba7 Mon Sep 17 00:00:00 2001 From: Niels Thykier Date: Sun, 3 Dec 2017 12:01:42 +0000 Subject: [PATCH] Rewrite documentation in sphinx Signed-off-by: Niels Thykier --- doc/conf.py | 172 ++++++++++++++++++ doc/contributing-to-britney.rst | 40 ++++ doc/index.rst | 23 +++ ...ified.md => short-intro-to-migrations.rst} | 158 ++++------------ doc/solutions-to-common-policy-issues.rst | 105 +++++++++++ doc/what-is-britney.rst | 13 ++ 6 files changed, 389 insertions(+), 122 deletions(-) create mode 100644 doc/conf.py create mode 100644 doc/contributing-to-britney.rst create mode 100644 doc/index.rst rename doc/{migrations-simplified.md => short-intro-to-migrations.rst} (66%) create mode 100644 doc/solutions-to-common-policy-issues.rst create mode 100644 doc/what-is-britney.rst diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..5e2f425 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# britney2 documentation build configuration file, created by +# sphinx-quickstart on Sat Dec 2 12:34:27 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +# Add .. so sphinx can find the britney2 modules. +sys.path.insert(0, os.path.abspath('..')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'britney2' +copyright = '2017, Niels Thykier and others' +author = 'Niels Thykier' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'classic' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'britney2doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'britney2.tex', 'britney2 Documentation', + 'Niels Thykier', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'britney2', 'britney2 Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'britney2', 'britney2 Documentation', + author, 'britney2', 'One line description of project.', + 'Miscellaneous'), +] + diff --git a/doc/contributing-to-britney.rst b/doc/contributing-to-britney.rst new file mode 100644 index 0000000..2bf1523 --- /dev/null +++ b/doc/contributing-to-britney.rst @@ -0,0 +1,40 @@ +Contributing to the development of britney2 +=========================================== + +If you are interested in improving britney, you can obtain the source +code via:: + + git clone https://anonscm.debian.org/git/mirror/britney2.git + # Additional tests + git clone https://anonscm.debian.org/git/collab-maint/britney2-tests.git + +You will need some packages to run britney and the test suites:: + + # Runtime dependencies + apt install python3 python3-apt python3-yaml + # Test dependencies + apt install python3-pytest libclass-accessor-perl rsync + # Documentation generator + apt install python3-sphinx + + +Britney has some basic unit tests, which are handled by py.test. It +also has some larger integration tests (from the `britney2-tests` +repo). Running the tests are done via:: + + cd britney2 + # Basic unit tests + py.test-3 + # Integration tests + rm -fr ./test-out/ + ../britney2-tests/bin/runtests ./britney.py ../britney2-tests/t ./test-out + +The `runtests` command in `britney2-tests` supports running only a +subset of the tests. Please see its `--help` output for more +information. + +Finally, there is also some heavier tests based on some snapshots of +live data from Debian. The data files for these are available in the +`live-data` submodule of the `britney2-tests` repo. They consume +quite a bit of disk space and britney will need at least 1.3GB of RAM +to process them. diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..95ddab3 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,23 @@ +.. britney2 documentation master file, created by + sphinx-quickstart on Sat Dec 2 12:34:27 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to britney2's documentation! +==================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + what-is-britney + short-intro-to-migrations + solutions-to-common-policy-issues + contributing-to-britney + +* :ref:`search` + +.. Add these later if/when needed: +.. * :ref:`genindex` +.. * :ref:`modindex` + diff --git a/doc/migrations-simplified.md b/doc/short-intro-to-migrations.rst similarity index 66% rename from doc/migrations-simplified.md rename to doc/short-intro-to-migrations.rst index 16c9229..92d3336 100644 --- a/doc/migrations-simplified.md +++ b/doc/short-intro-to-migrations.rst @@ -1,13 +1,11 @@ -#! TITLE: Britney migration documentation -#! SUBTITLE: Understanding britney's workflow - -# Migrations +A short introduction to migrations +================================== This is a technical introduction to how britney handles migrations. Being an introduction, it deliberately oversimplifies certain things at the expense of accuracy. It also covers common migration issues and how to fix -them. +them in :doc:`solutions-to-common-policy-issues`. The document is primarily aimed at contributors for distributions that want to understand the basics of @@ -21,7 +19,8 @@ abstract. Furthermore, the document assumes familiarity with Debian-based distribution practises and terminology (such as "suites" and "source package"). -## A high level overview of britney and migrations +A high level overview of britney and migrations +----------------------------------------------- The purpose of britney is to (semi-)automatically select a number of migration items from a series of source suites @@ -34,9 +33,12 @@ of the following points: 1. The migration items pass a number of policies for the target suite. Most of these policies are basically that the migration items do not regress on selected QA checks. + * An item satisfying this part is called a `valid candiate`. - 1. Installability will not regress as a result of + + 2. Installability will not regress as a result of migrating the migration items. + * An item that (also) satisfies this part will be selected for migration. @@ -48,14 +50,15 @@ issue (as it is not a regression). This only leaves the definition of a migration items. They come in several variants defined in the next section. -## Migration items +Migration items +--------------- Internally, britney groups packages into migration items based on a few rules. There are several kinds of migration items and this document will only describe the source migration item. -> A source migration item is one upload of a source package, with -> associated binary packages once built. + A source migration item is one upload of a source package, with + associated binary packages once built. Once a new version of a source package appears in the source suite, britney will create track it with a source migration item. As the @@ -72,7 +75,8 @@ As implied earlier, there are several other migration types. But they are not covered in this document. They deal with cases like removals, rebuilds of existing binaries, etc. -## Migration phase 1: Policies / Excuses +Migration phase 1: Policies / Excuses +------------------------------------- To begin with, britney will apply a number of policies to all migration items. Each policy will rate each migration @@ -100,19 +104,25 @@ data that britney has access to. This mean that without any change to the items themselves: 1. Items that passed originally may fail in a later britney run. - 1. Likewise, items may go from a "permanent failure" to a pass. + + 2. Likewise, items may go from a "permanent failure" to a pass. This can be seen in the following example case: 1. A new version of package is uploaded. + * Britney processes the package and concludes that there no blocking bugs, so the package passes the bug policy. - 1. Then before it migrates, someone files a blocking bug against + + 2. Then before it migrates, someone files a blocking bug against the new version. + * Britney reprocesses the package and now concludes it has a regression in the bug policy (i.e. the policy verdict goes from "pass" to "permanent fail"). - 1. The bug is examined and it is determined that the bug also affects the + + 3. The bug is examined and it is determined that the bug also affects the version in the target suite. The bug tracker is updated to reflect this. + * Britney reprocesses the package again and now concludes there is a blocking bug, but it is not a regression (since it also affects the target suite). This means the policy verdict now go from "fail" to "pass". @@ -131,7 +141,8 @@ a hinted migration generally implies that the problem will not prevent future migrations for newer versions of the same source package (assuming that the problem is deterministic). -## Migration phase 2: Installability regression testing +Migration phase 2: Installability regression testing +---------------------------------------------------- For the migration items that pass the previous phase, britney will do a test migration to see if anything becomes uninstallable. @@ -143,11 +154,12 @@ problems here, the britney log file has to be examined. This requires a bit more technical insight as it has not been polished as much as the excuses. -### Confirming a migration +Confirming a migration +^^^^^^^^^^^^^^^^^^^^^^ To start with; if a migration is accepted and "committed" (i.e. it will not be rolled back), britney will include in a line starting with `final:` like -in this example: +in this example:: Apparently successful final: -cwltool,-libtest-redisserver-perl,-pinfo,-webdis,hol88 @@ -173,12 +185,13 @@ Reminder: Migration items generally use the name of the source package. There are exceptions to that "rule" (but they are not common cases covered by this document). -### Debugging failed migration attempts +Debugging failed migration attempts +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Start by confirming that the migration item was not accepted (as described in the above section). If the migration item does not appear on a `final:` line, then we need to debug the actual migration attempts. Migration attempts look -something like this: +something like this:: trying: -webdis accepted: -webdis @@ -214,8 +227,9 @@ later attempt. It is not always immediately obvious, which attempt is better for debugging. When in doubt, it is *usually* easiest to look at the attempt with the least amount of new uninstallable packages. -In the libaws example, a total of 4 binary packages become uninstallable on the -architecture `arm64`. Here is the output again with this information high lighted: +In the libaws example, a total of 4 binary packages become +uninstallable on the architecture `arm64`. Here is the output again +with this information high lighted:: migration item(s) being attemped vvvvvv @@ -230,7 +244,7 @@ architecture `arm64`. Here is the output again with this information high light Please note that britney is lazy and will often reject an item after proving that there is a regression on a single architecture. So in the above example, we are not actually sure whether this problem is architecture specific. For -`easy`-hints, the information is presented slightly different. +`easy`-hints, the information is presented slightly different:: Trying easy from autohinter: asis/2017-1 dh-ada-library/6.12 [...] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -255,103 +269,3 @@ things break. If there are few broken packages, it is often a question of looking for `Breaks`-relations or `Depends`-relations with upper bounds on versions / on old packages being removed. Alternatively, there are also tools like `dose-debcheck`, which attempts to analyse and explain problems like this. - -# Common issues with policy decisions - -## Britney complains about a fixed bug in the source suite (bug policy) - -All decisions about bugs are related to data set extracted -from the bug tracker. If britney says that the new version -introduces a bug, then it is because the data set from the bug -tracker lists that bug for *a* version in the source suite and -without it appearing for the version(s) in the target suite. - -Please note that these data sets do not include versions, so -britney is unable to tell exactly which versions are affected. -The only thing, it can tell, is what suite the bug affects. - -There is a number of common cases, where this is observed: - - * The metadata on the bug is wrong. A known example is the - Debian BTS, where if a bug has a `fixed` version equal to - a `found` version, the bug is considered unfixed. - - * The bug is fixed, but the old version is still around in - the source suite. In this case, britney will generally - mention a "missing build" or "old binaries". - -If the metadata is wrong, the solution is to fix it in the bug -tracker and wait until britney receives a new data set. In -the other case, the recommendation is to see the sections on -"missing builds" and "old binaries" below. As long as they -are present, the package may be blocked by bugs in the older -versions of the binaries. - -## Britney complains about "missing builds" - -A "missing build" happens when britney detects that the binaries -for a given architecture are missing or is not up to date. This -is detected by checking the "Packages" files in the archive, so -britney have no knowledge of *why* the build is missing. -Accordingly, this kind of issue is flagged as a "possibly permanent" -issue. - -If the omission is deliberate (e.g. the new version no longer -supports that architecture), then please have the old binaries -for that architecture removed from the *source* suite. Once -those are removed, britney will no longer see that as a problem. - -Otherwise, please check the build services for any issues with -building or uploading the package to the archive. - -**Common misconceptions**: If the architecture is no longer -supported, the removal of the old binaries should happen in -the *source* suite (e.g. Debian unstable). However, many -people mistakenly request a removal from the *target* suite -(e.g. Debian testing). Unfortunately, this is not the proper -solution (and, britney does not support architecture -specific removals so it may be difficult to do anyhow). - -## Britney complains about "old binaries" - -Depending on the configuration of the britney instance, this may -or may not be a blocker. If the distribution has chosen to enable -the "ignore_cruft" option, this is merely a warning/note. That -said, even in this mode it can block a package from migration. - -This appears when britney detects that there are older versions of -the binary packages around, which was built by (an older version of) -the same source package. - -This is common with distributions where their archive management -software is capable of keeping old binaries as long as something -depends on them (e.g. DAK as used by Debian). Therefore, the -most common solution is to ensure all reverse dependencies are -updated to use the new binaries and then have the old ones -removed (the latter commonly known as "decrufting"). Technically, -this is also solvable by "decrufting" without updating/rebuilding -other packages. Though whether this is an acceptable practise -depends on the distribution. - -Alternatively, if the distribution uses the "ignore_cruft" option, -this (in itself) is not a blocker. However, it commonly triggers -non-obvious issues: - - * If the bugs policy is enabled, an bug in the old binaries that - is fixed in the new version will still be a blocker. Here, the - best solution is to get rid of the old binaries. - - (Note: the bugs data is not versioned so britney cannot tell which - versions the bug applies to. Just which suite they affect) - - * Even if the migration item is a valid candidate (i.e. all policy - checked have passed), it may cause installability regressions as - britney will also attempt to keep the old binaries around as long - as they are used. The most often cause of this when the old - binaries are not co-installable with the new ones. - - (Note: Britney generally only works with the highest version of a - given binary. If you have libfoo1 depends on libfoo-data v1 and - then libfoo2 depends on libfoo-data v2, then libfoo1 will become - uninstallable as libfoo-data v2 will "shadow" libfoo-data v1) - diff --git a/doc/solutions-to-common-policy-issues.rst b/doc/solutions-to-common-policy-issues.rst new file mode 100644 index 0000000..ec1b43a --- /dev/null +++ b/doc/solutions-to-common-policy-issues.rst @@ -0,0 +1,105 @@ +Solutions to common policy decisions +==================================== + +.. contents:: + +Britney complains about a fixed bug in the source suite (bug policy) +-------------------------------------------------------------------- + +All decisions about bugs are related to data set extracted +from the bug tracker. If britney says that the new version +introduces a bug, then it is because the data set from the bug +tracker lists that bug for *a* version in the source suite and +without it appearing for the version(s) in the target suite. + +Please note that these data sets do not include versions, so +britney is unable to tell exactly which versions are affected. +The only thing, it can tell, is what suite the bug affects. + +There is a number of common cases, where this is observed: + + * The metadata on the bug is wrong. A known example is the + Debian BTS, where if a bug has a `fixed` version equal to + a `found` version, the bug is considered unfixed. + + * The bug is fixed, but the old version is still around in + the source suite. In this case, britney will generally + mention a "missing build" or "old binaries". + +If the metadata is wrong, the solution is to fix it in the bug +tracker and wait until britney receives a new data set. In +the other case, the recommendation is to see the sections on +"missing builds" and "old binaries" below. As long as they +are present, the package may be blocked by bugs in the older +versions of the binaries. + +Britney complains about "missing builds" +---------------------------------------- + +A "missing build" happens when britney detects that the binaries +for a given architecture are missing or is not up to date. This +is detected by checking the "Packages" files in the archive, so +britney have no knowledge of *why* the build is missing. +Accordingly, this kind of issue is flagged as a "possibly permanent" +issue. + +If the omission is deliberate (e.g. the new version no longer +supports that architecture), then please have the old binaries +for that architecture removed from the *source* suite. Once +those are removed, britney will no longer see that as a problem. + +Otherwise, please check the build services for any issues with +building or uploading the package to the archive. + +**Common misconceptions**: If the architecture is no longer +supported, the removal of the old binaries should happen in +the *source* suite (e.g. Debian unstable). However, many +people mistakenly request a removal from the *target* suite +(e.g. Debian testing). Unfortunately, this is not the proper +solution (and, britney does not support architecture +specific removals so it may be difficult to do anyhow). + +Britney complains about "old binaries" +-------------------------------------- + +Depending on the configuration of the britney instance, this may +or may not be a blocker. If the distribution has chosen to enable +the "ignore_cruft" option, this is merely a warning/note. That +said, even in this mode it can block a package from migration. + +This appears when britney detects that there are older versions of +the binary packages around, which was built by (an older version of) +the same source package. + +This is common with distributions where their archive management +software is capable of keeping old binaries as long as something +depends on them (e.g. DAK as used by Debian). Therefore, the +most common solution is to ensure all reverse dependencies are +updated to use the new binaries and then have the old ones +removed (the latter commonly known as "decrufting"). Technically, +this is also solvable by "decrufting" without updating/rebuilding +other packages. Though whether this is an acceptable practise +depends on the distribution. + +Alternatively, if the distribution uses the "ignore_cruft" option, +this (in itself) is not a blocker. However, it commonly triggers +non-obvious issues: + + * If the bugs policy is enabled, an bug in the old binaries that + is fixed in the new version will still be a blocker. Here, the + best solution is to get rid of the old binaries. + + * Note: the bugs data is not versioned so britney cannot tell which + versions the bug applies to. Just which suite they affect. + + * Even if the migration item is a valid candidate (i.e. all policy + checked have passed), it may cause installability regressions as + britney will also attempt to keep the old binaries around as long + as they are used. The most often cause of this when the old + binaries are not co-installable with the new ones. + + * Note: Britney generally only works with the highest version of a + given binary. If you have libfoo1 depends on libfoo-data v1 and + then libfoo2 depends on libfoo-data v2, then libfoo1 will become + uninstallable as libfoo-data v2 will "shadow" libfoo-data v1. + diff --git a/doc/what-is-britney.rst b/doc/what-is-britney.rst new file mode 100644 index 0000000..28ad0ab --- /dev/null +++ b/doc/what-is-britney.rst @@ -0,0 +1,13 @@ +What is britney and britney2 +============================ + +Britney2 is a program that can be used to `migrate` debian-based +packages from one suite to another. It is the tool used by Debian and +Ubuntu to choose which packages are ready for wider testing. + +Britney2 is the reimplementation and replacement of the original +britney program. Through out this documentation, any mention of +britney generally refers to the britney2 implementation. When there +is a need to reference the original britney implementation, this +documentation will use britney1. +