Add PiupartsPolicy to avoid piuparts regressions

Closes: Debian/britney2#16
Signed-off-by: Niels Thykier <niels@thykier.net>

britney.py

@@ -197,7 +197,7 @@ from britney2.excuse import Excuse
 from britney2.hints import HintParser
 from britney2.installability.builder import build_installability_tester
 from britney2.migrationitem import MigrationItem
-from britney2.policies.policy import AgePolicy, RCBugPolicy, PolicyVerdict
+from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, PolicyVerdict
 from britney2.utils import (old_libraries_format, undo_changes,
                             compute_reverse_tree, possibly_compressed,
                             read_nuninst, write_nuninst, write_heidi,
@@ -504,6 +504,7 @@ class Britney(object):
 
         self.policies.append(AgePolicy(self.options, self.suite_info, MINDAYS))
         self.policies.append(RCBugPolicy(self.options, self.suite_info))
+        self.policies.append(PiupartsPolicy(self.options, self.suite_info))
 
         for policy in self.policies:
             policy.register_hints(self._hint_parser)

britney2/policies/policy.py

@@ -1,3 +1,4 @@
+import json
 import os
 import time
 from abc import abstractmethod
@@ -537,3 +538,101 @@ class RCBugPolicy(BasePolicy):
                 bugs[pkg] = set()
             bugs[pkg].update(l[1].split(","))
         return bugs
+
+
+class PiupartsPolicy(BasePolicy):
+
+    def __init__(self, options, suite_info):
+        super().__init__('piuparts', options, suite_info, {'unstable'})
+        self._piuparts = {
+            'unstable': None,
+            'testing': None,
+        }
+
+    def register_hints(self, hint_parser):
+        hint_parser.register_hint_type('ignore-piuparts', split_into_one_hint_per_package)
+
+    def initialise(self, britney):
+        super().initialise(britney)
+        try:
+            filename_unstable = os.path.join(self.options.state_dir, 'piuparts-summary-unstable.json')
+            filename_testing = os.path.join(self.options.state_dir, 'piuparts-summary-testing.json')
+        except AttributeError as e:  # pragma: no cover
+            raise RuntimeError("Please set STATE_DIR in the britney configuration") from e
+        self._piuparts['unstable'] = self._read_piuparts_summary(filename_unstable, keep_url=True)
+        self._piuparts['testing'] = self._read_piuparts_summary(filename_testing, keep_url=False)
+
+    def apply_policy_impl(self, piuparts_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse):
+        if source_name in self._piuparts['testing']:
+            testing_state = self._piuparts['testing'][source_name][0]
+        else:
+            testing_state = 'X'
+        if source_name in self._piuparts['unstable']:
+            unstable_state, url = self._piuparts['unstable'][source_name]
+        else:
+            unstable_state = 'X'
+            url = None
+
+        if unstable_state == 'P':
+            # Not a regression
+            msg = 'Piuparts tested OK - {0}'.format(url)
+            result = PolicyVerdict.PASS
+            piuparts_info['test-results'] = 'pass'
+        elif unstable_state == 'F':
+            if testing_state != unstable_state:
+                piuparts_info['test-results'] = 'regression'
+                msg = 'Rejected due to piuparts regression - {0}'.format(url)
+                result = PolicyVerdict.REJECTED_PERMANENTLY
+            else:
+                piuparts_info['test-results'] = 'failed'
+                msg = 'Ignoring piuparts failure (Not a regression) - {0}'.format(url)
+                result = PolicyVerdict.PASS
+        elif unstable_state == 'W':
+            msg = 'Waiting for piuparts test results (stalls testing migration) - {0}'.format(url)
+            result = PolicyVerdict.REJECTED_TEMPORARILY
+            piuparts_info['test-results'] = 'waiting-for-test-results'
+        else:
+            msg = 'Cannot be tested (not a blocker) - {0}'.format(url)
+            piuparts_info['test-results'] = 'cannot-be-tested'
+            result = PolicyVerdict.PASS
+
+        piuparts_info['piuparts-test-url'] = url
+        excuse.addhtml(msg)
+
+        if result.is_rejected:
+            for ignore_hint in self.hints.search('ignore-piuparts',
+                                                 package=source_name,
+                                                 version=source_data_srcdist.version):
+                piuparts_info['ignored-piuparts'] = {
+                    'issued-by': ignore_hint.user
+                }
+                result = PolicyVerdict.PASS_HINTED
+                excuse.addhtml("Ignoring piuparts issue as requested by {0}".format(ignore_hint.user))
+                break
+
+        return result
+
+    def _read_piuparts_summary(self, filename, keep_url=True):
+        summary = {}
+        self.log("Loading piuparts report from {0}".format(filename))
+        with open(filename) as fd:
+            if os.fstat(fd.fileno()).st_size < 1:
+                return summary
+            data = json.load(fd)
+        try:
+            if data['_id'] != 'Piuparts Package Test Results Summary' or data['_version'] != '1.0':  # pragma: no cover
+                raise ValueError('Piuparts results in {0} does not have the correct ID or version'.format(filename))
+        except KeyError as e:  # pragma: no cover
+            raise ValueError('Piuparts results in {0} is missing id or version field'.format(filename)) from e
+        for source, suite_data in data['packages'].items():
+            if len(suite_data) != 1:  # pragma: no cover
+                raise ValueError('Piuparts results in {0}, the source {1} does not have exactly one result set'.format(
+                    filename, source
+                ))
+            item = next(iter(suite_data.values()))
+            state, _, url = item
+            if not keep_url:
+                keep_url = None
+            summary[source] = (state, url)
+
+        return summary

tests/policy-test-data/piuparts/basic/piuparts-summary-testing.json

@@ -0,0 +1,30 @@
+{
+  "_comment": "Debian Piuparts Package Results - https://anonscm.debian.org/cgit/piuparts/piuparts.git/tree/piupartslib/pkgsummary.py",
+  "_date": "Thu Nov 24 01:24:25 UTC 2016",
+  "_id": "Piuparts Package Test Results Summary",
+  "_type": "source",
+  "_version": "1.0",
+  "packages": {
+    "pass": {
+      "testing": [
+        "P",
+        0,
+        "https://piuparts.debian.org/stretch/source/p/pass.html"
+      ]
+    },
+    "failed-not-regression": {
+      "testing": [
+        "F",
+        0,
+        "https://piuparts.debian.org/stretch/source/f/failed-not-regression.html"
+      ]
+    },
+    "regression": {
+      "testing": [
+        "P",
+        0,
+        "https://piuparts.debian.org/stretch/source/r/regression.html"
+      ]
+    }
+  }
+}

tests/policy-test-data/piuparts/basic/piuparts-summary-unstable.json

@@ -0,0 +1,37 @@
+{
+  "_comment": "Debian Piuparts Package Results - https://anonscm.debian.org/cgit/piuparts/piuparts.git/tree/piupartslib/pkgsummary.py",
+  "_date": "Thu Nov 24 01:24:25 UTC 2016",
+  "_id": "Piuparts Package Test Results Summary",
+  "_type": "source",
+  "_version": "1.0",
+  "packages": {
+    "pass": {
+      "sid": [
+        "P",
+        0,
+        "https://piuparts.debian.org/sid/source/p/pass.html"
+      ]
+    },
+    "regression": {
+      "sid": [
+        "F",
+        0,
+        "https://piuparts.debian.org/sid/source/r/regression.html"
+      ]
+    },
+    "failed-not-regression": {
+      "sid": [
+        "F",
+        0,
+        "https://piuparts.debian.org/sid/source/f/failed-not-regression.html"
+      ]
+    },
+    "not-tested-yet": {
+      "sid": [
+        "W",
+        0,
+        "https://piuparts.debian.org/sid/source/n/not-tested-yet.html"
+      ]
+    }
+  }
+}

tests/test_policy.py

@@ -4,7 +4,7 @@ import os
 from britney2 import SuiteInfo, SourcePackage
 from britney2.excuse import Excuse
 from britney2.hints import HintParser
-from britney2.policies.policy import AgePolicy, RCBugPolicy, PolicyVerdict
+from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, PolicyVerdict
 
 POLICY_DATA_BASE_DIR = os.path.join(os.path.dirname(__file__), 'policy-test-data')
 TEST_HINTER = 'test-hinter'
@@ -131,7 +131,6 @@ class TestRCBugsPolicy(unittest.TestCase):
         policy = initialize_policy('rc-bugs/basic', RCBugPolicy, hints=hints)
         verdict = policy.apply_policy(policy_info, 'unstable', src_name, src_t, src_u, excuse)
         assert verdict == PolicyVerdict.REJECTED_PERMANENTLY
-        print(str(policy_info['rc-bugs']))
         assert set(policy_info['rc-bugs']['unique-source-bugs']) == {'100003'}
         assert set(policy_info['rc-bugs']['unique-target-bugs']) == {'100001', '100002'}
         assert set(policy_info['rc-bugs']['ignored-bugs']['bugs']) == {'100000'}
@@ -166,5 +165,55 @@ class TestAgePolicy(unittest.TestCase):
             if os.path.exists(age_file):
                 os.unlink(age_file)
 
+
+class TestPiupartsPolicy(unittest.TestCase):
+
+    def test_passes(self):
+        src_name = 'pass'
+        src_t, src_u, excuse, policy_info = create_policy_objects(src_name, '1.0', '2.0')
+        policy = initialize_policy('piuparts/basic', PiupartsPolicy)
+        verdict = policy.apply_policy(policy_info, 'unstable', src_name, src_t, src_u, excuse)
+        assert verdict == PolicyVerdict.PASS
+        assert policy_info['piuparts']['test-results'] == 'pass'
+        assert policy_info['piuparts']['piuparts-test-url'] == 'https://piuparts.debian.org/sid/source/p/pass.html'
+
+    def test_regression(self):
+        src_name = 'regression'
+        src_t, src_u, excuse, policy_info = create_policy_objects(src_name, '1.0', '2.0')
+        policy = initialize_policy('piuparts/basic', PiupartsPolicy)
+        verdict = policy.apply_policy(policy_info, 'unstable', src_name, src_t, src_u, excuse)
+        assert verdict == PolicyVerdict.REJECTED_PERMANENTLY
+        assert policy_info['piuparts']['test-results'] == 'regression'
+        assert policy_info['piuparts']['piuparts-test-url'] == 'https://piuparts.debian.org/sid/source/r/regression.html'
+
+    def test_regression_hinted(self):
+        src_name = 'regression'
+        hints = ['ignore-piuparts regression/2.0']
+        src_t, src_u, excuse, policy_info = create_policy_objects(src_name, '1.0', '2.0')
+        policy = initialize_policy('piuparts/basic', PiupartsPolicy, hints=hints)
+        verdict = policy.apply_policy(policy_info, 'unstable', src_name, src_t, src_u, excuse)
+        assert verdict == PolicyVerdict.PASS_HINTED
+        assert policy_info['piuparts']['test-results'] == 'regression'
+        assert policy_info['piuparts']['piuparts-test-url'] == 'https://piuparts.debian.org/sid/source/r/regression.html'
+        assert policy_info['piuparts']['ignored-piuparts']['issued-by'] == TEST_HINTER
+
+    def test_not_tested_yet(self):
+        src_name = 'not-tested-yet'
+        src_t, src_u, excuse, policy_info = create_policy_objects(src_name, '1.0', '2.0')
+        policy = initialize_policy('piuparts/basic', PiupartsPolicy)
+        verdict = policy.apply_policy(policy_info, 'unstable', src_name, src_t, src_u, excuse)
+        assert verdict == PolicyVerdict.REJECTED_TEMPORARILY
+        assert policy_info['piuparts']['test-results'] == 'waiting-for-test-results'
+        assert policy_info['piuparts']['piuparts-test-url'] == 'https://piuparts.debian.org/sid/source/n/not-tested-yet.html'
+
+    def test_failed_not_regression(self):
+        src_name = 'failed-not-regression'
+        src_t, src_u, excuse, policy_info = create_policy_objects(src_name, '1.0', '2.0')
+        policy = initialize_policy('piuparts/basic', PiupartsPolicy)
+        verdict = policy.apply_policy(policy_info, 'unstable', src_name, src_t, src_u, excuse)
+        assert verdict == PolicyVerdict.PASS
+        assert policy_info['piuparts']['test-results'] == 'failed'
+        assert policy_info['piuparts']['piuparts-test-url'] == 'https://piuparts.debian.org/sid/source/f/failed-not-regression.html'
+
 if __name__ == '__main__':
     unittest.main()