feat(cloud): Add reporting

Add optional reporting of cloud tests. Cloud reporting includes the
package name, version, series and whether the test result was
pass/fail/error.

Cloud reporting requested by partners to be able to determine the amount
of packages tested.

britney.conf

@@ -140,6 +140,9 @@ PIUPARTS_ENABLE = no
 
 # run cloud tests on packages
 CLOUD_ENABLE            = no
+# Enable reporting of package name, version, series and whether the test
+# resulted in a pass/fail/error.
+CLOUD_ENABLE_REPORTING  = no
 # A directory to store Cloud test results and logs. Is created at the start of
 # each policy run and deleted after test results are parsed.
 CLOUD_WORK_DIR          = cloud_tests

britney2/policies/cloud.py

@@ -70,6 +70,8 @@ class CloudPolicy(BasePolicy):
         self.error_emails = getattr(self.options, "cloud_error_emails", self.DEFAULT_EMAILS)
         self.state_filename = getattr(self.options, "cloud_state_file", self.STATE_FILE)
 
+        self.reporting_enabled = getattr(self.options, "cloud_enable_reporting", "no")
+
         self.state = {}
 
         adt_ppas = getattr(self.options, "adt_ppas", "")
@@ -114,10 +116,19 @@ class CloudPolicy(BasePolicy):
         self.failures = {}
         self.errors = {}
 
+        if self.reporting_enabled == "yes":
+            self._report_test_start(item.package, source_data_srcdist.version, self.options.series)
+
         self._run_cloud_tests(item.package, source_data_srcdist.version, self.options.series,
                               self.sources, self.source_type)
 
         if len(self.failures) > 0 or len(self.errors) > 0:
+            if self.reporting_enabled == "yes":
+                if len(self.failures) > 0:
+                    self._report_test_result(item.package, source_data_srcdist.version, self.options.series, "fail")
+                else:
+                    self._report_test_result(item.package, source_data_srcdist.version, self.options.series, "error")
+
             if self.email_needed:
                 self._send_emails_if_needed(item.package, source_data_srcdist.version, self.options.series)
 
@@ -127,6 +138,9 @@ class CloudPolicy(BasePolicy):
             excuse.add_verdict_info(verdict, info)
             return verdict
         else:
+            if self.reporting_enabled == "yes":
+                self._report_test_result(item.package, source_data_srcdist.version, self.options.series, "pass")
+
             self._cleanup_work_directory()
             verdict = PolicyVerdict.PASS
             excuse.add_verdict_info(verdict, "Cloud tests passed.")
@@ -551,6 +565,37 @@ class CloudPolicy(BasePolicy):
 
         return cloud_ppas
 
+    def _report_test_start(self, package, version, series):
+        """
+        Add an entry to the report to show that the package has started tests for this series and version.
+
+        :param package The name of the package to test
+        :param version Version of the package
+        :param series The Ubuntu codename for the series (e.g. jammy)
+        """
+        params = [
+            "/snap/bin/ctf-britney-tools", "table-commands", "write-data",
+            series, package, version
+        ]
+
+        subprocess.run(params)
+
+    def _report_test_result(self, package, version, series, result):
+        """
+        Provides a simple summary of the test result to the report; whether it was a pass, fail or error.
+
+        :param package The name of the package to test
+        :param version Version of the package
+        :param series The Ubuntu codename for the series (e.g. jammy)
+        :param result A string describing the result (pass, fail, error)
+        """
+        params = [
+            "/snap/bin/ctf-britney-tools", "table-commands", "write-test-result",
+            series, package, version, result
+        ]
+
+        subprocess.run(params)
+
     def _setup_work_directory(self):
         """Create a directory for tests to be run in."""
         self._cleanup_work_directory()

tests/test_cloud.py

@@ -180,6 +180,39 @@ class T(unittest.TestCase):
         mock_run.assert_not_called()
         self.assertEqual(smtp.mock_calls, [])
 
+    @patch("britney2.policies.cloud.CloudPolicy._report_test_result")
+    @patch("britney2.policies.cloud.CloudPolicy._report_test_start")
+    @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests")
+    def test_reporting_of_cloud_tests(self, mock_run, mock_report, mock_report_result):
+        self.fake_options.cloud_enable_reporting = "yes"
+        policy = CloudPolicy(self.fake_options, {}, dry_run=False)
+
+        policy.package_set = set(["chromium-browser"])
+        policy.options.series = "jammy"
+
+        policy.apply_src_policy_impl(
+            None, FakeItem, None, FakeSourceData, MagicMock()
+        )
+
+        mock_report.assert_called_once_with(
+            "chromium-browser", "55.0", "jammy"
+        )
+
+    @patch("britney2.policies.cloud.CloudPolicy._report_test_start")
+    @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests")
+    def test_reporting_of_cloud_tests_is_disabled(self, mock_run, mock_report):
+        self.fake_options.cloud_reporting_enabled = "no"
+        policy = CloudPolicy(self.fake_options, {}, dry_run=False)
+
+        policy.package_set = set(["chromium-browser"])
+        policy.options.series = "jammy"
+
+        policy.apply_src_policy_impl(
+            None, FakeItem, None, FakeSourceData, MagicMock()
+        )
+
+        mock_report.assert_not_called()
+
     def test_finding_results_file(self):
         """Ensure result file output from Cloud Test Framework can be found"""
         path = os.path.join(self.policy.work_dir, "TEST-FakeTests-20230101010101.xml")