diff --git a/britney.conf b/britney.conf index d27e968..a043f5c 100644 --- a/britney.conf +++ b/britney.conf @@ -140,6 +140,9 @@ PIUPARTS_ENABLE = no # run cloud tests on packages CLOUD_ENABLE = no +# Enable reporting of package name, version, series and whether the test +# resulted in a pass/fail/error. +CLOUD_ENABLE_REPORTING = no # A directory to store Cloud test results and logs. Is created at the start of # each policy run and deleted after test results are parsed. CLOUD_WORK_DIR = cloud_tests diff --git a/britney2/policies/cloud.py b/britney2/policies/cloud.py index ad0e591..855701a 100644 --- a/britney2/policies/cloud.py +++ b/britney2/policies/cloud.py @@ -70,6 +70,8 @@ class CloudPolicy(BasePolicy): self.error_emails = getattr(self.options, "cloud_error_emails", self.DEFAULT_EMAILS) self.state_filename = getattr(self.options, "cloud_state_file", self.STATE_FILE) + self.reporting_enabled = getattr(self.options, "cloud_enable_reporting", "no") + self.state = {} adt_ppas = getattr(self.options, "adt_ppas", "") @@ -114,10 +116,19 @@ class CloudPolicy(BasePolicy): self.failures = {} self.errors = {} + if self.reporting_enabled == "yes": + self._report_test_start(item.package, source_data_srcdist.version, self.options.series) + self._run_cloud_tests(item.package, source_data_srcdist.version, self.options.series, self.sources, self.source_type) if len(self.failures) > 0 or len(self.errors) > 0: + if self.reporting_enabled == "yes": + if len(self.failures) > 0: + self._report_test_result(item.package, source_data_srcdist.version, self.options.series, "fail") + else: + self._report_test_result(item.package, source_data_srcdist.version, self.options.series, "error") + if self.email_needed: self._send_emails_if_needed(item.package, source_data_srcdist.version, self.options.series) @@ -127,6 +138,9 @@ class CloudPolicy(BasePolicy): excuse.add_verdict_info(verdict, info) return verdict else: + if self.reporting_enabled == "yes": + self._report_test_result(item.package, source_data_srcdist.version, self.options.series, "pass") + self._cleanup_work_directory() verdict = PolicyVerdict.PASS excuse.add_verdict_info(verdict, "Cloud tests passed.") @@ -551,6 +565,37 @@ class CloudPolicy(BasePolicy): return cloud_ppas + def _report_test_start(self, package, version, series): + """ + Add an entry to the report to show that the package has started tests for this series and version. + + :param package The name of the package to test + :param version Version of the package + :param series The Ubuntu codename for the series (e.g. jammy) + """ + params = [ + "/snap/bin/ctf-britney-tools", "table-commands", "write-data", + series, package, version + ] + + subprocess.run(params) + + def _report_test_result(self, package, version, series, result): + """ + Provides a simple summary of the test result to the report; whether it was a pass, fail or error. + + :param package The name of the package to test + :param version Version of the package + :param series The Ubuntu codename for the series (e.g. jammy) + :param result A string describing the result (pass, fail, error) + """ + params = [ + "/snap/bin/ctf-britney-tools", "table-commands", "write-test-result", + series, package, version, result + ] + + subprocess.run(params) + def _setup_work_directory(self): """Create a directory for tests to be run in.""" self._cleanup_work_directory() diff --git a/tests/test_cloud.py b/tests/test_cloud.py index 1bd23b0..cc4aa39 100644 --- a/tests/test_cloud.py +++ b/tests/test_cloud.py @@ -180,6 +180,39 @@ class T(unittest.TestCase): mock_run.assert_not_called() self.assertEqual(smtp.mock_calls, []) + @patch("britney2.policies.cloud.CloudPolicy._report_test_result") + @patch("britney2.policies.cloud.CloudPolicy._report_test_start") + @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests") + def test_reporting_of_cloud_tests(self, mock_run, mock_report, mock_report_result): + self.fake_options.cloud_enable_reporting = "yes" + policy = CloudPolicy(self.fake_options, {}, dry_run=False) + + policy.package_set = set(["chromium-browser"]) + policy.options.series = "jammy" + + policy.apply_src_policy_impl( + None, FakeItem, None, FakeSourceData, MagicMock() + ) + + mock_report.assert_called_once_with( + "chromium-browser", "55.0", "jammy" + ) + + @patch("britney2.policies.cloud.CloudPolicy._report_test_start") + @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests") + def test_reporting_of_cloud_tests_is_disabled(self, mock_run, mock_report): + self.fake_options.cloud_reporting_enabled = "no" + policy = CloudPolicy(self.fake_options, {}, dry_run=False) + + policy.package_set = set(["chromium-browser"]) + policy.options.series = "jammy" + + policy.apply_src_policy_impl( + None, FakeItem, None, FakeSourceData, MagicMock() + ) + + mock_report.assert_not_called() + def test_finding_results_file(self): """Ensure result file output from Cloud Test Framework can be found""" path = os.path.join(self.policy.work_dir, "TEST-FakeTests-20230101010101.xml")