diff --git a/britney.py b/britney.py index b5deff2..673a027 100755 --- a/britney.py +++ b/britney.py @@ -223,6 +223,7 @@ from britney2.policies.autopkgtest import AutopkgtestPolicy from britney2.policies.sourceppa import SourcePPAPolicy from britney2.policies.sruadtregression import SRUADTRegressionPolicy from britney2.policies.email import EmailPolicy +from britney2.policies.cloud import CloudPolicy from britney2.policies.lpexcusebugs import LPExcuseBugsPolicy from britney2.utils import (log_and_format_old_libraries, read_nuninst, write_nuninst, write_heidi, @@ -552,6 +553,7 @@ class Britney(object): self._policy_engine.add_policy(EmailPolicy(self.options, self.suite_info, dry_run=add_email_policy == 'dry-run')) + self._policy_engine.add_policy(CloudPolicy(self.options, self.suite_info, dry_run=self.options.dry_run)) @property def hints(self): diff --git a/britney2/policies/cloud.py b/britney2/policies/cloud.py new file mode 100644 index 0000000..f97453a --- /dev/null +++ b/britney2/policies/cloud.py @@ -0,0 +1,285 @@ +import json +import os +from pathlib import PurePath +import re +import shutil +import smtplib +import socket +import subprocess +import xunitparser + +from britney2 import SuiteClass +from britney2.policies.policy import BasePolicy +from britney2.policies import PolicyVerdict + +FAIL_MESSAGE = """From: Ubuntu Release Team +To: {recipients} +X-Proposed-Migration: notice +Subject: [proposed-migration] {package} {version} in {series}-{pocket} failed Cloud tests. + +Hi, + +{package} {version} needs attention. + +This package fails the following tests: + +{results} + +If you have any questions about this email, please ask them in #ubuntu-release channel on libera.chat. + +Regards, Ubuntu Release Team. +""" + +ERR_MESSAGE = """From: Ubuntu Release Team +To: {recipients} +X-Proposed-Migration: notice +Subject: [proposed-migration] {package} {version} in {series}-{pocket} had errors running Cloud Tests. + +Hi, + +During Cloud tests of {package} {version} the following errors occurred: + +{results} + +If you have any questions about this email, please ask them in #ubuntu-release channel on libera.chat. + +Regards, Ubuntu Release Team. +""" +class CloudPolicy(BasePolicy): + WORK_DIR = "cloud_tests" + PACKAGE_SET_FILE = "cloud_package_set" + EMAILS = ["cpc@canonical.com"] + SERIES_TO_URN = { + "lunar": "Canonical:0001-com-ubuntu-server-lunar-daily:23_04-daily-gen2:23.04.202301030", + "kinetic": "Canonical:0001-com-ubuntu-server-kinetic:22_10:22.10.202301040", + "jammy": "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:22.04.202212140", + "focal": "Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:20.04.202212140", + "bionic": "Canonical:UbuntuServer:18_04-lts-gen2:18.04.202212090" + } + + def __init__(self, options, suite_info, dry_run=False): + super().__init__( + "cloud", options, suite_info, {SuiteClass.PRIMARY_SOURCE_SUITE} + ) + self.dry_run = dry_run + if self.dry_run: + self.logger.info("Cloud Policy: Dry-run enabled") + + self.email_host = getattr(self.options, "email_host", "localhost") + self.logger.info( + "Cloud Policy: will send emails to: %s", self.email_host + ) + self.failures = {} + self.errors = {} + + def initialise(self, britney): + super().initialise(britney) + + primary_suite = self.suite_info.primary_source_suite + series, pocket = self._retrieve_series_and_pocket_from_path(primary_suite.path) + self.series = series + self.pocket = pocket + self.package_set = self._retrieve_cloud_package_set_for_series(self.series) + + def apply_src_policy_impl(self, policy_info, item, source_data_tdist, source_data_srcdist, excuse): + if item.package not in self.package_set: + return PolicyVerdict.PASS + + if self.dry_run: + self.logger.info( + "Cloud Policy: Dry run would test {} in {}-{}".format(item.package , self.series, self.pocket) + ) + return PolicyVerdict.PASS + + self._setup_work_directory() + self.failures = {} + self.errors = {} + + self._run_cloud_tests(item.package, self.series, self.pocket) + self._send_emails_if_needed(item.package, source_data_srcdist.version, self.series, self.pocket) + + self._cleanup_work_directory() + return PolicyVerdict.PASS + + def _retrieve_cloud_package_set_for_series(self, series): + """Retrieves a set of packages for the given series in which cloud + tests should be run. + + Temporarily a static list retrieved from file. Will be updated to + retrieve from a database at a later date. + + :param series The Ubuntu codename for the series (e.g. jammy) + """ + package_set = set() + + with open(self.PACKAGE_SET_FILE) as file: + for line in file: + package_set.add(line.strip()) + + return package_set + + def _retrieve_series_and_pocket_from_path(self, suite_path): + """Given the suite path return a tuple of series and pocket. + With input 'data/jammy-proposed' the expected output is a tuple of + (jammy, proposed) + + :param suite_path The path attribute of the suite + """ + result = (None, None) + match = re.search("([a-z]*)-([a-z]*)$", suite_path) + + if(match): + result = match.groups() + else: + raise RuntimeError( + "Could not determine series and pocket from the path: %s" %suite_path + ) + return result + + def _run_cloud_tests(self, package, series, pocket): + """Runs any cloud tests for the given package. + Returns a list of failed tests or an empty list if everything passed. + + :param package The name of the package to test + :param series The Ubuntu codename for the series (e.g. jammy) + :param pocket The name of the pocket the package should be installed from (e.g. proposed) + """ + self._run_azure_tests(package, series, pocket) + + def _send_emails_if_needed(self, package, version, series, pocket): + """Sends email(s) if there are test failures and/or errors + + :param package The name of the package that was tested + :param version The version number of the package + :param series The Ubuntu codename for the series (e.g. jammy) + :param pocket The name of the pocket the package should be installed from (e.g. proposed) + """ + if len(self.failures) > 0: + emails = self.EMAILS + message = self._format_email_message( + FAIL_MESSAGE, emails, package, version, self.failures + ) + self.logger.info("Cloud Policy: Sending failure email for {}, to {}".format(package, emails)) + self._send_email(emails, message) + + if len(self.errors) > 0: + emails = self.EMAILS + message = self._format_email_message( + ERR_MESSAGE, emails, package, version, self.errors + ) + self.logger.info("Cloud Policy: Sending error email for {}, to {}".format(package, emails)) + self._send_email(emails, message) + + def _run_azure_tests(self, package, series, pocket): + """Runs Azure's required package tests. + + :param package The name of the package to test + :param series The Ubuntu codename for the series (e.g. jammy) + :param pocket The name of the pocket the package should be installed from (e.g. proposed) + """ + urn = self.SERIES_TO_URN.get(series, None) + if urn is None: + return + + self.logger.info("Cloud Policy: Running Azure tests for: {} in {}-{}".format(package, series, pocket)) + subprocess.run( + [ + "/snap/bin/cloud-test-framework", + "--instance-prefix", "britney-{}-{}-{}".format(package, series, pocket), + "--install-archive-package", "{}/{}".format(package, pocket), + "azure_gen2", + "--location", "westeurope", + "--vm-size", "Standard_D2s_v5", + "--urn", urn, + "run-test", "package-install-with-reboot", + ], + cwd=self.WORK_DIR + ) + + results_file_paths = self._find_results_files(r"TEST-NetworkTests-[0-9]*.xml") + self._parse_xunit_test_results("Azure", results_file_paths) + + def _find_results_files(self, file_regex): + """Find any test results files that match the given regex pattern. + + :param file_regex A regex pattern to use for matching the name of the results file. + """ + file_paths = [] + for file in os.listdir(self.WORK_DIR): + if re.fullmatch(file_regex, file): + file_paths.append(PurePath(self.WORK_DIR, file)) + + return file_paths + + def _parse_xunit_test_results(self, cloud, results_file_paths): + """Parses and stores any failure or error test results. + + :param cloud The name of the cloud, use for storing the results. + :results_file_paths List of paths to results files + """ + for file_path in results_file_paths: + with open(file_path) as file: + ts, tr = xunitparser.parse(file) + + for testcase, message in tr.failures: + self.store_test_result(self.failures, cloud, testcase.methodname, message) + + for testcase, message in tr.errors: + self.store_test_result(self.errors, cloud, testcase.methodname, message) + + def store_test_result(self, results, cloud, test_name, message): + """Adds the test to the results hash under the given cloud. + + Results format: + { + cloud1: { + test_name1: message1 + test_name2: message2 + }, + cloud2: ... + } + + :param results A hash to add results to + :param cloud The name of the cloud + :param message The exception or assertion error given by the test + """ + if cloud not in results: + results[cloud] = {} + + results[cloud][test_name] = message + + def _format_email_message(self, template, emails, package, version, test_results): + """Insert given parameters into the email template.""" + series = self.series + pocket = self.pocket + results = json.dumps(test_results, indent=4) + recipients = ", ".join(emails) + message= template.format(**locals()) + + return message + + def _send_email(self, emails, message): + """Send an email + + :param emails List of emails to send to + :param message The content of the email + """ + try: + server = smtplib.SMTP(self.email_host) + server.sendmail("noreply+proposed-migration@ubuntu.com", emails, message) + server.quit() + except socket.error as err: + self.logger.error("Cloud Policy: Failed to send mail! Is SMTP server running?") + self.logger.error(err) + + def _setup_work_directory(self): + """Create a directory for tests to be run in.""" + self._cleanup_work_directory() + + os.makedirs(self.WORK_DIR) + + def _cleanup_work_directory(self): + """Delete the the directory used for running tests.""" + if os.path.exists(self.WORK_DIR): + shutil.rmtree(self.WORK_DIR) + diff --git a/cloud_package_set b/cloud_package_set new file mode 100644 index 0000000..f97a293 --- /dev/null +++ b/cloud_package_set @@ -0,0 +1,600 @@ +openssh-client +libmpfr6 +python3-problem-report +rsync +libdb5 +libattr1 +locales +xz-utils +libpam-modules-bin +strace +xkb-data +dosfstools +libpolkit-agent-1-0 +usbutils +dmsetup +libfwupdplugin1 +ucf +libdbus-1-3 +vim-runtime +parted +libxmlsec1-openssl +debconf-i18n +libdevmapper-event1 +libjson-c4 +libmbim-proxy +sound-theme-freedesktop +python3-urllib3 +libgcc-s1 +libp11-kit0 +libparted2 +keyutils +libplymouth5 +libsemanage-common +procps +python3-httplib2 +libxslt1 +glib-networking-common +nano +libkeyutils1 +libx11-6 +linux-base-sgx +libreadline8 +libtext-charwidth-perl +libk5crypto3 +pciutils +sensible-utils +init-system-helpers +libatm1 +python3-requests +python3-idna +pci +vim-tiny +libnss3 +ncurses-bin +udev +cryptsetup-initramfs +libasound2-data +modemmanager +bsdmainutils +libevent-2 +xfsprogs +libbsd0 +ubuntu-advantage-pro +libfido2-1 +libzstd1 +chrony +libxmlb2 +python3-pyrsistent +gnupg +dmidecode +os-prober +libbz2-1 +libfastjson4 +efibootmgr +libdconf1 +libtdb1 +libldap-common +libreadline5 +netcat-openbsd +python3-gi +libpng16-16 +packagekit-tools +python3-twisted-bin +python3-cryptography +software-properties-common +popularity-contest +sg3-utils +walinuxagent +btrfs-progs +python3-lib2to3 +libyaml-0-2 +python3-serial +base-passwd +libsigsegv2 +keyboard-configuration +libsasl2-2 +gpgv +python-apt-common +grub-pc +gpg-wks-client +manpages +python3-gdbm +apport +hdparm +libdns-export1109 +vim +xdg-user-dirs +libxmuu1 +python3-cffi-backend +gdisk +libstdc +lsb-base +libip6tc2 +htop +linux-image-azure +linux-tools-azure +kpartx +libcanberra0 +libpam-modules +liberror-perl +motd-news-config +libgpg-error0 +libarchive13 +squashfs-tools +lshw +python3-incremental +libogg0 +telnet +libgstreamer1 +libheimntlm0-heimdal +python3-jinja2 +libslang2 +libpam-systemd +dconf-service +mount +python3-automat +python3-debian +python3-jsonpatch +dbus +ubuntu-minimal +packagekit +python3-more-itertools +python3-distro-info +at +libtext-iconv-perl +libpci3 +base-files +libblockdev-part2 +libsqlite3-0 +libkmod2 +libkrb5support0 +iputils-ping +libwrap0 +libcom-err2 +irqbalance +bcache-tools +update-notifier-common +libncurses6 +logsave +ubuntu-release-upgrader-core +libmbim-glib4 +bash-completion +lxd-agent-loader +python3-json-pointer +usb-modeswitch-data +fdisk +libestr0 +libsystemd0 +git-man +findutils +libhcrypto4-heimdal +libpsl5 +perl-modules-5 +python3-importlib-metadata +xxd +libtinfo6 +sbsigntool +file +libext2fs2 +passwd +sysvinit-utils +libasound2 +libunistring2 +libksba8 +plymouth-theme-ubuntu-text +distro-info-data +libtevent0 +libmount1 +libsasl2-modules +python3-jwt +libntfs-3g883 +gzip +publicsuffix +openssh-server +python3-hamcrest +iputils-tracepath +gpg-agent +iproute2 +libmm-glib0 +apt +libfwupdplugin5 +libtext-wrapi18n-perl +libvolume-key1 +libsodium23 +kmod +cloud-guest-utils +python3-apport +python3-openssl +linux-cloud-tools-azure +gir1 +libx11-data +libuv1 +unattended-upgrades +python3-six +run-one +linux-headers-azure +byobu +bc +libudisks2-0 +libargon2-1 +libdevmapper1 +cryptsetup +policykit-1 +libcap-ng0 +libgusb2 +libkrb5-26-heimdal +gsettings-desktop-schemas +ssh-import-id +python3-requests-unixsocket +thin-provisioning-tools +libc6 +libefivar1 +libusb-1 +login +libssl1 +libklibc +dpkg +python3-distro +eatmydata +bash +python3-chardet +libsmartcols1 +usb +libheimbase1-heimdal +fonts-ubuntu-console +libnss-systemd +libroken18-heimdal +libpolkit-gobject-1-0 +zerofree +initramfs-tools +libnettle7 +bind9-dnsutils +ubuntu-server +glib-networking-services +linux-tools-common +python3-markupsafe +liburcu6 +diffutils +python3-pexpect +libpcap0 +ncurses-base +libxml2 +libuchardet0 +libblkid1 +powermgmt-base +dbus-user-session +ca-certificates +sosreport +libtasn1-6 +busybox-initramfs +mawk +eject +libblockdev-fs2 +e2fsprogs +libdrm-common +python3-secretstorage +vim-common +grub-efi-amd64-bin +libudev1 +systemd +python3-certifi +ed +libbrotli1 +linux-image-5 +libfuse2 +python3-click +python3-jsonschema +bind9-libs +libsgutils2-2 +distro-info +libssh-4 +libtss2-esys0 +plymouth +zlib1g +libeatmydata1 +dconf-gsettings-backend +libapparmor1 +libblockdev-swap2 +libfdisk1 +libgcrypt20 +friendly-recovery +libkrb5-3 +libgpm2 +gawk +initramfs-tools-core +pollinate +libelf1 +gettext-base +kbd +libxmlsec1 +gpg +libnetplan0 +python3 +libcurl3-gnutls +apport-symptoms +libgpgme11 +python3-debconf +libnewt0 +isc-dhcp-common +language-selector-common +coreutils +grub-common +libsoup2 +console-setup +sudo +command-not-found +libdebconfclient0 +libjson-glib-1 +python3-launchpadlib +grep +cifs-utils +whiptail +linux-cloud-tools-common +libjcat1 +libisc-export1105 +cron +python3-pkg-resources +libblockdev-part-err2 +libnuma1 +libxau6 +libaudit-common +libglib2 +libselinux1 +libicu66 +git +python3-wadllib +libsepol1 +tmux +python3-commandnotfound +isc-dhcp-client +libpython3 +dmeventd +liblzma5 +python3-setuptools +tpm-udev +libunwind8 +grub-efi-amd64-signed +libtalloc2 +openssl +libmagic-mgc +libmpdec2 +libisns0 +libnfnetlink0 +libpam0g +linux-modules-5 +libffi7 +libaio1 +klibc-utils +libsmbios-c2 +python3-yaml +python3-entrypoints +psmisc +libutempter0 +linux-azure +libmaxminddb0 +libhx509-5-heimdal +python3-zipp +grub-pc-bin +rsyslog +libfwupd2 +python3-update-manager +libgirepository-1 +liblz4-1 +lsb-release +fwupd-signed +libassuan0 +fwupd +screen +python3-distutils +python3-pyasn1-modules +libfl2 +usb-modeswitch +libpipeline1 +liblocale-gettext-perl +libltdl7 +libmagic1 +krb5-locales +libaccountsservice0 +libsemanage1 +libpcre2-8-0 +pastebinit +linux-tools-5 +groff-base +landscape-common +ubuntu-standard +libgmp10 +libproxy1v5 +curl +finalrd +ethtool +python3-netifaces +info +libasn1-8-heimdal +libgnutls30 +libuuid1 +libpam-cap +python3-pymacaroons +libexpat1 +busybox-static +shared-mime-info +libwind0-heimdal +open-iscsi +ncurses-term +libparted-fs-resize0 +libcbor0 +bsdutils +python3-oauthlib +ubuntu-keyring +overlayroot +python3-colorama +mime-support +python3-newt +libsasl2-modules-db +multipath-tools +iso-codes +libidn2-0 +perl-base +python3-simplejson +python3-hyperlink +cpio +libnftnl11 +fuse +libxcb1 +libnetfilter-conntrack3 +gnupg-utils +liblmdb0 +cloud-initramfs-copymods +libmspack0 +libss2 +open-vm-tools +ftp +libblockdev-crypto2 +perl +accountsservice +iptables +linux-azure-5 +gpg-wks-server +hostname +grub-gfxpayload-lists +libnpth0 +readline-common +apparmor +libapt-pkg6 +gcc-10-base +linux-headers-5 +bolt +alsa-ucm-conf +lvm2 +libhogweed5 +ntfs-3g +systemd-sysv +tzdata +libpython3-stdlib +libwbclient0 +pinentry-curses +gnupg-l10n +secureboot-db +libedit2 +libcap2 +sed +python3-zope +python3-service-identity +libgssapi-krb5-2 +python3-keyring +install-info +netplan +python3-software-properties +lsscsi +ufw +libgdbm-compat4 +libaudit1 +libacl1 +python3-lazr +lsof +mtr-tiny +libdw1 +libefiboot1 +libpopt0 +python3-distupgrade +libgssapi3-heimdal +adduser +dirmngr +libvorbis0a +bind9-host +cryptsetup-bin +ltrace +python3-dbus +update-manager-core +python3-configobj +libpam-runtime +python3-systemd +python3-twisted +dash +uuid-runtime +libvorbisfile3 +shim-signed +libpcre3 +liblzo2-2 +debconf +snapd +cloud-init +libblockdev-utils2 +linux-base +python3-nacl +mdadm +libcurl4 +python3-ptyprocess +networkd-dispatcher +cloud-initramfs-dyn-netconf +util-linux +libgcab-1 +libnghttp2-14 +gpgsm +libblockdev2 +man-db +python3-constantly +liblvm2cmd2 +python3-minimal +initramfs-tools-bin +libldap-2 +libappstream4 +python3-pyasn1 +libfribidi0 +libblockdev-loop2 +tcpdump +udisks2 +xauth +libxmlb1 +libmnl0 +libncursesw6 +libc-bin +libseccomp2 +tar +libqmi-glib5 +grub2-common +alsa-topology-conf +time +libqmi-proxy +libgdbm6 +libprocps8 +libxtables12 +python3-attr +gpgconf +ubuntu-advantage-tools +console-setup-linux +librtmp1 +mokutil +glib-networking +libxdmcp6 +cryptsetup-run +debianutils +libip4tc2 +lz4 +python3-blinker +libpackagekit-glib2-18 +patch +libstemmer0d +libfreetype6 +less +init +systemd-timesyncd +python3-parted +libcrypt1 +netbase +libcap2-bin +linux-cloud-tools-5 +libcryptsetup12 +bzip2 +libdrm2 +logrotate +libperl5 +libatasmart4 +libxext6 +apt-utils +openssh-sftp-server +libgudev-1 +libnspr4 +sg3-utils-udev +wget +python3-apt diff --git a/tests/test_cloud.py b/tests/test_cloud.py new file mode 100644 index 0000000..1182295 --- /dev/null +++ b/tests/test_cloud.py @@ -0,0 +1,181 @@ +#!/usr/bin/python3 +# (C) 2022 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import json +import os +import pathlib +import sys +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PROJECT_DIR) + +from britney2.policies.cloud import CloudPolicy, ERR_MESSAGE +from tests.test_sourceppa import FakeOptions + +class FakeItem: + package = "chromium-browser" + version = "0.0.1" + +class FakeSourceData: + version = "55.0" + +class T(unittest.TestCase): + def setUp(self): + self.policy = CloudPolicy(FakeOptions, {}) + self.policy._setup_work_directory() + + def tearDown(self): + self.policy._cleanup_work_directory() + + def test_retrieve_series_and_pocket_from_path(self): + """Retrieves the series and pocket from the suite path. + Ensure an exception is raised if the regex fails to match. + """ + result = self.policy._retrieve_series_and_pocket_from_path("data/jammy-proposed") + self.assertTupleEqual(result, ("jammy", "proposed")) + + self.assertRaises( + RuntimeError, self.policy._retrieve_series_and_pocket_from_path, "data/badpath" + ) + + @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests") + def test_run_cloud_tests_called_for_package_in_manifest(self, mock_run): + """Cloud tests should run for a package in the cloud package set. + """ + self.policy.package_set = set(["chromium-browser"]) + self.policy.series = "jammy" + self.policy.pocket = "proposed" + + self.policy.apply_src_policy_impl( + None, FakeItem, None, FakeSourceData, None + ) + + mock_run.assert_called_once_with("chromium-browser", "jammy", "proposed") + + @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests") + def test_run_cloud_tests_not_called_for_package_not_in_manifest(self, mock_run): + """Cloud tests should not run for packages not in the cloud package set""" + + self.policy.package_set = set(["vim"]) + self.policy.series = "jammy" + self.policy.pocket = "proposed" + + self.policy.apply_src_policy_impl( + None, FakeItem, None, FakeSourceData, None + ) + + mock_run.assert_not_called() + + @patch("britney2.policies.cloud.smtplib") + @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests") + def test_no_tests_run_during_dry_run(self, mock_run, smtp): + self.policy = CloudPolicy(FakeOptions, {}, dry_run=True) + self.policy.package_set = set(["chromium-browser"]) + self.policy.series = "jammy" + self.policy.pocket = "proposed" + + self.policy.apply_src_policy_impl( + None, FakeItem, None, FakeSourceData, None + ) + + mock_run.assert_not_called() + self.assertEqual(smtp.mock_calls, []) + + def test_finding_results_file(self): + """Ensure result file output from Cloud Test Framework can be found""" + path = pathlib.PurePath(CloudPolicy.WORK_DIR, "TEST-FakeTests-20230101010101.xml") + path2 = pathlib.PurePath(CloudPolicy.WORK_DIR, "Test-OtherTests-20230101010101.xml") + with open(path, "a"): pass + with open(path2, "a"): pass + + regex = r"TEST-FakeTests-[0-9]*.xml" + results_file_paths = self.policy._find_results_files(regex) + + self.assertEqual(len(results_file_paths), 1) + self.assertEqual(results_file_paths[0], path) + + def test_parsing_of_xunit_results_file(self): + """Test that parser correctly sorts and stores test failures and errors""" + path = self._create_fake_test_result_file(num_pass=4, num_err=2, num_fail=3) + self.policy._parse_xunit_test_results("Azure", [path]) + + azure_failures = self.policy.failures.get("Azure", {}) + azure_errors = self.policy.errors.get("Azure", {}) + + self.assertEqual(len(azure_failures), 3) + self.assertEqual(len(azure_errors), 2) + + test_names = azure_failures.keys() + self.assertIn("failing_test_1", test_names) + + self.assertEqual( + azure_failures.get("failing_test_1"), "AssertionError: A useful error message" + ) + + def test_email_formatting(self): + """Test that information is inserted correctly in the email template""" + failures = { + "Azure": { + "failing_test1": "Error reason 1", + "failing_test2": "Error reason 2" + } + } + self.policy.series = "jammy" + self.policy.pocket = "proposed" + message = self.policy._format_email_message(ERR_MESSAGE, ["work@canonical.com"], "vim", "9.0", failures) + + self.assertIn("To: work@canonical.com", message) + self.assertIn("vim 9.0", message) + self.assertIn("Error reason 2", message) + + def _create_fake_test_result_file(self, num_pass=1, num_err=0, num_fail=0): + """Helper function to generate an xunit test result file. + + :param num_pass The number of passing tests to include + :param num_err The number of erroring tests to include + :param num_fail The number of failing tests to include + + Returns the path to the created file. + """ + os.makedirs(CloudPolicy.WORK_DIR, exist_ok=True) + path = pathlib.PurePath(CloudPolicy.WORK_DIR, "TEST-FakeTests-20230101010101.xml") + + root = ET.Element("testsuite", attrib={"name": "FakeTests-1234567890"}) + + for x in range(0, num_pass): + case_attrib = {"classname": "FakeTests", "name": "passing_test_{}".format(x), "time":"0.001"} + ET.SubElement(root, "testcase", attrib=case_attrib) + + for x in range(0, num_err): + case_attrib = {"classname": "FakeTests", "name": "erroring_test_{}".format(x), "time":"0.001"} + testcase = ET.SubElement(root, "testcase", attrib=case_attrib) + + err_attrib = {"type": "Exception", "message": "A useful error message" } + ET.SubElement(testcase, "error", attrib=err_attrib) + + for x in range(0, num_fail): + case_attrib = {"classname": "FakeTests", "name": "failing_test_{}".format(x), "time":"0.001"} + testcase = ET.SubElement(root, "testcase", attrib=case_attrib) + + fail_attrib = {"type": "AssertionError", "message": "A useful error message" } + ET.SubElement(testcase, "failure", attrib=fail_attrib) + + + tree = ET.ElementTree(root) + ET.indent(tree, space="\t", level=0) + + with open(path, "w") as file: + tree.write(file, encoding="unicode", xml_declaration=True) + + return path + +if __name__ == "__main__": + unittest.main()