Email Policy, send emails when packages are rejected.

britney.py

@@ -205,6 +205,7 @@ from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, Pol
 from britney2.policies.policy import LPBlockBugPolicy
 from britney2.policies.autopkgtest import AutopkgtestPolicy
 from britney2.policies.sourceppa import SourcePPAPolicy
+from britney2.policies.email import EmailPolicy
 from britney2.utils import (old_libraries_format, undo_changes,
                             compute_reverse_tree, possibly_compressed,
                             read_nuninst, write_nuninst, write_heidi,
@@ -537,6 +538,7 @@ class Britney(object):
         if getattr(self.options, 'adt_enable') == 'yes':
             self.policies.append(AutopkgtestPolicy(self.options, self.suite_info))
         self.policies.append(SourcePPAPolicy(self.options, self.suite_info))
+        self.policies.append(EmailPolicy(self.options, self.suite_info))
 
         for policy in self.policies:
             policy.register_hints(self._hint_parser)

britney2/policies/email.py

@@ -0,0 +1,171 @@
+import os
+import re
+import json
+import smtplib
+
+from urllib.parse import unquote
+from collections import defaultdict
+from email.mime.text import MIMEText
+
+from britney2.policies.rest import Rest
+from britney2.policies.policy import BasePolicy, PolicyVerdict
+
+
+API_PREFIX = 'https://api.launchpad.net/1.0/'
+USER = API_PREFIX + '~'
+
+# Don't send emails to these bots
+BOTS = {
+    USER + 'ci-train-bot',
+    USER + 'bileto-bot',
+    USER + 'ubuntu-archive-robot',
+    USER + 'katie',
+}
+
+MESSAGE_BODY = """{source_name} {version} needs attention.
+
+It has been stuck in {series}-proposed for over a day.
+
+You either sponsored or uploaded this package, please investigate why it hasn't been approved for migration.
+
+http://people.canonical.com/~ubuntu-archive/proposed-migration/{series}/update_excuses.html#{source_name}
+
+https://wiki.ubuntu.com/ProposedMigration
+
+If you have any questions about this email, please ask them in #ubuntu-release channel on Freenode IRC.
+"""
+
+
+def person_chooser(source):
+    """Assign blame for the current source package."""
+    people = {
+        source['package_signer_link'],
+        source['sponsor_link'],
+        source['creator_link'],
+    } - {None} - BOTS
+    if source['package_signer_link'] in BOTS:
+        people.add(source['package_creator_link'])
+    return people
+
+
+def address_chooser(addresses):
+    """Prefer @ubuntu and @canonical addresses."""
+    first = None
+    canonical = None
+    for address in addresses:
+        if address.endswith('@ubuntu.com'):
+            return address
+        if address.endswith('@canonical.com'):
+            canonical = address
+        if first is None:
+            first = address
+    return canonical or first
+
+
+class EmailPolicy(BasePolicy, Rest):
+    """Send an email when a package has been rejected."""
+
+    def __init__(self, options, suite_info):
+        super().__init__('email', options, suite_info, {'unstable'})
+        self.filename = os.path.join(options.unstable, 'EmailCache')
+        # Dict of dicts; maps pkg name -> pkg version -> boolean
+        self.emails_by_pkg = defaultdict(dict)
+        # self.cache contains self.emails_by_pkg from previous run
+        self.cache = {}
+
+    def initialise(self, britney):
+        """Load cached source ppa data"""
+        super().initialise(britney)
+
+        if os.path.exists(self.filename):
+            with open(self.filename, encoding='utf-8') as data:
+                self.cache = json.load(data)
+            self.log("Loaded cached email data from %s" % self.filename)
+
+    def _scrape_gpg_emails(self, person):
+        """Find email addresses from one person's GPG keys."""
+        addresses = []
+        gpg = self.query_lp_rest_api(person + '/gpg_keys', {})
+        for key in gpg['entries']:
+            details = self.query_rest_api('http://keyserver.ubuntu.com/pks/lookup', {
+                'op': 'index',
+                'search': '0x' + key['fingerprint'],
+                'exact': 'on',
+                'options': 'mr',
+            })
+            for line in details.splitlines():
+                parts = line.split(':')
+                if parts[0] == 'info':
+                    assert int(parts[1]) == 1  # Version
+                    assert int(parts[2]) <= 1  # Count
+                if parts[0] == 'uid':
+                    flags = parts[4]
+                    if 'e' in flags or 'r' in flags:
+                        continue
+                    uid = unquote(parts[1])
+                    match = re.match(r'^.*<(.+@.+)>$', uid)
+                    if match:
+                        addresses.append(match.group(1))
+        return addresses
+
+    def scrape_gpg_emails(self, people):
+        """Find email addresses from GPG keys."""
+        addresses = []
+        for person in people or []:
+            address = address_chooser(self._scrape_gpg_emails(person))
+            addresses.append(address)
+        return addresses
+
+    def lp_get_emails(self, pkg, version):
+        """Ask LP who uploaded this package."""
+        data = self.query_lp_rest_api('%s/+archive/primary' % self.options.distribution, {
+            'ws.op': 'getPublishedSources',
+            'distro_series': '/%s/%s' % (self.options.distribution, self.options.series),
+            'exact_match': 'true',
+            'order_by_date': 'true',
+            'pocket': 'Proposed',
+            'source_name': pkg,
+            'version': version,
+        })
+        try:
+            source = data['entries'][0]
+        # IndexError means no packages in -proposed matched this name/version,
+        # which is expected to happen when bileto runs britney.
+        except IndexError:
+            self.log('Email getPublishedSources IndexError (%s %s)' % (pkg, version))
+            return None
+        return self.scrape_gpg_emails(person_chooser(source))
+
+    def apply_policy_impl(self, email_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse):
+        """Send email if package is rejected."""
+        # Have more patience for things blocked in update_output.txt
+        max_age = 5 if excuse.is_valid else 1
+        series = self.options.series
+        version = source_data_srcdist.version
+        sent = self.cache.get(source_name, {}).get(version, False)
+        stuck = (excuse.daysold or 0) >= max_age
+        if stuck and not sent:
+            msg = MIMEText(MESSAGE_BODY.format(**locals()))
+            msg['X-Proposed-Migration'] = 'notice'
+            msg['Subject'] = '[proposed-migration] {} {} stuck in {}-proposed'.format(source_name, version, series)
+            msg['From'] = 'noreply@canonical.com'
+            emails = self.lp_get_emails(source_name, version)
+            if emails:
+                msg['To'] = ', '.join(emails)
+                try:
+                    with smtplib.SMTP('localhost') as smtp:
+                        smtp.send_message(msg)
+                        sent = True
+                except ConnectionRefusedError as err:
+                    self.log("Failed to send mail! Is SMTP server running?")
+                    self.log(err)
+        self.emails_by_pkg[source_name][version] = sent
+        return PolicyVerdict.PASS
+
+    def save_state(self, britney):
+        """Write source ppa data to disk"""
+        tmp = self.filename + '.tmp'
+        with open(tmp, 'w', encoding='utf-8') as data:
+            json.dump(self.emails_by_pkg, data)
+        os.rename(tmp, self.filename)
+        self.log("Wrote email data to %s" % self.filename)

britney2/policies/rest.py

@@ -0,0 +1,59 @@
+import json
+import socket
+import urllib.request
+import urllib.parse
+
+from collections import defaultdict
+from urllib.error import HTTPError
+
+
+LAUNCHPAD_URL = 'https://api.launchpad.net/1.0/'
+
+
+class Rest:
+    """Wrap common REST APIs with some retry logic."""
+
+    def query_rest_api(self, obj, query):
+        """Do a REST request
+
+        Request <obj>?<query>.
+
+        Returns string received from web service.
+        Raises HTTPError, ValueError, or ConnectionError based on different
+        transient failures connecting.
+        """
+
+        for retry in range(5):
+            url = '%s?%s' % (obj, urllib.parse.urlencode(query))
+            try:
+                with urllib.request.urlopen(url, timeout=30) as req:
+                    code = req.getcode()
+                    if 200 <= code < 300:
+                        return req.read().decode('UTF-8')
+                    raise ConnectionError('Failed to reach launchpad, HTTP %s'
+                                          % code)
+            except socket.timeout as e:
+                self.log("Timeout downloading '%s', will retry %d more times."
+                         % (url, 5 - retry - 1))
+                exc = e
+            except HTTPError as e:
+                if e.code != 503:
+                    raise
+                self.log("Caught error 503 downloading '%s', will retry %d more times."
+                         % (url, 5 - retry - 1))
+                exc = e
+        else:
+            raise exc
+
+    def query_lp_rest_api(self, obj, query):
+        """Do a Launchpad REST request
+
+        Request <LAUNCHPAD_URL><obj>?<query>.
+
+        Returns dict of parsed json result from launchpad.
+        Raises HTTPError, ValueError, or ConnectionError based on different
+        transient failures connecting to launchpad.
+        """
+        if not obj.startswith(LAUNCHPAD_URL):
+            obj = LAUNCHPAD_URL + obj
+        return json.loads(self.query_rest_api(obj, query))

britney2/policies/sourceppa.py

@@ -7,6 +7,7 @@ import urllib.parse
 from collections import defaultdict
 from urllib.error import HTTPError
 
+from britney2.policies.rest import Rest
 from britney2.policies.policy import BasePolicy, PolicyVerdict
 
 
@@ -21,7 +22,7 @@ IGNORE = [
 ]
 
 
-class SourcePPAPolicy(BasePolicy):
+class SourcePPAPolicy(BasePolicy, Rest):
     """Migrate packages copied from same source PPA together
 
     This policy will query launchpad to determine what source PPA packages
@@ -40,38 +41,6 @@ class SourcePPAPolicy(BasePolicy):
         # self.cache contains self.source_ppas_by_pkg from previous run
         self.cache = {}
 
-    def query_lp_rest_api(self, obj, query):
-        """Do a Launchpad REST request
-
-        Request <LAUNCHPAD_URL><obj>?<query>.
-
-        Returns dict of parsed json result from launchpad.
-        Raises HTTPError, ValueError, or ConnectionError based on different
-        transient failures connecting to launchpad.
-        """
-
-        for retry in range(5):
-            url = '%s%s?%s' % (LAUNCHPAD_URL, obj, urllib.parse.urlencode(query))
-            try:
-                with urllib.request.urlopen(url, timeout=30) as req:
-                    code = req.getcode()
-                    if 200 <= code < 300:
-                        return json.loads(req.read().decode('UTF-8'))
-                    raise ConnectionError('Failed to reach launchpad, HTTP %s'
-                                          % code)
-            except socket.timeout as e:
-                self.log("Timeout downloading '%s', will retry %d more times."
-                         % (url, 5 - retry - 1))
-                exc = e
-            except HTTPError as e:
-                if e.code != 503:
-                    raise
-                self.log("Caught error 503 downloading '%s', will retry %d more times."
-                         % (url, 5 - retry - 1))
-                exc = e
-        else:
-            raise exc
-
     def lp_get_source_ppa(self, pkg, version):
         """Ask LP what source PPA pkg was copied from"""
         cached = self.cache.get(pkg, {}).get(version)

tests/test_autopkgtest.py

@@ -89,6 +89,12 @@ class T(TestBase):
             'newgreen': {'2': ''},
         }
 
+        self.email_cache = {}
+        for pkg, vals in self.sourceppa_cache.items():
+            for version, empty in vals.items():
+                self.email_cache.setdefault(pkg, {})
+                self.email_cache[pkg][version] = True
+
         # create mock Swift server (but don't start it yet, as tests first need
         # to poke in results)
         self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085)
@@ -116,13 +122,17 @@ class T(TestBase):
             self.sourceppa_cache.setdefault(pkg, {})
             if fields['Version'] not in self.sourceppa_cache[pkg]:
                 self.sourceppa_cache[pkg][fields['Version']] = ''
+            self.email_cache.setdefault(pkg, {})
+            self.email_cache[pkg][fields['Version']] = True
 
         # Set up sourceppa cache for testing
         sourceppa_path = os.path.join(self.data.dirs[True], 'SourcePPA')
         with open(sourceppa_path, 'w', encoding='utf-8') as sourceppa:
             sourceppa.write(json.dumps(self.sourceppa_cache))
-        # with open(sourceppa_path, encoding='utf-8') as data:
+
-        #     print(data.read())
+        email_path = os.path.join(self.data.dirs[True], 'EmailCache')
+        with open(email_path, 'w', encoding='utf-8') as email:
+            email.write(json.dumps(self.email_cache))
 
         self.swift.start()
         (excuses_yaml, excuses_html, out) = self.run_britney()

tests/test_email.py

@@ -0,0 +1,220 @@
+#!/usr/bin/python3
+# (C) 2017 Canonical Ltd.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+import os
+import sys
+import unittest
+from unittest.mock import DEFAULT, patch, call
+
+PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_DIR)
+
+from britney2.policies.policy import PolicyVerdict
+from britney2.policies.email import EmailPolicy, person_chooser, address_chooser
+
+from tests.test_sourceppa import FakeOptions
+
+
+# Example of a direct upload by core dev: openstack-doc-tools 1.5.0-0ubuntu1
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7524835
+UPLOAD = dict(
+    creator_link=None,
+    package_creator_link='https://api.launchpad.net/1.0/~zulcss',
+    package_signer_link='https://api.launchpad.net/1.0/~zulcss',
+    sponsor_link=None,
+)
+
+# Example of a sponsored upload: kerneloops 0.12+git20140509-2ubuntu1
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7013597
+SPONSORED_UPLOAD = dict(
+    creator_link=None,
+    package_creator_link='https://api.launchpad.net/1.0/~smb',
+    package_signer_link='https://api.launchpad.net/1.0/~apw',
+    sponsor_link=None,
+)
+
+# Example of a bileto upload: autopilot 1.6.0+17.04.20170302-0ubuntu1
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7525085
+# (dobey clicked 'build' and sil2100 clicked 'publish')
+BILETO = dict(
+    creator_link='https://api.launchpad.net/1.0/~sil2100',
+    package_creator_link='https://api.launchpad.net/1.0/~dobey',
+    package_signer_link='https://api.launchpad.net/1.0/~ci-train-bot',
+    sponsor_link='https://api.launchpad.net/1.0/~ubuntu-archive-robot',
+)
+
+# Example of a non-sponsored copy: linux 4.10.0-11.13
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7522481
+# (the upload to the PPA was sponsored but the copy was done directly)
+UNSPONSORED_COPY = dict(
+    creator_link='https://api.launchpad.net/1.0/~timg-tpi',
+    package_creator_link='https://api.launchpad.net/1.0/~sforshee',
+    package_signer_link='https://api.launchpad.net/1.0/~timg-tpi',
+    sponsor_link=None,
+)
+
+# Example of a sponsored copy: pagein 0.00.03-1
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7533336
+SPONSORED_COPY = dict(
+    creator_link='https://api.launchpad.net/1.0/~colin-king',
+    package_creator_link='https://api.launchpad.net/1.0/~colin-king',
+    package_signer_link=None,
+    sponsor_link='https://api.launchpad.net/1.0/~mapreri',
+)
+
+# Example of a manual debian sync: systemd 232-19
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7522736
+MANUAL_SYNC = dict(
+    creator_link='https://api.launchpad.net/1.0/~costamagnagianfranco',
+    package_creator_link='https://api.launchpad.net/1.0/~pkg-systemd-maintainers',
+    package_signer_link=None,
+    sponsor_link=None,
+)
+
+# Example of a sponsored manual debian sync: python-pymysql 0.7.9-2
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7487820
+SPONSORED_MANUAL_SYNC = dict(
+    creator_link='https://api.launchpad.net/1.0/~lars-tangvald',
+    package_creator_link='https://api.launchpad.net/1.0/~openstack-1.0',
+    package_signer_link=None,
+    sponsor_link='https://api.launchpad.net/1.0/~racb',
+)
+
+# Example of an automatic debian sync: gem2deb 0.33.1
+# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7255529
+AUTO_SYNC = dict(
+    creator_link='https://api.launchpad.net/1.0/~katie',
+    package_creator_link='https://api.launchpad.net/1.0/~pkg-ruby-extras-maintainers',
+    package_signer_link=None,
+    sponsor_link='https://api.launchpad.net/1.0/~ubuntu-archive-robot',
+)
+
+
+# address lists
+UBUNTU = ['personal@gmail.com', 'ubuntu@ubuntu.com', 'work@canonical.com']
+CANONICAL = ['personal@gmail.com', 'work@canonical.com']
+COMMUNITY = ['personal@gmail.com', 'other@gmail.com']
+
+
+def retvals(retvals):
+    """Return different retvals on different calls of mock."""
+    def returner(*args, **kwargs):
+        return retvals.pop()
+    return returner
+
+
+class FakeSourceData:
+    version = '55.0'
+
+
+class FakeExcuse:
+    is_valid = True
+    daysold = 0
+
+
+class T(unittest.TestCase):
+    maxDiff = None
+
+    def test_person_chooser(self):
+        """Find the correct person to blame for an upload."""
+        self.assertEqual(person_chooser(UPLOAD), {
+            'https://api.launchpad.net/1.0/~zulcss',
+        })
+        self.assertEqual(person_chooser(SPONSORED_UPLOAD), {
+            'https://api.launchpad.net/1.0/~apw',
+        })
+        self.assertEqual(person_chooser(BILETO), {
+            'https://api.launchpad.net/1.0/~dobey',
+            'https://api.launchpad.net/1.0/~sil2100',
+        })
+        self.assertEqual(person_chooser(UNSPONSORED_COPY), {
+            'https://api.launchpad.net/1.0/~timg-tpi',
+        })
+        self.assertEqual(person_chooser(SPONSORED_COPY), {
+            'https://api.launchpad.net/1.0/~colin-king',
+            'https://api.launchpad.net/1.0/~mapreri',
+        })
+        self.assertEqual(person_chooser(MANUAL_SYNC), {
+            'https://api.launchpad.net/1.0/~costamagnagianfranco',
+        })
+        self.assertSequenceEqual(person_chooser(SPONSORED_MANUAL_SYNC), {
+            'https://api.launchpad.net/1.0/~lars-tangvald',
+            'https://api.launchpad.net/1.0/~racb',
+        })
+        self.assertEqual(person_chooser(AUTO_SYNC), set())
+
+    def test_address_chooser(self):
+        """Prioritize email addresses correctly."""
+        self.assertEqual(address_chooser(UBUNTU), 'ubuntu@ubuntu.com')
+        self.assertEqual(address_chooser(CANONICAL), 'work@canonical.com')
+        self.assertEqual(address_chooser(COMMUNITY), 'personal@gmail.com')
+
+    @patch('britney2.policies.email.EmailPolicy.query_rest_api')
+    @patch('britney2.policies.email.EmailPolicy.query_lp_rest_api')
+    def test_email_scraping(self, lp, rest):
+        """Poke correct REST APIs to find email addresses."""
+        lp.side_effect = retvals([
+            dict(entries=[dict(fingerprint='DEFACED_ED1F1CE')]),
+            dict(entries=[UPLOAD]),
+        ])
+        rest.return_value = 'uid:Defaced Edifice <ex@example.com>:12345::'
+        e = EmailPolicy(FakeOptions, None)
+        self.assertEqual(e.lp_get_emails('openstack-doct-tools', '1.5.0-0ubuntu1'), ['ex@example.com'])
+        self.assertSequenceEqual(lp.mock_calls, [
+            call('testbuntu/+archive/primary', {
+                'distro_series': '/testbuntu/zazzy',
+                'exact_match': 'true',
+                'order_by_date': 'true',
+                'pocket': 'Proposed',
+                'source_name': 'openstack-doct-tools',
+                'version': '1.5.0-0ubuntu1',
+                'ws.op': 'getPublishedSources',
+            }),
+            call('https://api.launchpad.net/1.0/~zulcss/gpg_keys', {})
+        ])
+        self.assertSequenceEqual(rest.mock_calls, [
+            call('http://keyserver.ubuntu.com/pks/lookup', {
+                'exact': 'on',
+                'op': 'index',
+                'options': 'mr',
+                'search': '0xDEFACED_ED1F1CE',
+            })
+        ])
+
+    @patch('britney2.policies.email.EmailPolicy.lp_get_emails')
+    @patch('britney2.policies.email.smtplib')
+    def test_smtp_not_sent(self, smtp, lp):
+        """Know when not to send any emails."""
+        lp.return_value = ['example@email.com']
+        e = EmailPolicy(FakeOptions, None)
+        FakeExcuse.is_valid = False
+        FakeExcuse.daysold = 0
+        e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
+        FakeExcuse.is_valid = True
+        FakeExcuse.daysold = 4
+        e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
+        # Would email but no address found
+        FakeExcuse.daysold = 10
+        lp.return_value = []
+        e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
+        self.assertEqual(smtp.mock_calls, [])
+
+    @patch('britney2.policies.email.EmailPolicy.lp_get_emails')
+    @patch('britney2.policies.email.smtplib')
+    def test_smtp_sent(self, smtp, lp):
+        """Send emails correctly."""
+        lp.return_value = ['email@address.com']
+        e = EmailPolicy(FakeOptions, None)
+        FakeExcuse.is_valid = False
+        FakeExcuse.daysold = 100
+        e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
+        smtp.SMTP.assert_called_once_with('localhost')
+
+
+if __name__ == '__main__':
+    unittest.main()