Robert Bruce Park
8 years ago
committed by
Iain Lane
parent
fe627aaa15
commit
2775a5435c
@ -0,0 +1,171 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import smtplib
|
||||
|
||||
from urllib.parse import unquote
|
||||
from collections import defaultdict
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
from britney2.policies.rest import Rest
|
||||
from britney2.policies.policy import BasePolicy, PolicyVerdict
|
||||
|
||||
|
||||
API_PREFIX = 'https://api.launchpad.net/1.0/'
|
||||
USER = API_PREFIX + '~'
|
||||
|
||||
# Don't send emails to these bots
|
||||
BOTS = {
|
||||
USER + 'ci-train-bot',
|
||||
USER + 'bileto-bot',
|
||||
USER + 'ubuntu-archive-robot',
|
||||
USER + 'katie',
|
||||
}
|
||||
|
||||
MESSAGE_BODY = """{source_name} {version} needs attention.
|
||||
|
||||
It has been stuck in {series}-proposed for over a day.
|
||||
|
||||
You either sponsored or uploaded this package, please investigate why it hasn't been approved for migration.
|
||||
|
||||
http://people.canonical.com/~ubuntu-archive/proposed-migration/{series}/update_excuses.html#{source_name}
|
||||
|
||||
https://wiki.ubuntu.com/ProposedMigration
|
||||
|
||||
If you have any questions about this email, please ask them in #ubuntu-release channel on Freenode IRC.
|
||||
"""
|
||||
|
||||
|
||||
def person_chooser(source):
|
||||
"""Assign blame for the current source package."""
|
||||
people = {
|
||||
source['package_signer_link'],
|
||||
source['sponsor_link'],
|
||||
source['creator_link'],
|
||||
} - {None} - BOTS
|
||||
if source['package_signer_link'] in BOTS:
|
||||
people.add(source['package_creator_link'])
|
||||
return people
|
||||
|
||||
|
||||
def address_chooser(addresses):
|
||||
"""Prefer @ubuntu and @canonical addresses."""
|
||||
first = None
|
||||
canonical = None
|
||||
for address in addresses:
|
||||
if address.endswith('@ubuntu.com'):
|
||||
return address
|
||||
if address.endswith('@canonical.com'):
|
||||
canonical = address
|
||||
if first is None:
|
||||
first = address
|
||||
return canonical or first
|
||||
|
||||
|
||||
class EmailPolicy(BasePolicy, Rest):
|
||||
"""Send an email when a package has been rejected."""
|
||||
|
||||
def __init__(self, options, suite_info):
|
||||
super().__init__('email', options, suite_info, {'unstable'})
|
||||
self.filename = os.path.join(options.unstable, 'EmailCache')
|
||||
# Dict of dicts; maps pkg name -> pkg version -> boolean
|
||||
self.emails_by_pkg = defaultdict(dict)
|
||||
# self.cache contains self.emails_by_pkg from previous run
|
||||
self.cache = {}
|
||||
|
||||
def initialise(self, britney):
|
||||
"""Load cached source ppa data"""
|
||||
super().initialise(britney)
|
||||
|
||||
if os.path.exists(self.filename):
|
||||
with open(self.filename, encoding='utf-8') as data:
|
||||
self.cache = json.load(data)
|
||||
self.log("Loaded cached email data from %s" % self.filename)
|
||||
|
||||
def _scrape_gpg_emails(self, person):
|
||||
"""Find email addresses from one person's GPG keys."""
|
||||
addresses = []
|
||||
gpg = self.query_lp_rest_api(person + '/gpg_keys', {})
|
||||
for key in gpg['entries']:
|
||||
details = self.query_rest_api('http://keyserver.ubuntu.com/pks/lookup', {
|
||||
'op': 'index',
|
||||
'search': '0x' + key['fingerprint'],
|
||||
'exact': 'on',
|
||||
'options': 'mr',
|
||||
})
|
||||
for line in details.splitlines():
|
||||
parts = line.split(':')
|
||||
if parts[0] == 'info':
|
||||
assert int(parts[1]) == 1 # Version
|
||||
assert int(parts[2]) <= 1 # Count
|
||||
if parts[0] == 'uid':
|
||||
flags = parts[4]
|
||||
if 'e' in flags or 'r' in flags:
|
||||
continue
|
||||
uid = unquote(parts[1])
|
||||
match = re.match(r'^.*<(.+@.+)>$', uid)
|
||||
if match:
|
||||
addresses.append(match.group(1))
|
||||
return addresses
|
||||
|
||||
def scrape_gpg_emails(self, people):
|
||||
"""Find email addresses from GPG keys."""
|
||||
addresses = []
|
||||
for person in people or []:
|
||||
address = address_chooser(self._scrape_gpg_emails(person))
|
||||
addresses.append(address)
|
||||
return addresses
|
||||
|
||||
def lp_get_emails(self, pkg, version):
|
||||
"""Ask LP who uploaded this package."""
|
||||
data = self.query_lp_rest_api('%s/+archive/primary' % self.options.distribution, {
|
||||
'ws.op': 'getPublishedSources',
|
||||
'distro_series': '/%s/%s' % (self.options.distribution, self.options.series),
|
||||
'exact_match': 'true',
|
||||
'order_by_date': 'true',
|
||||
'pocket': 'Proposed',
|
||||
'source_name': pkg,
|
||||
'version': version,
|
||||
})
|
||||
try:
|
||||
source = data['entries'][0]
|
||||
# IndexError means no packages in -proposed matched this name/version,
|
||||
# which is expected to happen when bileto runs britney.
|
||||
except IndexError:
|
||||
self.log('Email getPublishedSources IndexError (%s %s)' % (pkg, version))
|
||||
return None
|
||||
return self.scrape_gpg_emails(person_chooser(source))
|
||||
|
||||
def apply_policy_impl(self, email_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse):
|
||||
"""Send email if package is rejected."""
|
||||
# Have more patience for things blocked in update_output.txt
|
||||
max_age = 5 if excuse.is_valid else 1
|
||||
series = self.options.series
|
||||
version = source_data_srcdist.version
|
||||
sent = self.cache.get(source_name, {}).get(version, False)
|
||||
stuck = (excuse.daysold or 0) >= max_age
|
||||
if stuck and not sent:
|
||||
msg = MIMEText(MESSAGE_BODY.format(**locals()))
|
||||
msg['X-Proposed-Migration'] = 'notice'
|
||||
msg['Subject'] = '[proposed-migration] {} {} stuck in {}-proposed'.format(source_name, version, series)
|
||||
msg['From'] = 'noreply@canonical.com'
|
||||
emails = self.lp_get_emails(source_name, version)
|
||||
if emails:
|
||||
msg['To'] = ', '.join(emails)
|
||||
try:
|
||||
with smtplib.SMTP('localhost') as smtp:
|
||||
smtp.send_message(msg)
|
||||
sent = True
|
||||
except ConnectionRefusedError as err:
|
||||
self.log("Failed to send mail! Is SMTP server running?")
|
||||
self.log(err)
|
||||
self.emails_by_pkg[source_name][version] = sent
|
||||
return PolicyVerdict.PASS
|
||||
|
||||
def save_state(self, britney):
|
||||
"""Write source ppa data to disk"""
|
||||
tmp = self.filename + '.tmp'
|
||||
with open(tmp, 'w', encoding='utf-8') as data:
|
||||
json.dump(self.emails_by_pkg, data)
|
||||
os.rename(tmp, self.filename)
|
||||
self.log("Wrote email data to %s" % self.filename)
|
||||
@ -0,0 +1,59 @@
|
||||
import json
|
||||
import socket
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
|
||||
from collections import defaultdict
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
LAUNCHPAD_URL = 'https://api.launchpad.net/1.0/'
|
||||
|
||||
|
||||
class Rest:
|
||||
"""Wrap common REST APIs with some retry logic."""
|
||||
|
||||
def query_rest_api(self, obj, query):
|
||||
"""Do a REST request
|
||||
|
||||
Request <obj>?<query>.
|
||||
|
||||
Returns string received from web service.
|
||||
Raises HTTPError, ValueError, or ConnectionError based on different
|
||||
transient failures connecting.
|
||||
"""
|
||||
|
||||
for retry in range(5):
|
||||
url = '%s?%s' % (obj, urllib.parse.urlencode(query))
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as req:
|
||||
code = req.getcode()
|
||||
if 200 <= code < 300:
|
||||
return req.read().decode('UTF-8')
|
||||
raise ConnectionError('Failed to reach launchpad, HTTP %s'
|
||||
% code)
|
||||
except socket.timeout as e:
|
||||
self.log("Timeout downloading '%s', will retry %d more times."
|
||||
% (url, 5 - retry - 1))
|
||||
exc = e
|
||||
except HTTPError as e:
|
||||
if e.code != 503:
|
||||
raise
|
||||
self.log("Caught error 503 downloading '%s', will retry %d more times."
|
||||
% (url, 5 - retry - 1))
|
||||
exc = e
|
||||
else:
|
||||
raise exc
|
||||
|
||||
def query_lp_rest_api(self, obj, query):
|
||||
"""Do a Launchpad REST request
|
||||
|
||||
Request <LAUNCHPAD_URL><obj>?<query>.
|
||||
|
||||
Returns dict of parsed json result from launchpad.
|
||||
Raises HTTPError, ValueError, or ConnectionError based on different
|
||||
transient failures connecting to launchpad.
|
||||
"""
|
||||
if not obj.startswith(LAUNCHPAD_URL):
|
||||
obj = LAUNCHPAD_URL + obj
|
||||
return json.loads(self.query_rest_api(obj, query))
|
||||
@ -0,0 +1,220 @@
|
||||
#!/usr/bin/python3
|
||||
# (C) 2017 Canonical Ltd.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import DEFAULT, patch, call
|
||||
|
||||
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, PROJECT_DIR)
|
||||
|
||||
from britney2.policies.policy import PolicyVerdict
|
||||
from britney2.policies.email import EmailPolicy, person_chooser, address_chooser
|
||||
|
||||
from tests.test_sourceppa import FakeOptions
|
||||
|
||||
|
||||
# Example of a direct upload by core dev: openstack-doc-tools 1.5.0-0ubuntu1
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7524835
|
||||
UPLOAD = dict(
|
||||
creator_link=None,
|
||||
package_creator_link='https://api.launchpad.net/1.0/~zulcss',
|
||||
package_signer_link='https://api.launchpad.net/1.0/~zulcss',
|
||||
sponsor_link=None,
|
||||
)
|
||||
|
||||
# Example of a sponsored upload: kerneloops 0.12+git20140509-2ubuntu1
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7013597
|
||||
SPONSORED_UPLOAD = dict(
|
||||
creator_link=None,
|
||||
package_creator_link='https://api.launchpad.net/1.0/~smb',
|
||||
package_signer_link='https://api.launchpad.net/1.0/~apw',
|
||||
sponsor_link=None,
|
||||
)
|
||||
|
||||
# Example of a bileto upload: autopilot 1.6.0+17.04.20170302-0ubuntu1
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7525085
|
||||
# (dobey clicked 'build' and sil2100 clicked 'publish')
|
||||
BILETO = dict(
|
||||
creator_link='https://api.launchpad.net/1.0/~sil2100',
|
||||
package_creator_link='https://api.launchpad.net/1.0/~dobey',
|
||||
package_signer_link='https://api.launchpad.net/1.0/~ci-train-bot',
|
||||
sponsor_link='https://api.launchpad.net/1.0/~ubuntu-archive-robot',
|
||||
)
|
||||
|
||||
# Example of a non-sponsored copy: linux 4.10.0-11.13
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7522481
|
||||
# (the upload to the PPA was sponsored but the copy was done directly)
|
||||
UNSPONSORED_COPY = dict(
|
||||
creator_link='https://api.launchpad.net/1.0/~timg-tpi',
|
||||
package_creator_link='https://api.launchpad.net/1.0/~sforshee',
|
||||
package_signer_link='https://api.launchpad.net/1.0/~timg-tpi',
|
||||
sponsor_link=None,
|
||||
)
|
||||
|
||||
# Example of a sponsored copy: pagein 0.00.03-1
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7533336
|
||||
SPONSORED_COPY = dict(
|
||||
creator_link='https://api.launchpad.net/1.0/~colin-king',
|
||||
package_creator_link='https://api.launchpad.net/1.0/~colin-king',
|
||||
package_signer_link=None,
|
||||
sponsor_link='https://api.launchpad.net/1.0/~mapreri',
|
||||
)
|
||||
|
||||
# Example of a manual debian sync: systemd 232-19
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7522736
|
||||
MANUAL_SYNC = dict(
|
||||
creator_link='https://api.launchpad.net/1.0/~costamagnagianfranco',
|
||||
package_creator_link='https://api.launchpad.net/1.0/~pkg-systemd-maintainers',
|
||||
package_signer_link=None,
|
||||
sponsor_link=None,
|
||||
)
|
||||
|
||||
# Example of a sponsored manual debian sync: python-pymysql 0.7.9-2
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7487820
|
||||
SPONSORED_MANUAL_SYNC = dict(
|
||||
creator_link='https://api.launchpad.net/1.0/~lars-tangvald',
|
||||
package_creator_link='https://api.launchpad.net/1.0/~openstack-1.0',
|
||||
package_signer_link=None,
|
||||
sponsor_link='https://api.launchpad.net/1.0/~racb',
|
||||
)
|
||||
|
||||
# Example of an automatic debian sync: gem2deb 0.33.1
|
||||
# https://api.launchpad.net/1.0/ubuntu/+archive/primary/+sourcepub/7255529
|
||||
AUTO_SYNC = dict(
|
||||
creator_link='https://api.launchpad.net/1.0/~katie',
|
||||
package_creator_link='https://api.launchpad.net/1.0/~pkg-ruby-extras-maintainers',
|
||||
package_signer_link=None,
|
||||
sponsor_link='https://api.launchpad.net/1.0/~ubuntu-archive-robot',
|
||||
)
|
||||
|
||||
|
||||
# address lists
|
||||
UBUNTU = ['personal@gmail.com', 'ubuntu@ubuntu.com', 'work@canonical.com']
|
||||
CANONICAL = ['personal@gmail.com', 'work@canonical.com']
|
||||
COMMUNITY = ['personal@gmail.com', 'other@gmail.com']
|
||||
|
||||
|
||||
def retvals(retvals):
|
||||
"""Return different retvals on different calls of mock."""
|
||||
def returner(*args, **kwargs):
|
||||
return retvals.pop()
|
||||
return returner
|
||||
|
||||
|
||||
class FakeSourceData:
|
||||
version = '55.0'
|
||||
|
||||
|
||||
class FakeExcuse:
|
||||
is_valid = True
|
||||
daysold = 0
|
||||
|
||||
|
||||
class T(unittest.TestCase):
|
||||
maxDiff = None
|
||||
|
||||
def test_person_chooser(self):
|
||||
"""Find the correct person to blame for an upload."""
|
||||
self.assertEqual(person_chooser(UPLOAD), {
|
||||
'https://api.launchpad.net/1.0/~zulcss',
|
||||
})
|
||||
self.assertEqual(person_chooser(SPONSORED_UPLOAD), {
|
||||
'https://api.launchpad.net/1.0/~apw',
|
||||
})
|
||||
self.assertEqual(person_chooser(BILETO), {
|
||||
'https://api.launchpad.net/1.0/~dobey',
|
||||
'https://api.launchpad.net/1.0/~sil2100',
|
||||
})
|
||||
self.assertEqual(person_chooser(UNSPONSORED_COPY), {
|
||||
'https://api.launchpad.net/1.0/~timg-tpi',
|
||||
})
|
||||
self.assertEqual(person_chooser(SPONSORED_COPY), {
|
||||
'https://api.launchpad.net/1.0/~colin-king',
|
||||
'https://api.launchpad.net/1.0/~mapreri',
|
||||
})
|
||||
self.assertEqual(person_chooser(MANUAL_SYNC), {
|
||||
'https://api.launchpad.net/1.0/~costamagnagianfranco',
|
||||
})
|
||||
self.assertSequenceEqual(person_chooser(SPONSORED_MANUAL_SYNC), {
|
||||
'https://api.launchpad.net/1.0/~lars-tangvald',
|
||||
'https://api.launchpad.net/1.0/~racb',
|
||||
})
|
||||
self.assertEqual(person_chooser(AUTO_SYNC), set())
|
||||
|
||||
def test_address_chooser(self):
|
||||
"""Prioritize email addresses correctly."""
|
||||
self.assertEqual(address_chooser(UBUNTU), 'ubuntu@ubuntu.com')
|
||||
self.assertEqual(address_chooser(CANONICAL), 'work@canonical.com')
|
||||
self.assertEqual(address_chooser(COMMUNITY), 'personal@gmail.com')
|
||||
|
||||
@patch('britney2.policies.email.EmailPolicy.query_rest_api')
|
||||
@patch('britney2.policies.email.EmailPolicy.query_lp_rest_api')
|
||||
def test_email_scraping(self, lp, rest):
|
||||
"""Poke correct REST APIs to find email addresses."""
|
||||
lp.side_effect = retvals([
|
||||
dict(entries=[dict(fingerprint='DEFACED_ED1F1CE')]),
|
||||
dict(entries=[UPLOAD]),
|
||||
])
|
||||
rest.return_value = 'uid:Defaced Edifice <ex@example.com>:12345::'
|
||||
e = EmailPolicy(FakeOptions, None)
|
||||
self.assertEqual(e.lp_get_emails('openstack-doct-tools', '1.5.0-0ubuntu1'), ['ex@example.com'])
|
||||
self.assertSequenceEqual(lp.mock_calls, [
|
||||
call('testbuntu/+archive/primary', {
|
||||
'distro_series': '/testbuntu/zazzy',
|
||||
'exact_match': 'true',
|
||||
'order_by_date': 'true',
|
||||
'pocket': 'Proposed',
|
||||
'source_name': 'openstack-doct-tools',
|
||||
'version': '1.5.0-0ubuntu1',
|
||||
'ws.op': 'getPublishedSources',
|
||||
}),
|
||||
call('https://api.launchpad.net/1.0/~zulcss/gpg_keys', {})
|
||||
])
|
||||
self.assertSequenceEqual(rest.mock_calls, [
|
||||
call('http://keyserver.ubuntu.com/pks/lookup', {
|
||||
'exact': 'on',
|
||||
'op': 'index',
|
||||
'options': 'mr',
|
||||
'search': '0xDEFACED_ED1F1CE',
|
||||
})
|
||||
])
|
||||
|
||||
@patch('britney2.policies.email.EmailPolicy.lp_get_emails')
|
||||
@patch('britney2.policies.email.smtplib')
|
||||
def test_smtp_not_sent(self, smtp, lp):
|
||||
"""Know when not to send any emails."""
|
||||
lp.return_value = ['example@email.com']
|
||||
e = EmailPolicy(FakeOptions, None)
|
||||
FakeExcuse.is_valid = False
|
||||
FakeExcuse.daysold = 0
|
||||
e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
|
||||
FakeExcuse.is_valid = True
|
||||
FakeExcuse.daysold = 4
|
||||
e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
|
||||
# Would email but no address found
|
||||
FakeExcuse.daysold = 10
|
||||
lp.return_value = []
|
||||
e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
|
||||
self.assertEqual(smtp.mock_calls, [])
|
||||
|
||||
@patch('britney2.policies.email.EmailPolicy.lp_get_emails')
|
||||
@patch('britney2.policies.email.smtplib')
|
||||
def test_smtp_sent(self, smtp, lp):
|
||||
"""Send emails correctly."""
|
||||
lp.return_value = ['email@address.com']
|
||||
e = EmailPolicy(FakeOptions, None)
|
||||
FakeExcuse.is_valid = False
|
||||
FakeExcuse.daysold = 100
|
||||
e.apply_policy_impl(None, None, 'chromium-browser', None, FakeSourceData, FakeExcuse)
|
||||
smtp.SMTP.assert_called_once_with('localhost')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue