diff --git a/britney.py b/britney.py index c2d3997..3652aff 100755 --- a/britney.py +++ b/britney.py @@ -215,6 +215,7 @@ from britney_util import (old_libraries_format, undo_changes, ) from policies.policy import AgePolicy, RCBugPolicy, LPBlockBugPolicy, PolicyVerdict from policies.autopkgtest import AutopkgtestPolicy +from policies.sourceppa import SourcePPAPolicy # Check the "check_field_name" reflection before removing an import here. from consts import (SOURCE, SOURCEVER, ARCHITECTURE, CONFLICTS, DEPENDS, @@ -545,6 +546,7 @@ class Britney(object): self.policies.append(LPBlockBugPolicy(self.options)) if getattr(self.options, 'adt_enable') == 'yes': self.policies.append(AutopkgtestPolicy(self.options)) + self.policies.append(SourcePPAPolicy(self.options)) for policy in self.policies: policy.register_hints(self._hint_parser) diff --git a/policies/sourceppa.py b/policies/sourceppa.py new file mode 100644 index 0000000..90a7ca8 --- /dev/null +++ b/policies/sourceppa.py @@ -0,0 +1,115 @@ +import os +import json +import urllib.request +import urllib.parse + +from collections import defaultdict + +from policies.policy import BasePolicy, PolicyVerdict + + +LAUNCHPAD_URL = 'https://api.launchpad.net/1.0/' +PRIMARY = LAUNCHPAD_URL + 'ubuntu/+archive/primary' + + +def query_lp_rest_api(obj, query): + """Do a Launchpad REST request + + Request ?. + + Returns dict of parsed json result from launchpad. + Raises HTTPError, ValueError, or ConnectionError based on different + transient failures connecting to launchpad. + """ + url = '%s%s?%s' % (LAUNCHPAD_URL, obj, urllib.parse.urlencode(query)) + with urllib.request.urlopen(url, timeout=10) as req: + code = req.getcode() + if 200 <= code < 300: + return json.loads(req.read().decode('UTF-8')) + raise ConnectionError('Failed to reach launchpad, HTTP %s' % code) + + +class SourcePPAPolicy(BasePolicy): + """Migrate packages copied from same source PPA together + + This policy will query launchpad to determine what source PPA packages + were copied from, and ensure that all packages from the same PPA migrate + together. + """ + + def __init__(self, options): + super().__init__('source-ppa', options, {'unstable'}) + self.filename = os.path.join(options.unstable, 'SourcePPA') + # Dict of dicts; maps pkg name -> pkg version -> source PPA URL + self.source_ppas_by_pkg = defaultdict(dict) + # Dict of sets; maps source PPA URL -> set of source names + self.pkgs_by_source_ppa = defaultdict(set) + self.britney = None + # self.cache contains self.source_ppas_by_pkg from previous run + self.cache = {} + + def lp_get_source_ppa(self, pkg, version): + """Ask LP what source PPA pkg was copied from""" + cached = self.cache.get(pkg, {}).get(version) + if cached is not None: + return cached + + data = query_lp_rest_api('%s/%s' % (self.options.distribution, self.options.series), { + 'ws.op': 'getPackageUploads', + 'archive': PRIMARY, + 'pocket': 'Proposed', + 'name': pkg, + 'version': version, + 'exact_match': 'true', + }) + try: + return data['entries'][0]['copy_source_archive_link'] or '' + # IndexError means no packages in -proposed matched this name/version, + # which is expected to happen when bileto runs britney. + except IndexError: + return '' + + def initialise(self, britney): + """Load and discover all source ppa data""" + super().initialise(britney) + self.britney = britney + + try: + with open(self.filename, encoding='utf-8') as data: + self.cache = json.load(data) + self.log("Loaded cached source ppa data from %s" % self.filename) + except FileNotFoundError: + pass + + def apply_policy_impl(self, sourceppa_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse): + """Reject package if any other package copied from same PPA is invalid""" + accept = excuse.is_valid + britney_excuses = self.britney.excuses + version = source_data_srcdist.version + sourceppa = self.lp_get_source_ppa(source_name, version) + self.source_ppas_by_pkg[source_name][version] = sourceppa + if sourceppa: + # Check for other packages that might invalidate this one + for friend in self.pkgs_by_source_ppa[sourceppa]: + if not britney_excuses[friend].is_valid: + accept = False + self.pkgs_by_source_ppa[sourceppa].add(source_name) + if not accept: + # Invalidate all packages in this source ppa + shortppa = sourceppa.replace(LAUNCHPAD_URL, '') + for friend in self.pkgs_by_source_ppa[sourceppa]: + friend_exc = britney_excuses.get(friend, excuse) + friend_exc.is_valid = False + friend_exc.addreason('source-ppa') + self.log("Blocking %s because %s from %s" % (friend, source_name, shortppa)) + sourceppa_info[friend] = shortppa + return PolicyVerdict.REJECTED_PERMANENTLY + return PolicyVerdict.PASS + + def save_state(self, britney): + """Write source ppa data to disk""" + tmp = self.filename + '.tmp' + with open(tmp, 'w', encoding='utf-8') as data: + json.dump(self.source_ppas_by_pkg, data) + os.rename(tmp, self.filename) + self.log("Wrote source ppa data to %s" % self.filename) diff --git a/tests/data/sourceppa.json b/tests/data/sourceppa.json new file mode 100644 index 0000000..70e5538 --- /dev/null +++ b/tests/data/sourceppa.json @@ -0,0 +1,7 @@ +{ + "pal": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"}, + "buddy": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"}, + "friend": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"}, + "noppa": {"2.0": ""}, + "unused": {"2.0": ""} +} diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index 53c91c5..b8afaae 100755 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -72,6 +72,19 @@ class T(TestBase): 'Conflicts': 'green'}, testsuite='specialtest') + # Set up sourceppa cache for testing + self.sourceppa_cache = { + 'gcc-5': {'2': ''}, + 'gcc-snapshot': {'2': ''}, + 'green': {'2': '', '1.1': '', '3': ''}, + 'lightgreen': {'2': '', '1.1~beta': '', '3': ''}, + 'linux-meta-64only': {'1': ''}, + 'linux-meta-lts-grumpy': {'1': ''}, + 'linux-meta': {'0.2': '', '1': '', '2': ''}, + 'linux': {'2': ''}, + 'newgreen': {'2': ''}, + } + # create mock Swift server (but don't start it yet, as tests first need # to poke in results) self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085) @@ -96,6 +109,16 @@ class T(TestBase): ''' for (pkg, fields, testsuite) in unstable_add: self.data.add(pkg, True, fields, True, testsuite) + self.sourceppa_cache.setdefault(pkg, {}) + if fields['Version'] not in self.sourceppa_cache[pkg]: + self.sourceppa_cache[pkg][fields['Version']] = '' + + # Set up sourceppa cache for testing + sourceppa_path = os.path.join(self.data.dirs[True], 'SourcePPA') + with open(sourceppa_path, 'w', encoding='utf-8') as sourceppa: + sourceppa.write(json.dumps(self.sourceppa_cache)) + # with open(sourceppa_path, encoding='utf-8') as data: + # print(data.read()) self.swift.start() (excuses_yaml, excuses_html, out) = self.run_britney() @@ -2015,6 +2038,26 @@ class T(TestBase): with open(shared_path) as f: self.assertEqual(orig_contents, f.read()) + ################################################################ + # Tests for source ppa grouping + ################################################################ + + def test_sourceppa(self): + '''Packages from same source PPA should be grouped.''' + self.sourceppa_cache['green'] = {'2': 'team/ubuntu/ppa'} + self.sourceppa_cache['red'] = {'2': 'team/ubuntu/ppa'} + with open(os.path.join(self.data.path, 'data/series-proposed/Blocks'), 'w') as f: + f.write('green 12345 1471505000\ndarkgreen 98765 1471500000\n') + + exc = self.do_test( + [('green', {'Version': '2'}, 'autopkgtest'), + ('red', {'Version': '2'}, 'autopkgtest')], + {'green': (False, {'green': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}})}, + {'green': [('is-candidate', False)], + 'red': [('is-candidate', False)]} + )[1] + self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': 'team/ubuntu/ppa', 'green': 'team/ubuntu/ppa'}) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_sourceppa.py b/tests/test_sourceppa.py new file mode 100755 index 0000000..d1e5aa5 --- /dev/null +++ b/tests/test_sourceppa.py @@ -0,0 +1,133 @@ +#!/usr/bin/python3 +# (C) 2016 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import sys +import unittest +from unittest.mock import patch + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PROJECT_DIR) + +from policies.policy import PolicyVerdict +from policies.sourceppa import LAUNCHPAD_URL, SourcePPAPolicy + + +CACHE_FILE = os.path.join(PROJECT_DIR, 'tests', 'data', 'sourceppa.json') + + +class FakeOptions: + distribution = 'testbuntu' + series = 'zazzy' + unstable = '/tmp' + verbose = False + + +class FakeExcuse: + ver = ('1.0', '2.0') + is_valid = True + + def addreason(self, reason): + """Ignore reasons.""" + + +class FakeBritney: + def __init__(self): + self.excuses = dict( + pal=FakeExcuse(), + buddy=FakeExcuse(), + friend=FakeExcuse(), + noppa=FakeExcuse()) + + +class FakeData: + version = '2.0' + + +class T(unittest.TestCase): + maxDiff = None + + @patch('policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_no_entries(self, urlopen): + """Don't explode if LP reports no entries match pkg/version""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 200 + context.read.return_value = b'{"entries": []}' + pol = SourcePPAPolicy(FakeOptions) + self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), '') + + @patch('policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_no_source_ppa(self, urlopen): + """Identify when package has no source PPA""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 200 + context.read.return_value = b'{"entries": [{"copy_source_archive_link": null, "other_stuff": "ignored"}]}' + pol = SourcePPAPolicy(FakeOptions) + self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), '') + + @patch('policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_with_source_ppa(self, urlopen): + """Identify source PPA""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 200 + context.read.return_value = b'{"entries": [{"copy_source_archive_link": "https://api.launchpad.net/1.0/team/ubuntu/ppa", "other_stuff": "ignored"}]}' + pol = SourcePPAPolicy(FakeOptions) + self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), 'https://api.launchpad.net/1.0/team/ubuntu/ppa') + + @patch('policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_errors(self, urlopen): + """Report errors instead of swallowing them""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 500 + context.read.return_value = b'' + pol = SourcePPAPolicy(FakeOptions) + with self.assertRaisesRegex(ConnectionError, 'HTTP 500'): + pol.lp_get_source_ppa('hello', '1.0') + # Yes, I have really seen "success with no json returned" in the wild + context.getcode.return_value = 200 + context.read.return_value = b'' + with self.assertRaisesRegex(ValueError, 'Expecting value'): + pol.lp_get_source_ppa('hello', '1.0') + + def test_approve_ppa(self): + """Approve packages by their PPA.""" + pol = SourcePPAPolicy(FakeOptions) + pol.filename = CACHE_FILE + pol.initialise(FakeBritney()) + output = {} + for pkg in ('pal', 'buddy', 'friend', 'noppa'): + self.assertEqual(pol.apply_policy_impl(output, None, pkg, None, FakeData, FakeExcuse), PolicyVerdict.PASS) + self.assertEqual(output, {}) + + def test_reject_ppa(self): + """Reject packages by their PPA.""" + shortppa = 'team/ubuntu/ppa' + pol = SourcePPAPolicy(FakeOptions) + pol.filename = CACHE_FILE + brit = FakeBritney() + brit.excuses['buddy'].is_valid = False # Just buddy is invalid but whole ppa fails + pol.initialise(brit) + output = {} + # This one passes because the rejection isn't known yet + self.assertEqual(pol.apply_policy_impl(output, None, 'pal', None, FakeData, brit.excuses['pal']), PolicyVerdict.PASS) + # This one fails because it is itself invalid. + self.assertEqual(pol.apply_policy_impl(output, None, 'buddy', None, FakeData, brit.excuses['buddy']), PolicyVerdict.REJECTED_PERMANENTLY) + # This one fails because buddy failed before it. + self.assertEqual(pol.apply_policy_impl(output, None, 'friend', None, FakeData, brit.excuses['friend']), PolicyVerdict.REJECTED_PERMANENTLY) + # 'noppa' not from PPA so not rejected + self.assertEqual(pol.apply_policy_impl(output, None, 'noppa', None, FakeData, FakeExcuse), PolicyVerdict.PASS) + # All are rejected however + for pkg in ('pal', 'buddy', 'friend'): + self.assertFalse(brit.excuses[pkg].is_valid) + self.assertDictEqual(pol.pkgs_by_source_ppa, { + LAUNCHPAD_URL + shortppa: {'pal', 'buddy', 'friend'}}) + self.assertEqual(output, dict(pal=shortppa, buddy=shortppa, friend=shortppa)) + + +if __name__ == '__main__': + unittest.main()