diff --git a/britney.py b/britney.py index adf4083..4797a14 100755 --- a/britney.py +++ b/britney.py @@ -204,6 +204,7 @@ from britney2.migrationitem import MigrationItem from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, PolicyVerdict from britney2.policies.policy import LPBlockBugPolicy from britney2.policies.autopkgtest import AutopkgtestPolicy +from britney2.policies.sourceppa import SourcePPAPolicy from britney2.utils import (old_libraries_format, undo_changes, compute_reverse_tree, possibly_compressed, read_nuninst, write_nuninst, write_heidi, @@ -534,6 +535,7 @@ class Britney(object): self.policies.append(LPBlockBugPolicy(self.options, self.suite_info)) if getattr(self.options, 'adt_enable') == 'yes': self.policies.append(AutopkgtestPolicy(self.options, self.suite_info)) + self.policies.append(SourcePPAPolicy(self.options, self.suite_info)) for policy in self.policies: policy.register_hints(self._hint_parser) diff --git a/britney2/policies/sourceppa.py b/britney2/policies/sourceppa.py new file mode 100644 index 0000000..47f1eef --- /dev/null +++ b/britney2/policies/sourceppa.py @@ -0,0 +1,140 @@ +import os +import json +import socket +import urllib.request +import urllib.parse + +from collections import defaultdict + +from britney2.policies.policy import BasePolicy, PolicyVerdict + + +LAUNCHPAD_URL = 'https://api.launchpad.net/1.0/' +PRIMARY = LAUNCHPAD_URL + 'ubuntu/+archive/primary' +IGNORE = [ + None, + '', + 'IndexError', + LAUNCHPAD_URL + 'ubuntu/+archive/primary', + LAUNCHPAD_URL + 'debian/+archive/primary', +] + + +class SourcePPAPolicy(BasePolicy): + """Migrate packages copied from same source PPA together + + This policy will query launchpad to determine what source PPA packages + were copied from, and ensure that all packages from the same PPA migrate + together. + """ + + def __init__(self, options, suite_info): + super().__init__('source-ppa', options, suite_info, {'unstable'}) + self.filename = os.path.join(options.unstable, 'SourcePPA') + # Dict of dicts; maps pkg name -> pkg version -> source PPA URL + self.source_ppas_by_pkg = defaultdict(dict) + # Dict of sets; maps source PPA URL -> set of source names + self.pkgs_by_source_ppa = defaultdict(set) + self.britney = None + # self.cache contains self.source_ppas_by_pkg from previous run + self.cache = {} + + def query_lp_rest_api(self, obj, query, retries=5): + """Do a Launchpad REST request + + Request ?. + + Returns dict of parsed json result from launchpad. + Raises HTTPError, ValueError, or ConnectionError based on different + transient failures connecting to launchpad. + """ + assert retries > 0 + + url = '%s%s?%s' % (LAUNCHPAD_URL, obj, urllib.parse.urlencode(query)) + try: + with urllib.request.urlopen(url, timeout=30) as req: + code = req.getcode() + if 200 <= code < 300: + return json.loads(req.read().decode('UTF-8')) + raise ConnectionError('Failed to reach launchpad, HTTP %s' + % code) + except socket.timeout: + if retries > 1: + self.log("Timeout downloading '%s', will retry %d more times." + % (url, retries)) + return self.query_lp_rest_api(obj, query, retries - 1) + else: + raise + + def lp_get_source_ppa(self, pkg, version): + """Ask LP what source PPA pkg was copied from""" + cached = self.cache.get(pkg, {}).get(version) + if cached is not None: + return cached + + data = self.query_lp_rest_api('%s/%s' % (self.options.distribution, self.options.series), { + 'ws.op': 'getPackageUploads', + 'archive': PRIMARY, + 'pocket': 'Proposed', + 'name': pkg, + 'version': version, + 'exact_match': 'true', + }) + try: + return data['entries'][0]['copy_source_archive_link'] + # IndexError means no packages in -proposed matched this name/version, + # which is expected to happen when bileto runs britney. + except IndexError: + self.log('SourcePPA getPackageUploads IndexError (%s %s)' % (pkg, version)) + return 'IndexError' + + def initialise(self, britney): + """Load cached source ppa data""" + super().initialise(britney) + self.britney = britney + + if os.path.exists(self.filename): + with open(self.filename, encoding='utf-8') as data: + self.cache = json.load(data) + self.log("Loaded cached source ppa data from %s" % self.filename) + + def apply_policy_impl(self, sourceppa_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse): + """Reject package if any other package copied from same PPA is invalid""" + accept = excuse.is_valid + britney_excuses = self.britney.excuses + version = source_data_srcdist.version + sourceppa = self.lp_get_source_ppa(source_name, version) + self.source_ppas_by_pkg[source_name][version] = sourceppa + if sourceppa in IGNORE: + return PolicyVerdict.PASS + + shortppa = sourceppa.replace(LAUNCHPAD_URL, '') + sourceppa_info[source_name] = shortppa + # Check for other packages that might invalidate this one + for friend in self.pkgs_by_source_ppa[sourceppa]: + sourceppa_info[friend] = shortppa + if not britney_excuses[friend].is_valid: + accept = False + self.pkgs_by_source_ppa[sourceppa].add(source_name) + + if not accept: + # Invalidate all packages in this source ppa + for friend in self.pkgs_by_source_ppa[sourceppa]: + friend_exc = britney_excuses.get(friend, excuse) + if friend_exc.is_valid: + friend_exc.is_valid = False + friend_exc.addreason('source-ppa') + friend_exc.policy_info['source-ppa'] = sourceppa_info + self.log("Blocking %s because %s from %s" % (friend, source_name, shortppa)) + friend_exc.addhtml("Blocking because %s from the same PPA %s is invalid" % + (friend, shortppa)) + return PolicyVerdict.REJECTED_PERMANENTLY + return PolicyVerdict.PASS + + def save_state(self, britney): + """Write source ppa data to disk""" + tmp = self.filename + '.tmp' + with open(tmp, 'w', encoding='utf-8') as data: + json.dump(self.source_ppas_by_pkg, data) + os.rename(tmp, self.filename) + self.log("Wrote source ppa data to %s" % self.filename) diff --git a/tests/data/sourceppa.json b/tests/data/sourceppa.json new file mode 100644 index 0000000..70e5538 --- /dev/null +++ b/tests/data/sourceppa.json @@ -0,0 +1,7 @@ +{ + "pal": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"}, + "buddy": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"}, + "friend": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"}, + "noppa": {"2.0": ""}, + "unused": {"2.0": ""} +} diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py index afb785d..e1a0fd0 100755 --- a/tests/test_autopkgtest.py +++ b/tests/test_autopkgtest.py @@ -72,6 +72,19 @@ class T(TestBase): 'Conflicts': 'green'}, testsuite='specialtest') + # Set up sourceppa cache for testing + self.sourceppa_cache = { + 'gcc-5': {'2': ''}, + 'gcc-snapshot': {'2': ''}, + 'green': {'2': '', '1.1': '', '3': ''}, + 'lightgreen': {'2': '', '1.1~beta': '', '3': ''}, + 'linux-meta-64only': {'1': ''}, + 'linux-meta-lts-grumpy': {'1': ''}, + 'linux-meta': {'0.2': '', '1': '', '2': ''}, + 'linux': {'2': ''}, + 'newgreen': {'2': ''}, + } + # create mock Swift server (but don't start it yet, as tests first need # to poke in results) self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085) @@ -96,6 +109,16 @@ class T(TestBase): ''' for (pkg, fields, testsuite) in unstable_add: self.data.add(pkg, True, fields, True, testsuite) + self.sourceppa_cache.setdefault(pkg, {}) + if fields['Version'] not in self.sourceppa_cache[pkg]: + self.sourceppa_cache[pkg][fields['Version']] = '' + + # Set up sourceppa cache for testing + sourceppa_path = os.path.join(self.data.dirs[True], 'SourcePPA') + with open(sourceppa_path, 'w', encoding='utf-8') as sourceppa: + sourceppa.write(json.dumps(self.sourceppa_cache)) + # with open(sourceppa_path, encoding='utf-8') as data: + # print(data.read()) self.swift.start() (excuses_yaml, excuses_html, out) = self.run_britney() @@ -2036,6 +2059,54 @@ class T(TestBase): with open(shared_path) as f: self.assertEqual(orig_contents, f.read()) + ################################################################ + # Tests for source ppa grouping + ################################################################ + + def test_sourceppa_policy(self): + '''Packages from same source PPA get rejected for failed peer policy''' + + self.sourceppa_cache['green'] = {'2': 'team/ubuntu/ppa'} + self.sourceppa_cache['red'] = {'2': 'team/ubuntu/ppa'} + with open(os.path.join(self.data.path, 'data/series-proposed/Blocks'), 'w') as f: + f.write('green 12345 1471505000\ndarkgreen 98765 1471500000\n') + + exc = self.do_test( + [('green', {'Version': '2'}, 'autopkgtest'), + ('red', {'Version': '2'}, 'autopkgtest'), + ('gcc-5', {}, 'autopkgtest')], + {'green': (False, {'green': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}}), + 'red': (False, {'red': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}}), + 'gcc-5': (True, {}), + }, + {'green': [('reason', 'block')], + 'red': [('reason', 'source-ppa')]} + )[1] + self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': 'team/ubuntu/ppa', 'green': 'team/ubuntu/ppa'}) + + with open(os.path.join(self.data.path, 'data/series-proposed/SourcePPA')) as f: + res = json.load(f) + self.assertEqual(res, {'red': {'2': 'team/ubuntu/ppa'}, + 'green': {'2': 'team/ubuntu/ppa'}, + 'gcc-5': {'1': ''}}) + + def test_sourceppa_missingbuild(self): + '''Packages from same source PPA get rejected for failed peer FTBFS''' + + self.sourceppa_cache['green'] = {'2': 'team/ubuntu/ppa'} + self.sourceppa_cache['red'] = {'2': 'team/ubuntu/ppa'} + + self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'}) + self.data.add('libgreen1', True, {'Version': '2', 'Source': 'green', 'Architecture': 'i386'}, add_src=False) + + exc = self.do_test( + [('red', {'Version': '2'}, 'autopkgtest')], + {'green': (False, {}), 'red': (False, {})}, + {'green': [('missing-builds', {'on-architectures': ['amd64', 'arm64', 'armhf', 'powerpc', 'ppc64el'], + 'on-unimportant-architectures': []})], + 'red': [('reason', 'source-ppa')]} + )[1] + self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': 'team/ubuntu/ppa', 'green': 'team/ubuntu/ppa'}) if __name__ == '__main__': unittest.main() diff --git a/tests/test_sourceppa.py b/tests/test_sourceppa.py new file mode 100755 index 0000000..ce86af4 --- /dev/null +++ b/tests/test_sourceppa.py @@ -0,0 +1,149 @@ +#!/usr/bin/python3 +# (C) 2016 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +import os +import sys +import unittest +from unittest.mock import patch + +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PROJECT_DIR) + +from britney2.policies.policy import PolicyVerdict +from britney2.policies.sourceppa import LAUNCHPAD_URL, SourcePPAPolicy + + +CACHE_FILE = os.path.join(PROJECT_DIR, 'tests', 'data', 'sourceppa.json') + + +class FakeOptions: + distribution = 'testbuntu' + series = 'zazzy' + unstable = '/tmp' + verbose = False + + +class FakeExcuse: + ver = ('1.0', '2.0') + is_valid = True + policy_info = {} + + def addreason(self, reason): + """Ignore reasons.""" + + def addhtml(self, reason): + """Ignore reasons.""" + + +class FakeBritney: + def __init__(self): + self.excuses = dict( + pal=FakeExcuse(), + buddy=FakeExcuse(), + friend=FakeExcuse(), + noppa=FakeExcuse()) + + +class FakeData: + version = '2.0' + + +class T(unittest.TestCase): + maxDiff = None + + @patch('britney2.policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_no_entries(self, urlopen): + """Don't explode if LP reports no entries match pkg/version""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 200 + context.read.return_value = b'{"entries": []}' + pol = SourcePPAPolicy(FakeOptions, {}) + self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), 'IndexError') + + @patch('britney2.policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_no_source_ppa(self, urlopen): + """Identify when package has no source PPA""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 200 + context.read.return_value = b'{"entries": [{"copy_source_archive_link": null, "other_stuff": "ignored"}]}' + pol = SourcePPAPolicy(FakeOptions, {}) + self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), None) + + @patch('britney2.policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_with_source_ppa(self, urlopen): + """Identify source PPA""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 200 + context.read.return_value = b'{"entries": [{"copy_source_archive_link": "https://api.launchpad.net/1.0/team/ubuntu/ppa", "other_stuff": "ignored"}]}' + pol = SourcePPAPolicy(FakeOptions, {}) + self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), 'https://api.launchpad.net/1.0/team/ubuntu/ppa') + + @patch('britney2.policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_errors(self, urlopen): + """Report errors instead of swallowing them""" + context = urlopen.return_value.__enter__.return_value + context.getcode.return_value = 500 + context.read.return_value = b'' + pol = SourcePPAPolicy(FakeOptions, {}) + with self.assertRaisesRegex(ConnectionError, 'HTTP 500'): + pol.lp_get_source_ppa('hello', '1.0') + # Yes, I have really seen "success with no json returned" in the wild + context.getcode.return_value = 200 + context.read.return_value = b'' + with self.assertRaisesRegex(ValueError, 'Expecting value'): + pol.lp_get_source_ppa('hello', '1.0') + + @patch('britney2.policies.sourceppa.urllib.request.urlopen') + def test_lp_rest_api_timeout(self, urlopen): + """If we get a timeout connecting to LP, we try 5 times""" + import socket + # test that we're retried 5 times on timeout + urlopen.side_effect = socket.timeout + pol = SourcePPAPolicy(FakeOptions, {}) + with self.assertRaises(socket.timeout): + pol.lp_get_source_ppa('hello', '1.0') + self.assertEqual(urlopen.call_count, 5) + + def test_approve_ppa(self): + """Approve packages by their PPA.""" + shortppa = 'team/ubuntu/ppa' + pol = SourcePPAPolicy(FakeOptions, {}) + pol.filename = CACHE_FILE + pol.initialise(FakeBritney()) + output = {} + for pkg in ('pal', 'buddy', 'friend', 'noppa'): + self.assertEqual(pol.apply_policy_impl(output, None, pkg, None, FakeData, FakeExcuse), PolicyVerdict.PASS) + self.assertEqual(output, dict(pal=shortppa, buddy=shortppa, friend=shortppa)) + + def test_reject_ppa(self): + """Reject packages by their PPA.""" + shortppa = 'team/ubuntu/ppa' + pol = SourcePPAPolicy(FakeOptions, {}) + pol.filename = CACHE_FILE + brit = FakeBritney() + brit.excuses['buddy'].is_valid = False # Just buddy is invalid but whole ppa fails + pol.initialise(brit) + output = {} + # This one passes because the rejection isn't known yet + self.assertEqual(pol.apply_policy_impl(output, None, 'pal', None, FakeData, brit.excuses['pal']), PolicyVerdict.PASS) + # This one fails because it is itself invalid. + self.assertEqual(pol.apply_policy_impl(output, None, 'buddy', None, FakeData, brit.excuses['buddy']), PolicyVerdict.REJECTED_PERMANENTLY) + # This one fails because buddy failed before it. + self.assertEqual(pol.apply_policy_impl(output, None, 'friend', None, FakeData, brit.excuses['friend']), PolicyVerdict.REJECTED_PERMANENTLY) + # 'noppa' not from PPA so not rejected + self.assertEqual(pol.apply_policy_impl(output, None, 'noppa', None, FakeData, FakeExcuse), PolicyVerdict.PASS) + # All are rejected however + for pkg in ('pal', 'buddy', 'friend'): + self.assertFalse(brit.excuses[pkg].is_valid) + self.assertDictEqual(pol.pkgs_by_source_ppa, { + LAUNCHPAD_URL + shortppa: {'pal', 'buddy', 'friend'}}) + self.assertEqual(output, dict(pal=shortppa, buddy=shortppa, friend=shortppa)) + + +if __name__ == '__main__': + unittest.main()