Lubuntu/britney2-ubuntu
8
0
Fork 0
mirror of https://git.launchpad.net/~ubuntu-release/britney/+git/britney2-ubuntu synced 2025-10-31 16:44:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reject packages if entire source ppa won't migrate

Browse Source
This commit is contained in:
Robert Bruce Park 2016-10-04 16:55:40 -07:00 committed by Martin Pitt
parent a217ea6ade
commit c5c9c6f979
5 changed files with 369 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
britney.py
View File

@ -204,6 +204,7 @@ from britney2.migrationitem import MigrationItem
from britney2.policies.policy import AgePolicy, RCBugPolicy, PiupartsPolicy, PolicyVerdict
from britney2.policies.policy import LPBlockBugPolicy
from britney2.policies.autopkgtest import AutopkgtestPolicy
from britney2.policies.sourceppa import SourcePPAPolicy
from britney2.utils import (old_libraries_format, undo_changes,
compute_reverse_tree, possibly_compressed,
read_nuninst, write_nuninst, write_heidi,
@ -534,6 +535,7 @@ class Britney(object):
self.policies.append(LPBlockBugPolicy(self.options, self.suite_info))
if getattr(self.options, 'adt_enable') == 'yes':
self.policies.append(AutopkgtestPolicy(self.options, self.suite_info))
self.policies.append(SourcePPAPolicy(self.options, self.suite_info))
for policy in self.policies:
policy.register_hints(self._hint_parser)

140
britney2/policies/sourceppa.py Normal file
View File

@ -0,0 +1,140 @@
import os
import json
import socket
import urllib.request
import urllib.parse
from collections import defaultdict
from britney2.policies.policy import BasePolicy, PolicyVerdict
LAUNCHPAD_URL = 'https://api.launchpad.net/1.0/'
PRIMARY = LAUNCHPAD_URL + 'ubuntu/+archive/primary'
IGNORE = [
None,
'',
'IndexError',
LAUNCHPAD_URL + 'ubuntu/+archive/primary',
LAUNCHPAD_URL + 'debian/+archive/primary',
]
class SourcePPAPolicy(BasePolicy):
"""Migrate packages copied from same source PPA together
This policy will query launchpad to determine what source PPA packages
were copied from, and ensure that all packages from the same PPA migrate
together.
"""
def __init__(self, options, suite_info):
super().__init__('source-ppa', options, suite_info, {'unstable'})
self.filename = os.path.join(options.unstable, 'SourcePPA')
# Dict of dicts; maps pkg name -> pkg version -> source PPA URL
self.source_ppas_by_pkg = defaultdict(dict)
# Dict of sets; maps source PPA URL -> set of source names
self.pkgs_by_source_ppa = defaultdict(set)
self.britney = None
# self.cache contains self.source_ppas_by_pkg from previous run
self.cache = {}
def query_lp_rest_api(self, obj, query, retries=5):
"""Do a Launchpad REST request
Request <LAUNCHPAD_URL><obj>?<query>.
Returns dict of parsed json result from launchpad.
Raises HTTPError, ValueError, or ConnectionError based on different
transient failures connecting to launchpad.
"""
assert retries > 0
url = '%s%s?%s' % (LAUNCHPAD_URL, obj, urllib.parse.urlencode(query))
try:
with urllib.request.urlopen(url, timeout=30) as req:
code = req.getcode()
if 200 <= code < 300:
return json.loads(req.read().decode('UTF-8'))
raise ConnectionError('Failed to reach launchpad, HTTP %s'
% code)
except socket.timeout:
if retries > 1:
self.log("Timeout downloading '%s', will retry %d more times."
% (url, retries))
return self.query_lp_rest_api(obj, query, retries - 1)
else:
raise
def lp_get_source_ppa(self, pkg, version):
"""Ask LP what source PPA pkg was copied from"""
cached = self.cache.get(pkg, {}).get(version)
if cached is not None:
return cached
data = self.query_lp_rest_api('%s/%s' % (self.options.distribution, self.options.series), {
'ws.op': 'getPackageUploads',
'archive': PRIMARY,
'pocket': 'Proposed',
'name': pkg,
'version': version,
'exact_match': 'true',
})
try:
return data['entries'][0]['copy_source_archive_link']
# IndexError means no packages in -proposed matched this name/version,
# which is expected to happen when bileto runs britney.
except IndexError:
self.log('SourcePPA getPackageUploads IndexError (%s %s)' % (pkg, version))
return 'IndexError'
def initialise(self, britney):
"""Load cached source ppa data"""
super().initialise(britney)
self.britney = britney
if os.path.exists(self.filename):
with open(self.filename, encoding='utf-8') as data:
self.cache = json.load(data)
self.log("Loaded cached source ppa data from %s" % self.filename)
def apply_policy_impl(self, sourceppa_info, suite, source_name, source_data_tdist, source_data_srcdist, excuse):
"""Reject package if any other package copied from same PPA is invalid"""
accept = excuse.is_valid
britney_excuses = self.britney.excuses
version = source_data_srcdist.version
sourceppa = self.lp_get_source_ppa(source_name, version)
self.source_ppas_by_pkg[source_name][version] = sourceppa
if sourceppa in IGNORE:
return PolicyVerdict.PASS
shortppa = sourceppa.replace(LAUNCHPAD_URL, '')
sourceppa_info[source_name] = shortppa
# Check for other packages that might invalidate this one
for friend in self.pkgs_by_source_ppa[sourceppa]:
sourceppa_info[friend] = shortppa
if not britney_excuses[friend].is_valid:
accept = False
self.pkgs_by_source_ppa[sourceppa].add(source_name)
if not accept:
# Invalidate all packages in this source ppa
for friend in self.pkgs_by_source_ppa[sourceppa]:
friend_exc = britney_excuses.get(friend, excuse)
if friend_exc.is_valid:
friend_exc.is_valid = False
friend_exc.addreason('source-ppa')
friend_exc.policy_info['source-ppa'] = sourceppa_info
self.log("Blocking %s because %s from %s" % (friend, source_name, shortppa))
friend_exc.addhtml("Blocking because %s from the same PPA %s is invalid" %
(friend, shortppa))
return PolicyVerdict.REJECTED_PERMANENTLY
return PolicyVerdict.PASS
def save_state(self, britney):
"""Write source ppa data to disk"""
tmp = self.filename + '.tmp'
with open(tmp, 'w', encoding='utf-8') as data:
json.dump(self.source_ppas_by_pkg, data)
os.rename(tmp, self.filename)
self.log("Wrote source ppa data to %s" % self.filename)

7
tests/data/sourceppa.json Normal file
View File

@ -0,0 +1,7 @@
{
"pal": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"},
"buddy": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"},
"friend": {"2.0": "https://api.launchpad.net/1.0/team/ubuntu/ppa"},
"noppa": {"2.0": ""},
"unused": {"2.0": ""}
}

71
tests/test_autopkgtest.py
View File

@ -72,6 +72,19 @@ class T(TestBase):
'Conflicts': 'green'},
testsuite='specialtest')
# Set up sourceppa cache for testing
self.sourceppa_cache = {
'gcc-5': {'2': ''},
'gcc-snapshot': {'2': ''},
'green': {'2': '', '1.1': '', '3': ''},
'lightgreen': {'2': '', '1.1~beta': '', '3': ''},
'linux-meta-64only': {'1': ''},
'linux-meta-lts-grumpy': {'1': ''},
'linux-meta': {'0.2': '', '1': '', '2': ''},
'linux': {'2': ''},
'newgreen': {'2': ''},
}
# create mock Swift server (but don't start it yet, as tests first need
# to poke in results)
self.swift = mock_swift.AutoPkgTestSwiftServer(port=18085)
@ -96,6 +109,16 @@ class T(TestBase):
'''
for (pkg, fields, testsuite) in unstable_add:
self.data.add(pkg, True, fields, True, testsuite)
self.sourceppa_cache.setdefault(pkg, {})
if fields['Version'] not in self.sourceppa_cache[pkg]:
self.sourceppa_cache[pkg][fields['Version']] = ''
# Set up sourceppa cache for testing
sourceppa_path = os.path.join(self.data.dirs[True], 'SourcePPA')
with open(sourceppa_path, 'w', encoding='utf-8') as sourceppa:
sourceppa.write(json.dumps(self.sourceppa_cache))
# with open(sourceppa_path, encoding='utf-8') as data:
# print(data.read())
self.swift.start()
(excuses_yaml, excuses_html, out) = self.run_britney()
@ -2036,6 +2059,54 @@ class T(TestBase):
with open(shared_path) as f:
self.assertEqual(orig_contents, f.read())
################################################################
# Tests for source ppa grouping
################################################################
def test_sourceppa_policy(self):
'''Packages from same source PPA get rejected for failed peer policy'''
self.sourceppa_cache['green'] = {'2': 'team/ubuntu/ppa'}
self.sourceppa_cache['red'] = {'2': 'team/ubuntu/ppa'}
with open(os.path.join(self.data.path, 'data/series-proposed/Blocks'), 'w') as f:
f.write('green 12345 1471505000\ndarkgreen 98765 1471500000\n')
exc = self.do_test(
[('green', {'Version': '2'}, 'autopkgtest'),
('red', {'Version': '2'}, 'autopkgtest'),
('gcc-5', {}, 'autopkgtest')],
{'green': (False, {'green': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}}),
'red': (False, {'red': {'i386': 'RUNNING-ALWAYSFAIL', 'amd64': 'RUNNING-ALWAYSFAIL'}}),
'gcc-5': (True, {}),
},
{'green': [('reason', 'block')],
'red': [('reason', 'source-ppa')]}
)[1]
self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': 'team/ubuntu/ppa', 'green': 'team/ubuntu/ppa'})
with open(os.path.join(self.data.path, 'data/series-proposed/SourcePPA')) as f:
res = json.load(f)
self.assertEqual(res, {'red': {'2': 'team/ubuntu/ppa'},
'green': {'2': 'team/ubuntu/ppa'},
'gcc-5': {'1': ''}})
def test_sourceppa_missingbuild(self):
'''Packages from same source PPA get rejected for failed peer FTBFS'''
self.sourceppa_cache['green'] = {'2': 'team/ubuntu/ppa'}
self.sourceppa_cache['red'] = {'2': 'team/ubuntu/ppa'}
self.data.add_src('green', True, {'Version': '2', 'Testsuite': 'autopkgtest'})
self.data.add('libgreen1', True, {'Version': '2', 'Source': 'green', 'Architecture': 'i386'}, add_src=False)
exc = self.do_test(
[('red', {'Version': '2'}, 'autopkgtest')],
{'green': (False, {}), 'red': (False, {})},
{'green': [('missing-builds', {'on-architectures': ['amd64', 'arm64', 'armhf', 'powerpc', 'ppc64el'],
'on-unimportant-architectures': []})],
'red': [('reason', 'source-ppa')]}
)[1]
self.assertEqual(exc['red']['policy_info']['source-ppa'], {'red': 'team/ubuntu/ppa', 'green': 'team/ubuntu/ppa'})
if __name__ == '__main__':
unittest.main()

149
tests/test_sourceppa.py Executable file
View File

@ -0,0 +1,149 @@
#!/usr/bin/python3
# (C) 2016 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
import os
import sys
import unittest
from unittest.mock import patch
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, PROJECT_DIR)
from britney2.policies.policy import PolicyVerdict
from britney2.policies.sourceppa import LAUNCHPAD_URL, SourcePPAPolicy
CACHE_FILE = os.path.join(PROJECT_DIR, 'tests', 'data', 'sourceppa.json')
class FakeOptions:
distribution = 'testbuntu'
series = 'zazzy'
unstable = '/tmp'
verbose = False
class FakeExcuse:
ver = ('1.0', '2.0')
is_valid = True
policy_info = {}
def addreason(self, reason):
"""Ignore reasons."""
def addhtml(self, reason):
"""Ignore reasons."""
class FakeBritney:
def __init__(self):
self.excuses = dict(
pal=FakeExcuse(),
buddy=FakeExcuse(),
friend=FakeExcuse(),
noppa=FakeExcuse())
class FakeData:
version = '2.0'
class T(unittest.TestCase):
maxDiff = None
@patch('britney2.policies.sourceppa.urllib.request.urlopen')
def test_lp_rest_api_no_entries(self, urlopen):
"""Don't explode if LP reports no entries match pkg/version"""
context = urlopen.return_value.__enter__.return_value
context.getcode.return_value = 200
context.read.return_value = b'{"entries": []}'
pol = SourcePPAPolicy(FakeOptions, {})
self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), 'IndexError')
@patch('britney2.policies.sourceppa.urllib.request.urlopen')
def test_lp_rest_api_no_source_ppa(self, urlopen):
"""Identify when package has no source PPA"""
context = urlopen.return_value.__enter__.return_value
context.getcode.return_value = 200
context.read.return_value = b'{"entries": [{"copy_source_archive_link": null, "other_stuff": "ignored"}]}'
pol = SourcePPAPolicy(FakeOptions, {})
self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), None)
@patch('britney2.policies.sourceppa.urllib.request.urlopen')
def test_lp_rest_api_with_source_ppa(self, urlopen):
"""Identify source PPA"""
context = urlopen.return_value.__enter__.return_value
context.getcode.return_value = 200
context.read.return_value = b'{"entries": [{"copy_source_archive_link": "https://api.launchpad.net/1.0/team/ubuntu/ppa", "other_stuff": "ignored"}]}'
pol = SourcePPAPolicy(FakeOptions, {})
self.assertEqual(pol.lp_get_source_ppa('hello', '1.0'), 'https://api.launchpad.net/1.0/team/ubuntu/ppa')
@patch('britney2.policies.sourceppa.urllib.request.urlopen')
def test_lp_rest_api_errors(self, urlopen):
"""Report errors instead of swallowing them"""
context = urlopen.return_value.__enter__.return_value
context.getcode.return_value = 500
context.read.return_value = b''
pol = SourcePPAPolicy(FakeOptions, {})
with self.assertRaisesRegex(ConnectionError, 'HTTP 500'):
pol.lp_get_source_ppa('hello', '1.0')
# Yes, I have really seen "success with no json returned" in the wild
context.getcode.return_value = 200
context.read.return_value = b''
with self.assertRaisesRegex(ValueError, 'Expecting value'):
pol.lp_get_source_ppa('hello', '1.0')
@patch('britney2.policies.sourceppa.urllib.request.urlopen')
def test_lp_rest_api_timeout(self, urlopen):
"""If we get a timeout connecting to LP, we try 5 times"""
import socket
# test that we're retried 5 times on timeout
urlopen.side_effect = socket.timeout
pol = SourcePPAPolicy(FakeOptions, {})
with self.assertRaises(socket.timeout):
pol.lp_get_source_ppa('hello', '1.0')
self.assertEqual(urlopen.call_count, 5)
def test_approve_ppa(self):
"""Approve packages by their PPA."""
shortppa = 'team/ubuntu/ppa'
pol = SourcePPAPolicy(FakeOptions, {})
pol.filename = CACHE_FILE
pol.initialise(FakeBritney())
output = {}
for pkg in ('pal', 'buddy', 'friend', 'noppa'):
self.assertEqual(pol.apply_policy_impl(output, None, pkg, None, FakeData, FakeExcuse), PolicyVerdict.PASS)
self.assertEqual(output, dict(pal=shortppa, buddy=shortppa, friend=shortppa))
def test_reject_ppa(self):
"""Reject packages by their PPA."""
shortppa = 'team/ubuntu/ppa'
pol = SourcePPAPolicy(FakeOptions, {})
pol.filename = CACHE_FILE
brit = FakeBritney()
brit.excuses['buddy'].is_valid = False # Just buddy is invalid but whole ppa fails
pol.initialise(brit)
output = {}
# This one passes because the rejection isn't known yet
self.assertEqual(pol.apply_policy_impl(output, None, 'pal', None, FakeData, brit.excuses['pal']), PolicyVerdict.PASS)
# This one fails because it is itself invalid.
self.assertEqual(pol.apply_policy_impl(output, None, 'buddy', None, FakeData, brit.excuses['buddy']), PolicyVerdict.REJECTED_PERMANENTLY)
# This one fails because buddy failed before it.
self.assertEqual(pol.apply_policy_impl(output, None, 'friend', None, FakeData, brit.excuses['friend']), PolicyVerdict.REJECTED_PERMANENTLY)
# 'noppa' not from PPA so not rejected
self.assertEqual(pol.apply_policy_impl(output, None, 'noppa', None, FakeData, FakeExcuse), PolicyVerdict.PASS)
# All are rejected however
for pkg in ('pal', 'buddy', 'friend'):
self.assertFalse(brit.excuses[pkg].is_valid)
self.assertDictEqual(pol.pkgs_by_source_ppa, {
LAUNCHPAD_URL + shortppa: {'pal', 'buddy', 'friend'}})
self.assertEqual(output, dict(pal=shortppa, buddy=shortppa, friend=shortppa))
if __name__ == '__main__':
unittest.main()