From 7f311a285291d365ab070f87406eed6ad1868ce2 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Sun, 13 Feb 2011 14:40:14 +0200 Subject: [PATCH] ubuntutools/archive.py: Rewrite launchpadlib redirects to https when fetching dscs. --- debian/changelog | 7 +- ubuntutools/archive.py | 18 +++- ubuntutools/test/test_archive.py | 148 ++++++++++++++++++------------- ubuntutools/test/test_pylint.py | 3 +- 4 files changed, 110 insertions(+), 66 deletions(-) diff --git a/debian/changelog b/debian/changelog index efb37ec..88a28b8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,15 @@ ubuntu-dev-tools (0.117) UNRELEASED; urgency=low + [ Benjamin Drung ] * dgetlp, import-bug-from-debian, suspicious-source: Show error messages instead of having the import errors for recommended packages (Closes: #613101, LP: #693813). - -- Benjamin Drung Sat, 12 Feb 2011 23:12:22 +0100 + [ Stefano Rivera ] + * ubuntutools/archive.py: Rewrite launchpadlib redirects to https when + fetching dscs. + + -- Stefano Rivera Sun, 13 Feb 2011 14:39:50 +0200 ubuntu-dev-tools (0.116) unstable; urgency=low diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index f37383b..6be9c0b 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -50,6 +50,15 @@ class DownloadError(Exception): pass +class ForceHTTPSRedirectHandler(urllib2.HTTPRedirectHandler): + "Force redirects from http to https" + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + if newurl.startswith('http://'): + newurl = newurl.replace('http://', 'https://') + return urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, + msg, hdrs, newurl) + + class Dsc(debian.deb822.Dsc): "Extend deb822's Dsc with checksum verification abilities" @@ -280,8 +289,15 @@ class SourcePackage(object): else: Logger.normal('Downloading %s', logurl) + # Launchpad will try to redirect us to plain-http Launchpad Librarian, + # but we want SSL when fetching the dsc + if url.startswith('https') and url.endswith('.dsc'): + opener = urllib2.build_opener(ForceHTTPSRedirectHandler()) + else: + opener = urllib2.build_opener() + try: - in_ = urllib2.urlopen(url) + in_ = opener.open(url) except urllib2.URLError: return False diff --git a/ubuntutools/test/test_archive.py b/ubuntutools/test/test_archive.py index 023c049..8f37d10 100644 --- a/ubuntutools/test/test_archive.py +++ b/ubuntutools/test/test_archive.py @@ -91,8 +91,11 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): self.mox.StubOutWithMock(ubuntutools.archive, 'Distribution') self.mox.StubOutWithMock(ubuntutools.archive, 'rmadison') - self.urlopen = urllib2.urlopen - self.mox.StubOutWithMock(urllib2, 'urlopen') + + self.real_opener = urllib2.build_opener() + self.mox.StubOutWithMock(urllib2, 'build_opener') + self.mock_opener = self.mox.CreateMock(urllib2.OpenerDirector) + # Silence the tests a little: self.mox.stubs.Set(Logger, 'stdout', StringIO.StringIO()) self.mox.stubs.Set(Logger, 'stderr', StringIO.StringIO()) @@ -105,9 +108,8 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): "Grab the file from test-data" if destname is None: destname = os.path.basename(url) - return self.urlopen('file://' - + os.path.join(os.path.abspath('test-data'), - destname)) + destpath = os.path.join(os.path.abspath('test-data'), destname) + return self.real_opener.open('file://' + destpath) def urlopen_file(self, filename): "Wrapper for urlopen_proxy for named files" @@ -122,12 +124,13 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): raise urllib2.HTTPError(url, 404, "Not Found", {}, None) def test_local_copy(self): - urllib2.urlopen(mox.Regex('^file://.*\.dsc$') - ).WithSideEffects(self.urlopen) - urllib2.urlopen(mox.Regex('^file://.*\.orig\.tar\.gz$') - ).WithSideEffects(self.urlopen) - urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') - ).WithSideEffects(self.urlopen) + urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener) + self.mock_opener.open(mox.Regex('^file://.*\.dsc$') + ).WithSideEffects(self.real_opener.open) + self.mock_opener.open(mox.Regex('^file://.*\.orig\.tar\.gz$') + ).WithSideEffects(self.real_opener.open) + self.mock_opener.open(mox.Regex('^file://.*\.debian\.tar\.gz$') + ).WithSideEffects(self.real_opener.open) self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -166,10 +169,11 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): with open(os.path.join(self.workdir, 'example_1.0-1.debian.tar.gz'), 'r+b') as f: f.write('CORRUPTION') - urllib2.urlopen(mox.Regex('^file://.*\.dsc$') - ).WithSideEffects(self.urlopen) - urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$') - ).WithSideEffects(self.urlopen) + urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener) + self.mock_opener.open(mox.Regex('^file://.*\.dsc$') + ).WithSideEffects(self.real_opener.open) + self.mock_opener.open(mox.Regex('^file://.*\.debian\.tar\.gz$') + ).WithSideEffects(self.real_opener.open) self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -181,13 +185,16 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): dist = self.SourcePackage.distribution mirror = UDTConfig.defaults['%s_MIRROR' % dist.upper()] urlbase = '/pool/main/e/example/' - urllib2.urlopen('https://launchpad.net/%s/+archive/primary/' - '+files/example_1.0-1.dsc' % dist - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz' - ).WithSideEffects(self.urlopen_proxy) + urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler) + ).AndReturn(self.mock_opener) + self.mock_opener.open('https://launchpad.net/%s/+archive/primary/' + '+files/example_1.0-1.dsc' % dist + ).WithSideEffects(self.urlopen_proxy) + urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener) + self.mock_opener.open(mirror + urlbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + self.mock_opener.open(mirror + urlbase + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -199,16 +206,19 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): mirror = 'http://mirror' lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/' urlbase = '/pool/main/e/example/' - urllib2.urlopen(lpbase + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_null) - urllib2.urlopen(master + urlbase + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_404) - urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz' - ).WithSideEffects(self.urlopen_proxy) + urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler) + ).AndReturn(self.mock_opener) + self.mock_opener.open(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener) + self.mock_opener.open(mirror + urlbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_null) + self.mock_opener.open(master + urlbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + self.mock_opener.open(lpbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + self.mock_opener.open(mirror + urlbase + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -217,8 +227,10 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase): def test_dsc_missing(self): lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/' - urllib2.urlopen(lpbase + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_404) + urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler) + ).AndReturn(self.mock_opener) + self.mock_opener.open(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_404) self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -236,28 +248,32 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): debsec_mirror = 'http://mirror/debsec' lpbase = 'https://launchpad.net/debian/+archive/primary/+files/' base = '/pool/main/e/example/' - urllib2.urlopen(lpbase + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(debian_mirror + base + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_null) - urllib2.urlopen(debsec_mirror + base + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_404) - urllib2.urlopen(debian_master + base + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_404) - urllib2.urlopen(debsec_master + base + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_404) - urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_404) + urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler) + ).MultipleTimes().AndReturn(self.mock_opener) + urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener) + self.mox.StubOutWithMock(urllib2, 'urlopen') + self.mock_opener.open(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + self.mock_opener.open(debian_mirror + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_null) + self.mock_opener.open(debsec_mirror + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + self.mock_opener.open(debian_master + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + self.mock_opener.open(debsec_master + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) + self.mock_opener.open(lpbase + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_404) urllib2.urlopen('http://snapshot.debian.org/mr/package/example/1.0-1/' 'srcfiles?fileinfo=1' ).WithSideEffects(lambda x: StringIO.StringIO( '{"fileinfo": {"hashabc": [{"name": "example_1.0.orig.tar.gz"}]}}' )) - urllib2.urlopen('http://snapshot.debian.org/file/hashabc' - ).WithSideEffects(self.urlopen_file( - 'example_1.0.orig.tar.gz')) - urllib2.urlopen(debian_mirror + base + 'example_1.0-1.debian.tar.gz' - ).WithSideEffects(self.urlopen_proxy) + self.mock_opener.open('http://snapshot.debian.org/file/hashabc' + ).WithSideEffects(self.urlopen_file( + 'example_1.0.orig.tar.gz')) + self.mock_opener.open(debian_mirror + base + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) self.mox.ReplayAll() pkg = self.SourcePackage('example', '1.0-1', 'main', @@ -270,14 +286,17 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): mirror = 'http://mirror' lpbase = 'https://launchpad.net/debian/+archive/primary/+files/' base = '/pool/main/e/example/' - urllib2.urlopen(lpbase + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_404) - urllib2.urlopen(mirror + base + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(mirror + base + 'example_1.0.orig.tar.gz' - ).WithSideEffects(self.urlopen_proxy) - urllib2.urlopen(mirror + base + 'example_1.0-1.debian.tar.gz' - ).WithSideEffects(self.urlopen_proxy) + urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler) + ).AndReturn(self.mock_opener) + self.mock_opener.open(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_404) + urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener) + self.mock_opener.open(mirror + base + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) + self.mock_opener.open(mirror + base + 'example_1.0.orig.tar.gz' + ).WithSideEffects(self.urlopen_proxy) + self.mock_opener.open(mirror + base + 'example_1.0-1.debian.tar.gz' + ).WithSideEffects(self.urlopen_proxy) self.mox.StubOutWithMock(debian.deb822.GpgInfo, 'from_sequence') debian.deb822.GpgInfo.from_sequence(mox.IsA(str)).WithSideEffects( @@ -295,10 +314,13 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase): mirror = 'http://mirror' lpbase = 'https://launchpad.net/debian/+archive/primary/+files/' base = '/pool/main/e/example/' - urllib2.urlopen(lpbase + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_404) - urllib2.urlopen(mirror + base + 'example_1.0-1.dsc' - ).WithSideEffects(self.urlopen_proxy) + urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler) + ).AndReturn(self.mock_opener) + self.mock_opener.open(lpbase + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_404) + urllib2.build_opener().AndReturn(self.mock_opener) + self.mock_opener.open(mirror + base + 'example_1.0-1.dsc' + ).WithSideEffects(self.urlopen_proxy) self.mox.StubOutWithMock(debian.deb822.GpgInfo, 'from_sequence') debian.deb822.GpgInfo.from_sequence(mox.IsA(str)).WithSideEffects( diff --git a/ubuntutools/test/test_pylint.py b/ubuntutools/test/test_pylint.py index e467ac4..80455e2 100644 --- a/ubuntutools/test/test_pylint.py +++ b/ubuntutools/test/test_pylint.py @@ -26,7 +26,8 @@ WHITELIST = [re.compile(': %s$' % x) for x in ( # http://www.logilab.org/ticket/51250: r"Module 'hashlib' has no '(md5|sha(1|224|256|384|512))' member", # mox: - r"Instance of 'GpgInfo' has no 'WithSideEffects' member", + r"Instance of '.+' has no '(WithSideEffects|MultipleTimes|AndReturn)' " + r"member", )] class PylintTestCase(unittest.TestCase):