ubuntutools/archive.py: Rewrite launchpadlib redirects to https when

fetching dscs.

debian/changelog

@@ -1,10 +1,15 @@
 ubuntu-dev-tools (0.117) UNRELEASED; urgency=low
 
+  [ Benjamin Drung ]
   * dgetlp, import-bug-from-debian, suspicious-source:
     Show error messages instead of having the import errors for
     recommended packages (Closes: #613101, LP: #693813).
 
- -- Benjamin Drung <bdrung@debian.org>  Sat, 12 Feb 2011 23:12:22 +0100
+  [ Stefano Rivera ]
+  * ubuntutools/archive.py: Rewrite launchpadlib redirects to https when
+    fetching dscs.
+
+ -- Stefano Rivera <stefanor@ubuntu.com>  Sun, 13 Feb 2011 14:39:50 +0200
 
 ubuntu-dev-tools (0.116) unstable; urgency=low
 

ubuntutools/archive.py

@@ -50,6 +50,15 @@ class DownloadError(Exception):
     pass
 
 
+class ForceHTTPSRedirectHandler(urllib2.HTTPRedirectHandler):
+    "Force redirects from http to https"
+    def redirect_request(self, req, fp, code, msg, hdrs, newurl):
+        if newurl.startswith('http://'):
+            newurl = newurl.replace('http://', 'https://')
+        return urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code,
+                                                            msg, hdrs, newurl)
+
+
 class Dsc(debian.deb822.Dsc):
     "Extend deb822's Dsc with checksum verification abilities"
 
@@ -280,8 +289,15 @@ class SourcePackage(object):
         else:
             Logger.normal('Downloading %s', logurl)
 
+        # Launchpad will try to redirect us to plain-http Launchpad Librarian,
+        # but we want SSL when fetching the dsc
+        if url.startswith('https') and url.endswith('.dsc'):
+            opener = urllib2.build_opener(ForceHTTPSRedirectHandler())
+        else:
+            opener = urllib2.build_opener()
+
         try:
-            in_ = urllib2.urlopen(url)
+            in_ = opener.open(url)
         except urllib2.URLError:
             return False
 

ubuntutools/test/test_archive.py

@@ -91,8 +91,11 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
 
         self.mox.StubOutWithMock(ubuntutools.archive, 'Distribution')
         self.mox.StubOutWithMock(ubuntutools.archive, 'rmadison')
-        self.urlopen = urllib2.urlopen
+
-        self.mox.StubOutWithMock(urllib2, 'urlopen')
+        self.real_opener = urllib2.build_opener()
+        self.mox.StubOutWithMock(urllib2, 'build_opener')
+        self.mock_opener = self.mox.CreateMock(urllib2.OpenerDirector)
+
         # Silence the tests a little:
         self.mox.stubs.Set(Logger, 'stdout', StringIO.StringIO())
         self.mox.stubs.Set(Logger, 'stderr', StringIO.StringIO())
@@ -105,9 +108,8 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
         "Grab the file from test-data"
         if destname is None:
             destname = os.path.basename(url)
-        return self.urlopen('file://'
+        destpath = os.path.join(os.path.abspath('test-data'), destname)
-                            + os.path.join(os.path.abspath('test-data'),
+        return self.real_opener.open('file://' + destpath)
-                                           destname))
 
     def urlopen_file(self, filename):
         "Wrapper for urlopen_proxy for named files"
@@ -122,12 +124,13 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
         raise urllib2.HTTPError(url, 404, "Not Found", {}, None)
 
     def test_local_copy(self):
-        urllib2.urlopen(mox.Regex('^file://.*\.dsc$')
+        urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
-                       ).WithSideEffects(self.urlopen)
+        self.mock_opener.open(mox.Regex('^file://.*\.dsc$')
-        urllib2.urlopen(mox.Regex('^file://.*\.orig\.tar\.gz$')
+                             ).WithSideEffects(self.real_opener.open)
-                       ).WithSideEffects(self.urlopen)
+        self.mock_opener.open(mox.Regex('^file://.*\.orig\.tar\.gz$')
-        urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$')
+                             ).WithSideEffects(self.real_opener.open)
-                       ).WithSideEffects(self.urlopen)
+        self.mock_opener.open(mox.Regex('^file://.*\.debian\.tar\.gz$')
+                             ).WithSideEffects(self.real_opener.open)
         self.mox.ReplayAll()
 
         pkg = self.SourcePackage('example', '1.0-1', 'main',
@@ -166,10 +169,11 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
         with open(os.path.join(self.workdir, 'example_1.0-1.debian.tar.gz'),
                   'r+b') as f:
             f.write('CORRUPTION')
-        urllib2.urlopen(mox.Regex('^file://.*\.dsc$')
+        urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
-                       ).WithSideEffects(self.urlopen)
+        self.mock_opener.open(mox.Regex('^file://.*\.dsc$')
-        urllib2.urlopen(mox.Regex('^file://.*\.debian\.tar\.gz$')
+                             ).WithSideEffects(self.real_opener.open)
-                       ).WithSideEffects(self.urlopen)
+        self.mock_opener.open(mox.Regex('^file://.*\.debian\.tar\.gz$')
+                             ).WithSideEffects(self.real_opener.open)
         self.mox.ReplayAll()
 
         pkg = self.SourcePackage('example', '1.0-1', 'main',
@@ -181,12 +185,15 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
         dist = self.SourcePackage.distribution
         mirror = UDTConfig.defaults['%s_MIRROR' % dist.upper()]
         urlbase = '/pool/main/e/example/'
-        urllib2.urlopen('https://launchpad.net/%s/+archive/primary/'
+        urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
+                            ).AndReturn(self.mock_opener)
+        self.mock_opener.open('https://launchpad.net/%s/+archive/primary/'
                               '+files/example_1.0-1.dsc' % dist
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz'
+        urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
+        self.mock_opener.open(mirror + urlbase + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
+        self.mock_opener.open(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
         self.mox.ReplayAll()
 
@@ -199,15 +206,18 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
         mirror = 'http://mirror'
         lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/'
         urlbase = '/pool/main/e/example/'
-        urllib2.urlopen(lpbase + 'example_1.0-1.dsc'
+        urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
+                            ).AndReturn(self.mock_opener)
+        self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz'
+        urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
+        self.mock_opener.open(mirror + urlbase + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_null)
-        urllib2.urlopen(master + urlbase + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(master + urlbase + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_404)
-        urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(lpbase + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
+        self.mock_opener.open(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
         self.mox.ReplayAll()
 
@@ -217,7 +227,9 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
 
     def test_dsc_missing(self):
         lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/'
-        urllib2.urlopen(lpbase + 'example_1.0-1.dsc'
+        urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
+                            ).AndReturn(self.mock_opener)
+        self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_404)
         self.mox.ReplayAll()
 
@@ -236,27 +248,31 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
         debsec_mirror = 'http://mirror/debsec'
         lpbase = 'https://launchpad.net/debian/+archive/primary/+files/'
         base = '/pool/main/e/example/'
-        urllib2.urlopen(lpbase + 'example_1.0-1.dsc'
+        urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
+                            ).MultipleTimes().AndReturn(self.mock_opener)
+        urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
+        self.mox.StubOutWithMock(urllib2, 'urlopen')
+        self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(debian_mirror + base + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(debian_mirror + base + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_null)
-        urllib2.urlopen(debsec_mirror + base + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(debsec_mirror + base + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_404)
-        urllib2.urlopen(debian_master + base + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(debian_master + base + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_404)
-        urllib2.urlopen(debsec_master + base + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(debsec_master + base + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_404)
-        urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(lpbase + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_404)
         urllib2.urlopen('http://snapshot.debian.org/mr/package/example/1.0-1/'
                         'srcfiles?fileinfo=1'
                        ).WithSideEffects(lambda x: StringIO.StringIO(
             '{"fileinfo": {"hashabc": [{"name": "example_1.0.orig.tar.gz"}]}}'
             ))
-        urllib2.urlopen('http://snapshot.debian.org/file/hashabc'
+        self.mock_opener.open('http://snapshot.debian.org/file/hashabc'
                              ).WithSideEffects(self.urlopen_file(
                                  'example_1.0.orig.tar.gz'))
-        urllib2.urlopen(debian_mirror + base + 'example_1.0-1.debian.tar.gz'
+        self.mock_opener.open(debian_mirror + base + 'example_1.0-1.debian.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
         self.mox.ReplayAll()
 
@@ -270,13 +286,16 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
         mirror = 'http://mirror'
         lpbase = 'https://launchpad.net/debian/+archive/primary/+files/'
         base = '/pool/main/e/example/'
-        urllib2.urlopen(lpbase + 'example_1.0-1.dsc'
+        urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
+                            ).AndReturn(self.mock_opener)
+        self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_404)
-        urllib2.urlopen(mirror + base + 'example_1.0-1.dsc'
+        urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
+        self.mock_opener.open(mirror + base + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(mirror + base + 'example_1.0.orig.tar.gz'
+        self.mock_opener.open(mirror + base + 'example_1.0.orig.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
-        urllib2.urlopen(mirror + base + 'example_1.0-1.debian.tar.gz'
+        self.mock_opener.open(mirror + base + 'example_1.0-1.debian.tar.gz'
                              ).WithSideEffects(self.urlopen_proxy)
 
         self.mox.StubOutWithMock(debian.deb822.GpgInfo, 'from_sequence')
@@ -295,9 +314,12 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
         mirror = 'http://mirror'
         lpbase = 'https://launchpad.net/debian/+archive/primary/+files/'
         base = '/pool/main/e/example/'
-        urllib2.urlopen(lpbase + 'example_1.0-1.dsc'
+        urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
+                            ).AndReturn(self.mock_opener)
+        self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_404)
-        urllib2.urlopen(mirror + base + 'example_1.0-1.dsc'
+        urllib2.build_opener().AndReturn(self.mock_opener)
+        self.mock_opener.open(mirror + base + 'example_1.0-1.dsc'
                              ).WithSideEffects(self.urlopen_proxy)
 
         self.mox.StubOutWithMock(debian.deb822.GpgInfo, 'from_sequence')

ubuntutools/test/test_pylint.py

@@ -26,7 +26,8 @@ WHITELIST = [re.compile(': %s$' % x) for x in (
     # http://www.logilab.org/ticket/51250:
     r"Module 'hashlib' has no '(md5|sha(1|224|256|384|512))' member",
     # mox:
-    r"Instance of 'GpgInfo' has no 'WithSideEffects' member",
+    r"Instance of '.+' has no '(WithSideEffects|MultipleTimes|AndReturn)' "
+    r"member",
 )]
 
 class PylintTestCase(unittest.TestCase):