Use httplib2 everywhere that we do https. The python stdlib doesn't do

certificate verification.
This commit is contained in:
Stefano Rivera 2011-11-22 15:57:02 +02:00
parent b70681960f
commit 70a035f13d
9 changed files with 162 additions and 135 deletions

2
debian/changelog vendored
View File

@ -2,6 +2,8 @@ ubuntu-dev-tools (0.136ubuntu1) UNRELEASED; urgency=low
* mk-sbuild: Make --eatmydata command line flag actually work.
* Remove dgetlp. No longer needed.
* Use httplib2 everywhere that we do https. The python stdlib doesn't do
certificate verification.
-- Stefano Rivera <stefanor@debian.org> Mon, 21 Nov 2011 09:47:00 +0200

2
debian/control vendored
View File

@ -17,6 +17,7 @@ Build-Depends: dctrl-tools,
python-apt (>= 0.7.93~),
python-debian (>= 0.1.20~),
python-distro-info (>= 0.4~),
python-httplib2,
python-launchpadlib (>= 1.5.7),
python-mox,
python-setuptools,
@ -38,6 +39,7 @@ Depends: binutils,
python-apt (>= 0.7.93~),
python-debian (>= 0.1.20~),
python-distro-info (>= 0.4~),
python-httplib2,
python-launchpadlib (>= 1.5.7),
python-lazr.restfulclient,
sudo,

View File

@ -20,11 +20,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import urllib2
import json
from httplib2 import Http, HttpLib2Error
import ubuntutools.misc
def main():
ubuntutools.misc.require_utf8()
if len(sys.argv) > 1:
@ -36,8 +38,19 @@ def main():
'restricted', 'restricted-manual',
'universe', 'universe-manual',
'multiverse', 'multiverse-manual'):
page = urllib2.urlopen('http://merges.ubuntu.com/%s.json' % component)
for merge in json.load(page):
url = 'https://merges.ubuntu.com/%s.json' % component
try:
headers, page = Http().request(url)
except HttpLib2Error, e:
print >> sys.stderr, str(e)
sys.exit(1)
if headers.status != 200:
print >> sys.stderr, "%s: %s %s" % (url, headers.status,
headers.reason)
sys.exit(1)
for merge in json.loads(page):
package = merge['source_package']
author, uploader = '', ''
if 'user' in merge:

View File

@ -39,6 +39,7 @@ import sys
from debian.changelog import Changelog, Version
import debian.deb822
import debian.debian_support
import httplib2
from devscripts.logger import Logger
@ -47,20 +48,12 @@ from ubuntutools.lp.lpapicache import (Launchpad, Distribution,
SourcePackagePublishingHistory)
from ubuntutools import subprocess
class DownloadError(Exception):
"Unable to pull a source package"
pass
class ForceHTTPSRedirectHandler(urllib2.HTTPRedirectHandler):
"Force redirects from http to https"
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
if newurl.startswith('http://'):
newurl = newurl.replace('http://', 'https://')
return urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code,
msg, hdrs, newurl)
class Dsc(debian.deb822.Dsc):
"Extend deb822's Dsc with checksum verification abilities"
@ -248,18 +241,19 @@ class SourcePackage(object):
def _download_dsc(self, url):
"Download specified dscfile and parse"
# Launchpad will try to redirect us to plain-http Launchpad Librarian,
# but we want SSL when fetching the dsc
if url.startswith('https') and url.endswith('.dsc'):
opener = urllib2.build_opener(ForceHTTPSRedirectHandler())
parsed = urlparse.urlparse(url)
if parsed.scheme == 'file':
with open(parsed.path, 'r') as f:
body = f.read()
else:
opener = urllib2.build_opener()
try:
f = opener.open(url)
self._dsc = Dsc(f.read())
f.close()
except urllib2.URLError:
raise DownloadError('dsc not found')
try:
response, body = httplib2.Http().request(url)
except httplib2.HttpLib2Error, e:
raise DownloadError(e)
if response.status != 200:
raise DownloadError("%s: %s %s" % (url, response.status,
response.reason))
self._dsc = Dsc(body)
self._dsc_fetched = True
def _check_dsc(self, verify_signature=False):
@ -315,15 +309,18 @@ class SourcePackage(object):
if entry['name'] == filename]
assert len(size) == 1
size = int(size[0])
host = urlparse.urlparse(url).hostname
parsed = urlparse.urlparse(url)
if not self.quiet:
Logger.normal('Downloading %s from %s (%0.3f MiB)',
filename, host, size / 1024.0 / 1024)
filename, parsed.hostname, size / 1024.0 / 1024)
try:
in_ = urllib2.build_opener().open(url)
except urllib2.URLError:
return False
if parsed.scheme == 'file':
in_ = open(parsed.path, 'r')
else:
try:
in_ = urllib2.urlopen(url)
except urllib2.URLError:
return False
downloaded = 0
bar_width = 60

View File

@ -26,9 +26,9 @@
#httplib2.debuglevel = 1
import sys
import urllib2
from debian.changelog import Changelog, Version
from httplib2 import Http, HttpLib2Error
from launchpadlib.launchpad import Launchpad as LP
from launchpadlib.errors import HTTPError
from lazr.restfulclient.resource import Entry
@ -392,10 +392,15 @@ class SourcePackagePublishingHistory(BaseWrapper):
return None
try:
self._changelog = urllib2.urlopen(url).read()
except urllib2.HTTPError, error:
print >> sys.stderr, ('%s: %s' % (url, error))
response, changelog = Http().request(url)
except HttpLib2Error, e:
print >> sys.stderr, str(e)
return None
if response.status != 200:
print >> sys.stderr, ('%s: %s %s' % (url, response.status,
response.reason))
return None
self._changelog = changelog
if since_version is None:
return self._changelog

View File

@ -14,7 +14,8 @@
import json
import os
import urllib2
import httplib2
class RDependsException(Exception):
@ -29,7 +30,7 @@ def query_rdepends(package, release, arch,
url = os.path.join(server, 'v1', release, arch, package)
try:
return json.load(urllib2.urlopen(url))
except urllib2.HTTPError, e:
raise RDependsException(e.read().strip())
response, data = httplib2.Http().request(url)
if response.status != 200:
raise RDependsException(data.strip())
return json.loads(data)

View File

@ -21,10 +21,11 @@
# of the GNU General Public License license.
import re
import urllib2
from debian.deb822 import Changes
from devscripts.logger import Logger
from distro_info import DebianDistroInfo
from httplib2 import Http, HttpLib2Error
from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam,
DistributionSourcePackage)
@ -109,7 +110,17 @@ def get_ubuntu_delta_changelog(srcpkg):
if changes_url is None:
# Native sync
break
changes = Changes(urllib2.urlopen(changes_url))
try:
response, body = Http().request(changes_url)
except HttpLib2Error, e:
Logger.error(str(e))
break
if response.status != 200:
Logger.error("%s: %s %s", changes_url, response.status,
response.reason)
break
changes = Changes(Http().request(changes_url)[1])
for line in changes['Changes'].splitlines():
line = line[1:]
m = topline.match(line)

View File

@ -21,6 +21,7 @@ import urllib
import debian.debian_support
import distro_info
import httplib2
from devscripts.logger import Logger
@ -58,9 +59,17 @@ class BugTask(object):
for url in source_files:
filename = urllib.unquote(os.path.basename(url))
Logger.info("Downloading %s..." % (filename))
urllib.urlretrieve(url, filename)
# HttpLib2 isn't suitable for large files (it reads into memory),
# but we want its https certificate validation on the .dsc
if url.endswith(".dsc"):
response, data = httplib2.Http().request(url)
assert response.status == 200
with open(filename, 'w') as f:
f.write(data)
dsc_file = os.path.join(os.getcwd(), filename)
else:
urllib.urlretrieve(url, filename)
assert os.path.isfile(dsc_file), "%s does not exist." % (dsc_file)
return dsc_file

View File

@ -25,6 +25,7 @@ import types
import urllib2
import debian.deb822
import httplib2
import mox
from devscripts.logger import Logger
@ -94,9 +95,12 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
self.mox.StubOutWithMock(ubuntutools.archive, 'Distribution')
self.mox.StubOutWithMock(ubuntutools.archive, 'rmadison')
self.real_opener = urllib2.build_opener()
self.mox.StubOutWithMock(urllib2, 'build_opener')
self.mock_opener = self.mox.CreateMock(urllib2.OpenerDirector)
self.real_urlopen = urllib2.urlopen
self.mox.StubOutWithMock(urllib2, 'urlopen')
self.real_http = httplib2.Http()
self.mox.StubOutWithMock(httplib2, 'Http')
self.mock_http = self.mox.CreateMock(httplib2.Http)
# Silence the tests a little:
self.mox.stubs.Set(Logger, 'stdout', StringIO.StringIO())
@ -107,11 +111,11 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
shutil.rmtree(self.workdir)
def urlopen_proxy(self, url, destname=None):
"Grab the file from test-data"
"urllib2 proxy for grabbing the file from test-data"
if destname is None:
destname = os.path.basename(url)
destpath = os.path.join(os.path.abspath('test-data'), destname)
return self.real_opener.open('file://' + destpath)
return self.real_urlopen('file://' + destpath)
def urlopen_file(self, filename):
"Wrapper for urlopen_proxy for named files"
@ -125,16 +129,22 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
"urlopen for errors"
raise urllib2.HTTPError(url, 404, "Not Found", {}, None)
def test_local_copy(self):
urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
self.mock_opener.open(mox.Regex('^file://.*\.dsc$')
).WithSideEffects(self.real_opener.open)
self.mock_opener.open(mox.Regex('^file://.*\.orig\.tar\.gz$')
).WithSideEffects(self.real_opener.open)
self.mock_opener.open(mox.Regex('^file://.*\.debian\.tar\.gz$')
).WithSideEffects(self.real_opener.open)
self.mox.ReplayAll()
def request_proxy(self, url, destname=None):
"httplib2 proxy for grabbing the file from test-data"
if destname is None:
destname = os.path.basename(url)
destpath = os.path.join(os.path.abspath('test-data'), destname)
response = httplib2.Response({})
with open(destpath, 'r') as f:
body = f.read()
return response, body
def request_404(self, url):
"httplib2 for errors"
response = httplib2.Response({'status': 404})
return response, "I'm a 404 Error"
def test_local_copy(self):
pkg = self.SourcePackage('example', '1.0-1', 'main',
dscfile='test-data/example_1.0-1.dsc',
workdir=self.workdir)
@ -146,10 +156,6 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
shutil.copy2('test-data/example_1.0.orig.tar.gz', self.workdir)
shutil.copy2('test-data/example_1.0-1.debian.tar.gz', self.workdir)
urllib2.build_opener().AndReturn(self.mock_opener)
self.mock_opener.open(mox.Regex('^file://.*\.dsc$')
).WithSideEffects(self.real_opener.open)
self.mox.ReplayAll()
pkg = self.SourcePackage(dscfile=os.path.join(self.workdir,
'example_1.0-1.dsc'),
workdir=self.workdir)
@ -161,10 +167,6 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
shutil.copy2('test-data/example_1.0.orig.tar.gz', self.workdir)
shutil.copy2('test-data/example_1.0-1.debian.tar.gz', self.workdir)
urllib2.build_opener().AndReturn(self.mock_opener)
self.mock_opener.open(mox.Regex('^file://.*\.dsc$')
).WithSideEffects(self.real_opener.open)
self.mox.ReplayAll()
pkg = self.SourcePackage('example', '1.0-1', 'main',
dscfile=os.path.join(self.workdir,
'example_1.0-1.dsc'),
@ -179,12 +181,6 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
with open(os.path.join(self.workdir, 'example_1.0-1.debian.tar.gz'),
'r+b') as f:
f.write('CORRUPTION')
urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
self.mock_opener.open(mox.Regex('^file://.*\.dsc$')
).WithSideEffects(self.real_opener.open)
self.mock_opener.open(mox.Regex('^file://.*\.debian\.tar\.gz$')
).WithSideEffects(self.real_opener.open)
self.mox.ReplayAll()
pkg = self.SourcePackage('example', '1.0-1', 'main',
dscfile='test-data/example_1.0-1.dsc',
@ -195,16 +191,14 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
dist = self.SourcePackage.distribution
mirror = UDTConfig.defaults['%s_MIRROR' % dist.upper()]
urlbase = '/pool/main/e/example/'
urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
).AndReturn(self.mock_opener)
self.mock_opener.open('https://launchpad.net/%s/+archive/primary/'
'+files/example_1.0-1.dsc' % dist
).WithSideEffects(self.urlopen_proxy)
urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
self.mock_opener.open(mirror + urlbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_proxy)
self.mock_opener.open(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request('https://launchpad.net/%s/+archive/primary/'
'+files/example_1.0-1.dsc' % dist
).WithSideEffects(self.request_proxy)
urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_proxy)
urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
self.mox.ReplayAll()
pkg = self.SourcePackage('example', '1.0-1', 'main',
@ -216,19 +210,17 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
mirror = 'http://mirror'
lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/'
urlbase = '/pool/main/e/example/'
urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
).AndReturn(self.mock_opener)
self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_proxy)
urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
self.mock_opener.open(mirror + urlbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_null)
self.mock_opener.open(master + urlbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
self.mock_opener.open(lpbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_proxy)
self.mock_opener.open(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.request_proxy)
urllib2.urlopen(mirror + urlbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_null)
urllib2.urlopen(master + urlbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_proxy)
urllib2.urlopen(mirror + urlbase + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
self.mox.ReplayAll()
pkg = self.SourcePackage('example', '1.0-1', 'main',
@ -237,10 +229,9 @@ class LocalSourcePackageTestCase(mox.MoxTestBase, unittest.TestCase):
def test_dsc_missing(self):
lpbase = 'https://launchpad.net/ubuntu/+archive/primary/+files/'
urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
).AndReturn(self.mock_opener)
self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_404)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.request_404)
self.mox.ReplayAll()
pkg = self.SourcePackage('example', '1.0-1', 'main',
@ -258,32 +249,30 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
debsec_mirror = 'http://mirror/debsec'
lpbase = 'https://launchpad.net/debian/+archive/primary/+files/'
base = '/pool/main/e/example/'
urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
).MultipleTimes().AndReturn(self.mock_opener)
urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
self.mox.StubOutWithMock(urllib2, 'urlopen')
self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_proxy)
self.mock_opener.open(debian_mirror + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_null)
self.mock_opener.open(debsec_mirror + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
self.mock_opener.open(debian_master + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
self.mock_opener.open(debsec_master + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
self.mock_opener.open(lpbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.request_proxy)
urllib2.urlopen(debian_mirror + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_null)
urllib2.urlopen(debsec_mirror + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
urllib2.urlopen(debian_master + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
urllib2.urlopen(debsec_master + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
urllib2.urlopen(lpbase + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_404)
urllib2.urlopen('http://snapshot.debian.org/mr/package/example/1.0-1/'
'srcfiles?fileinfo=1'
).WithSideEffects(lambda x: StringIO.StringIO(
'{"fileinfo": {"hashabc": [{"name": "example_1.0.orig.tar.gz"}]}}'
))
self.mock_opener.open('http://snapshot.debian.org/file/hashabc'
).WithSideEffects(self.urlopen_file(
'example_1.0.orig.tar.gz'))
self.mock_opener.open(debian_mirror + base + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
urllib2.urlopen('http://snapshot.debian.org/file/hashabc'
).WithSideEffects(self.urlopen_file(
'example_1.0.orig.tar.gz'))
urllib2.urlopen(debian_mirror + base + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
self.mox.ReplayAll()
pkg = self.SourcePackage('example', '1.0-1', 'main',
@ -296,17 +285,16 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
mirror = 'http://mirror'
lpbase = 'https://launchpad.net/debian/+archive/primary/+files/'
base = '/pool/main/e/example/'
urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
).AndReturn(self.mock_opener)
self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_404)
urllib2.build_opener().MultipleTimes().AndReturn(self.mock_opener)
self.mock_opener.open(mirror + base + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_proxy)
self.mock_opener.open(mirror + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_proxy)
self.mock_opener.open(mirror + base + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.request_404)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(mirror + base + 'example_1.0-1.dsc'
).WithSideEffects(self.request_proxy)
urllib2.urlopen(mirror + base + 'example_1.0.orig.tar.gz'
).WithSideEffects(self.urlopen_proxy)
urllib2.urlopen(mirror + base + 'example_1.0-1.debian.tar.gz'
).WithSideEffects(self.urlopen_proxy)
def fake_gpg_info(self, message, keyrings=None):
return debian.deb822.GpgInfo.from_output(
@ -330,13 +318,12 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
mirror = 'http://mirror'
lpbase = 'https://launchpad.net/debian/+archive/primary/+files/'
base = '/pool/main/e/example/'
urllib2.build_opener(mox.IsA(urllib2.HTTPRedirectHandler)
).AndReturn(self.mock_opener)
self.mock_opener.open(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_404)
urllib2.build_opener().AndReturn(self.mock_opener)
self.mock_opener.open(mirror + base + 'example_1.0-1.dsc'
).WithSideEffects(self.urlopen_proxy)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(lpbase + 'example_1.0-1.dsc'
).WithSideEffects(self.request_404)
httplib2.Http().AndReturn(self.mock_http)
self.mock_http.request(mirror + base + 'example_1.0-1.dsc'
).WithSideEffects(self.request_proxy)
def fake_gpg_info(self, message, keyrings=None):
return debian.deb822.GpgInfo.from_output(