Lubuntu/ubuntu-dev-tools
3
0
Fork 0
mirror of https://git.launchpad.net/ubuntu-dev-tools synced 2025-05-12 09:21:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

pull-pkg: update archive.py SourcePackage

Browse Source 
verify it can parse/load DSC when created
throw PackageNotFoundException from lp_spph if can't find package
update test case expected order of url processing
This commit is contained in:
Dan Streetman 2017-03-23 11:58:51 -04:00 committed by Dan Streetman
parent 506e3db601
commit b0c22e1d57
2 changed files with 110 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
ubuntutools/archive.py
View File

@ -27,9 +27,9 @@ Approach:
3. Verify checksums. 3. Verify checksums.
""" """
from urllib.error import URLError, HTTPError from urllib.error import (URLError, HTTPError)
from urllib.parse import urlparse from urllib.parse import urlparse
from urllib.request import ProxyHandler, build_opener, urlopen from urllib.request import urlopen
import codecs import codecs
import hashlib import hashlib
import json import json
@ -42,10 +42,13 @@ from debian.changelog import Changelog
import debian.deb822 import debian.deb822
import httplib2 import httplib2
from contextlib import closing
from ubuntutools.config import UDTConfig from ubuntutools.config import UDTConfig
from ubuntutools.lp.lpapicache import (Launchpad, Distribution, from ubuntutools.lp.lpapicache import (Launchpad, Distribution,
SourcePackagePublishingHistory, SourcePackagePublishingHistory,
BinaryPackagePublishingHistory) BinaryPackagePublishingHistory)
from ubuntutools.lp.udtexceptions import PackageNotFoundException
from ubuntutools.logger import Logger from ubuntutools.logger import Logger
from ubuntutools.version import Version from ubuntutools.version import Version
@ -131,50 +134,57 @@ class SourcePackage(object):
self._lp = lp self._lp = lp
self.workdir = workdir self.workdir = workdir
self.quiet = quiet self.quiet = quiet
self._dsc_source = dscfile
# Cached values: # Cached values:
self._distribution = None
self._component = component self._component = component
self._dsc = None self._dsc = None
self._spph = None self._spph = None
self._version = Version(version) if version else None
# State:
self._dsc_fetched = False
# Mirrors # Mirrors
self._dsc_source = dscfile
self.mirrors = list(mirrors) self.mirrors = list(mirrors)
if self.distribution: if self.distribution:
self.masters = [UDTConfig.defaults['%s_MIRROR' self.masters = [UDTConfig.defaults['%s_MIRROR'
% self.distribution.upper()]] % self.distribution.upper()]]
if dscfile is not None:
if self.source is None:
self.source = 'unknown'
if version is None:
version = 'unknown'
self.version = Version(version) # if a dsc was specified, pull it to get the source/version info
if self._dsc_source:
# uses default proxies from the environment self.pull_dsc()
proxy = ProxyHandler()
self.url_opener = build_opener(proxy)
@property @property
def lp_spph(self): def lp_spph(self):
"Return the LP Source Package Publishing History entry" "Return the LP Source Package Publishing History entry"
if not self._spph: if self._spph:
return self._spph
if not Launchpad.logged_in: if not Launchpad.logged_in:
if self._lp: if self._lp:
Launchpad.login_existing(self._lp) Launchpad.login_existing(self._lp)
else: else:
Launchpad.login_anonymously() Launchpad.login_anonymously()
spph = Distribution(self.distribution).getArchive().getPublishedSources(
source_name=self.source, distro = self.getDistribution()
version=self.version.full_version, archive = self.getArchive()
exact_match=True, params = {'exact_match': True, 'order_by_date': True}
) params['version'] = self._version.full_version
self._spph = SourcePackagePublishingHistory(spph[0]) spphs = archive.getPublishedSources(source_name=self.source, **params)
if spphs:
self._spph = SourcePackagePublishingHistory(spphs[0])
return self._spph return self._spph
msg = "No {} package found".format(self.source)
msg += " for version {}".format(self._version.full_version)
raise PackageNotFoundException(msg)
@property
def version(self):
"Return Package version"
if not self._version:
self._version = Version(self.lp_spph.getVersion())
return self._version
@property @property
def component(self): def component(self):
"Cached archive component, in available" "Cached archive component, in available"
@ -200,6 +210,15 @@ class SourcePackage(object):
self.pull_dsc() self.pull_dsc()
return self._dsc return self._dsc
def getDistribution(self):
if not self._distribution:
self._distribution = Distribution(self.distribution)
return self._distribution
def getArchive(self):
return self.getDistribution().getArchive()
def _mirror_url(self, mirror, filename): def _mirror_url(self, mirror, filename):
"Build a source package URL on a mirror" "Build a source package URL on a mirror"
if self.source.startswith('lib'): if self.source.startswith('lib'):
@ -264,13 +283,12 @@ class SourcePackage(object):
raise DownloadError("%s: %s %s" % (url, response.status, raise DownloadError("%s: %s %s" % (url, response.status,
response.reason)) response.reason))
self._dsc = Dsc(body) self._dsc = Dsc(body)
self._dsc_fetched = True
def _check_dsc(self, verify_signature=False): def _check_dsc(self, verify_signature=False):
"Check that the dsc matches what we are expecting" "Check that the dsc matches what we are expecting"
assert self._dsc is not None assert self._dsc is not None
self.source = self.dsc['Source'] self.source = self.dsc['Source']
self.version = Version(self.dsc['Version']) self._version = Version(self.dsc['Version'])
valid = False valid = False
message = None message = None
@ -312,6 +330,43 @@ class SourcePackage(object):
with open(self.dsc_pathname, 'wb') as f: with open(self.dsc_pathname, 'wb') as f:
f.write(self.dsc.raw_text) f.write(self.dsc.raw_text)
def _download_file_helper(self, f, pathname, size):
"Perform the dowload."
BLOCKSIZE = 16 * 1024
with open(pathname, 'wb') as out:
if not (Logger.isEnabledFor(logging.INFO) and
sys.stderr.isatty() and
size):
shutil.copyfileobj(f, out, BLOCKSIZE)
return
XTRALEN = len('[] 99%')
downloaded = 0
bar_width = 60
term_width = os.get_terminal_size(sys.stderr.fileno())[0]
if term_width < bar_width + XTRALEN + 1:
bar_width = term_width - XTRALEN - 1
try:
while True:
block = f.read(BLOCKSIZE)
if not block:
break
out.write(block)
downloaded += len(block)
pct = float(downloaded) / size
bar = ('=' * int(pct * bar_width))[:-1] + '>'
fmt = '[{bar:<%d}]{pct:>3}%%\r' % bar_width
sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100)))
sys.stderr.flush()
finally:
sys.stderr.write(' ' * (bar_width + XTRALEN) + '\r')
if downloaded < size:
Logger.error('Partial download: %0.3f MiB of %0.3f MiB' %
(downloaded / 1024.0 / 1024,
size / 1024.0 / 1024))
def _download_file(self, url, filename, verify=True): def _download_file(self, url, filename, verify=True):
"Download url to filename in workdir." "Download url to filename in workdir."
pathname = os.path.join(self.workdir, filename) pathname = os.path.join(self.workdir, filename)
@ -325,51 +380,32 @@ class SourcePackage(object):
assert len(size) == 1 assert len(size) == 1
size = int(size[0]) size = int(size[0])
parsed = urlparse(url) if urlparse(url).scheme in ["", "file"]:
frompath = os.path.abspath(urlparse(url).path)
if parsed.scheme == 'file': Logger.info("Copying %s from %s" % (filename, frompath))
in_ = open(parsed.path, 'rb') shutil.copyfile(frompath, pathname)
if not size:
size = int(os.stat(parsed.path).st_size)
else: else:
try: try:
in_ = self.url_opener.open(url) with closing(urlopen(url)) as f:
Logger.debug("Using URL '%s'", url) Logger.debug("Using URL '%s'", f.geturl())
except URLError as e:
Logger.debug("URLError opening '%s': %s", url, e)
return False
if not size: if not size:
contentlen = in_.info().get('Content-Length')
if not contentlen:
Logger.error("Invalid response, no Content-Length")
return False
size = int(contentlen)
if not self.quiet:
Logger.normal('Downloading %s from %s (%0.3f MiB)',
filename, parsed.hostname, size / 1024.0 / 1024)
downloaded = 0
bar_width = 60
try: try:
with open(pathname, 'wb') as out: size = int(f.info().get('Content-Length'))
while True: except (AttributeError, TypeError, ValueError):
block = in_.read(10240) pass
if block == b'':
break Logger.info('Downloading %s from %s%s' %
downloaded += len(block) (filename, urlparse(url).hostname,
out.write(block) ' (%0.3f MiB)' % (size / 1024.0 / 1024)
if not self.quiet: if size else ''))
percent = downloaded * 100 // size
bar = '=' * int(round(downloaded * bar_width / size)) self._download_file_helper(f, pathname, size)
bar = (bar + '>' + ' ' * bar_width)[:bar_width] except HTTPError as e:
Logger.stdout.write('[%s] %#3i%%\r' % (bar, percent)) # It's ok if the file isn't found; we try multiple places to download
Logger.stdout.flush() if e.code == 404:
in_.close() return False
finally: raise e
if not self.quiet:
Logger.stdout.write(' ' * (bar_width + 7) + '\r')
Logger.stdout.flush()
if verify and not self.dsc.verify_file(pathname): if verify and not self.dsc.verify_file(pathname):
Logger.error('Checksum for %s does not match.', filename) Logger.error('Checksum for %s does not match.', filename)
return False return False
@ -630,7 +666,8 @@ class FakeSPPH(object):
self.name + '_' + pkgversion, self.name + '_' + pkgversion,
'changelog' + extension) 'changelog' + extension)
try: try:
self._changelog = urlopen(url).read() with closing(urlopen(url)) as f:
self._changelog = f.read()
except HTTPError as error: except HTTPError as error:
print(('%s: %s' % (url, error)), file=sys.stderr) print(('%s: %s' % (url, error)), file=sys.stderr)
return None return None

2
ubuntutools/test/test_archive.py
View File

@ -232,8 +232,6 @@ class DebianLocalSourcePackageTestCase(LocalSourcePackageTestCase):
debsec_mirror = 'http://mirror/debsec' debsec_mirror = 'http://mirror/debsec'
sequence = [self.urlopen_null, sequence = [self.urlopen_null,
self.urlopen_404,
self.urlopen_404,
self.urlopen_404, self.urlopen_404,
self.urlopen_404, self.urlopen_404,
lambda x: BytesIO( lambda x: BytesIO(