From ea30b9f5bc62020be5cc8d0edae43e7e8a41a1bc Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Mar 2020 08:25:22 +0100 Subject: [PATCH] ubuntutools/misc: add download() function Move the download functionality from the archive.py into the common misc.py file. --- ubuntutools/archive.py | 67 +++++++----------------------------------- ubuntutools/misc.py | 62 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 60 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index 1855516..abe6ed4 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -50,7 +50,7 @@ from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, from ubuntutools.lp.udtexceptions import (PackageNotFoundException, SeriesNotFoundException, InvalidDistroValueError) -from ubuntutools.misc import verify_file_checksum +from ubuntutools.misc import (download, verify_file_checksum) from ubuntutools.version import Version import logging @@ -402,43 +402,6 @@ class SourcePackage(object): with open(self.dsc_pathname, 'wb') as f: f.write(self.dsc.raw_text) - def _download_file_helper(self, f, pathname, size): - "Perform the dowload." - BLOCKSIZE = 16 * 1024 - - with open(pathname, 'wb') as out: - if not (Logger.isEnabledFor(logging.INFO) and - sys.stderr.isatty() and - size): - shutil.copyfileobj(f, out, BLOCKSIZE) - return - - XTRALEN = len('[] 99%') - downloaded = 0 - bar_width = 60 - term_width = os.get_terminal_size(sys.stderr.fileno())[0] - if term_width < bar_width + XTRALEN + 1: - bar_width = term_width - XTRALEN - 1 - - try: - while True: - block = f.read(BLOCKSIZE) - if not block: - break - out.write(block) - downloaded += len(block) - pct = float(downloaded) / size - bar = ('=' * int(pct * bar_width))[:-1] + '>' - fmt = '[{bar:<%d}]{pct:>3}%%\r' % bar_width - sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100))) - sys.stderr.flush() - finally: - sys.stderr.write(' ' * (bar_width + XTRALEN) + '\r') - if downloaded < size: - Logger.error('Partial download: %0.3f MiB of %0.3f MiB' % - (downloaded / 1024.0 / 1024, - size / 1024.0 / 1024)) - def _verify_file(self, pathname, dscverify=False, sha1sum=False, sha256sum=False, size=0): if not os.path.exists(pathname): return False @@ -468,26 +431,8 @@ class SourcePackage(object): Logger.info("Copying %s from %s" % (filename, frompath)) shutil.copyfile(frompath, pathname) else: - try: - with closing(urlopen(url)) as f: - Logger.debug("Using URL '%s'", f.geturl()) - if not size: - try: - size = int(f.info().get('Content-Length')) - except (AttributeError, TypeError, ValueError): - pass + download(url, pathname, size) - Logger.info('Downloading %s from %s%s' % - (filename, urlparse(url).hostname, - ' (%0.3f MiB)' % (size / 1024.0 / 1024) - if size else '')) - - self._download_file_helper(f, pathname, size) - except HTTPError as e: - # It's ok if the file isn't found; we try multiple places to download - if e.code == 404: - return False - raise e return self._verify_file(pathname, dscverify, sha1sum, sha256sum, size) def pull(self): @@ -500,6 +445,10 @@ class SourcePackage(object): if self._download_file(url, name, int(entry['size']), dscverify=True): break except HTTPError as e: + # It's ok if the file isn't found; we try multiple places to download + if e.code == 404: + Logger.info("File not found at %s" % url) + continue Logger.info('HTTP Error %i: %s', e.code, str(e)) except URLError as e: Logger.info('URL Error: %s', e.reason) @@ -536,6 +485,10 @@ class SourcePackage(object): total += 1 break except HTTPError as e: + # It's ok if the file isn't found; we try multiple places to download + if e.code == 404: + Logger.info("File not found at %s" % url) + continue Logger.info('HTTP Error %i: %s', e.code, str(e)) except URLError as e: Logger.info('URL Error: %s', e.reason) diff --git a/ubuntutools/misc.py b/ubuntutools/misc.py index a12d785..9ad470c 100644 --- a/ubuntutools/misc.py +++ b/ubuntutools/misc.py @@ -22,14 +22,16 @@ # # ################################################################## -# Modules. +import distro_info import hashlib import locale import os +import shutil import sys -from subprocess import check_output, CalledProcessError -import distro_info +from subprocess import check_output, CalledProcessError +from urllib.parse import urlparse +from urllib.request import urlopen from ubuntutools.lp.udtexceptions import PocketDoesNotExistError @@ -219,3 +221,57 @@ def verify_file_checksum(pathname, alg, checksum, size=0): Logger.error('File %s checksum (%s) mismatch: got %s expected %s', filename, alg, h.hexdigest(), checksum) return match + + +def download(src, dst=None, size=0): + """ download/copy a file/url to local file """ + filename = os.path.basename(urlparse(src).path) + + if not dst: + dst = filename + + with urlopen(src) as fsrc, open(dst, 'wb') as fdst: + url = fsrc.geturl() + Logger.debug(f"Using URL: {url}") + + if not size: + try: + size = int(fsrc.info().get('Content-Length')) + except (AttributeError, TypeError, ValueError): + pass + + hostname = urlparse(url).hostname + sizemb = ' (%0.3f MiB)' % (size / 1024.0 / 1024) if size else '' + Logger.info(f'Downloading {filename} from {hostname}{sizemb}') + + if not all((Logger.isEnabledFor(logging.INFO), + sys.stderr.isatty(), size)): + shutil.copyfileobj(fsrc, fdst) + return + + blocksize = 4096 + XTRALEN = len('[] 99%') + downloaded = 0 + bar_width = 60 + term_width = os.get_terminal_size(sys.stderr.fileno())[0] + if term_width < bar_width + XTRALEN + 1: + bar_width = term_width - XTRALEN - 1 + + try: + while True: + block = fsrc.read(blocksize) + if not block: + break + fdst.write(block) + downloaded += len(block) + pct = float(downloaded) / size + bar = ('=' * int(pct * bar_width))[:-1] + '>' + fmt = '\r[{bar:<%d}]{pct:>3}%%\r' % bar_width + sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100))) + sys.stderr.flush() + finally: + sys.stderr.write('\r' + ' ' * (term_width - 1) + '\r') + if downloaded < size: + Logger.error('Partial download: %0.3f MiB of %0.3f MiB' % + (downloaded / 1024.0 / 1024, + size / 1024.0 / 1024))