misc: Change download() method to use python requests and optional authentication

Since private PPAs require authentication, use python requests library instead
of urlopen(), since requests handles authentication easily

debian/control

@@ -139,6 +139,7 @@ Depends:
  python3-httplib2,
  python3-launchpadlib,
  python3-lazr.restfulclient,
+ python3-requests,
  sensible-utils,
  ${misc:Depends},
  ${python3:Depends},

ubuntutools/archive.py

@@ -27,7 +27,6 @@ Approach:
 3. Verify checksums.
 """
 
-from urllib.error import (URLError, HTTPError)
 from urllib.request import urlopen
 import codecs
 import functools
@@ -46,8 +45,12 @@ import debian.deb822
 from contextlib import closing
 
 from ubuntutools.config import UDTConfig
-from ubuntutools.lp.lpapicache import (Launchpad, Distribution, PersonTeam, Project,
+from ubuntutools.lp.lpapicache import (Launchpad,
-                                       SourcePackagePublishingHistory)
+                                       Distribution,
+                                       PersonTeam,
+                                       Project,
+                                       SourcePackagePublishingHistory,
+                                       HTTPError)
 from ubuntutools.lp.udtexceptions import (PackageNotFoundException,
                                           SeriesNotFoundException,
                                           PocketDoesNotExistError,
@@ -56,7 +59,8 @@ from ubuntutools.misc import (download,
                               download_bytes,
                               verify_file_checksum,
                               verify_file_checksums,
-                              DownloadError)
+                              DownloadError,
+                              NotFoundError)
 from ubuntutools.version import Version
 
 import logging
@@ -397,15 +401,12 @@ class SourcePackage(ABC):
                 if self._download_file(url, filename, size, dscverify=dscverify,
                                        sha1sum=sha1sum, sha256sum=sha256sum):
                     return
-            except HTTPError as e:
+            except NotFoundError:
-                if e.code == 404:
+                # It's ok if the file isn't found, just try the next url
-                    # It's ok if the file isn't found, just try the next url
+                Logger.debug(f'File not found at {url}')
-                    Logger.debug("File not found at %s" % url)
+            except DownloadError as e:
-                else:
+                Logger.error(f'Download Error: {e}')
-                    Logger.error('HTTP Error %i: %s', e.code, str(e))
+        raise DownloadError(f'Failed to download {filename}')
-            except URLError as e:
-                Logger.error('URL Error: %s', e.reason)
-        raise DownloadError('Failed to download %s' % filename)
 
     def pull_dsc(self):
         '''DEPRECATED

ubuntutools/misc.py

@@ -26,6 +26,7 @@ import distro_info
 import hashlib
 import locale
 import os
+import requests
 import shutil
 import sys
 import tempfile
@@ -34,7 +35,6 @@ from contextlib import suppress
 from pathlib import Path
 from subprocess import check_output, CalledProcessError
 from urllib.parse import urlparse
-from urllib.request import urlopen
 
 from ubuntutools.lp.udtexceptions import PocketDoesNotExistError
 
@@ -58,6 +58,11 @@ class DownloadError(Exception):
     pass
 
 
+class NotFoundError(DownloadError):
+    "Source package not found"
+    pass
+
+
 def system_distribution_chain():
     """ system_distribution_chain() -> [string]
 
@@ -286,74 +291,97 @@ def download(src, dst, size=0):
 
     src: str or Path
         Source to copy from (file path or url)
-    dst: str
+    dst: str or Path
         Destination dir or filename
     size: int
         Size of source, if known
 
-    This calls urllib.request.urlopen() so it may raise the same
+    If the URL contains authentication data in the URL 'netloc',
-    exceptions as that method (URLError or HTTPError)
+    it will be stripped from the URL and passed to the requests library.
+
+    This may throw a DownloadError.
     """
     src = str(src)
-    if not urlparse(src).scheme:
+    parsedsrc = urlparse(src)
-        src = 'file://%s' % os.path.abspath(os.path.expanduser(src))
-    dst = os.path.abspath(os.path.expanduser(dst))
 
-    filename = os.path.basename(urlparse(src).path)
+    dst = Path(dst).expanduser().resolve()
+    if dst.is_dir():
+        dst = dst / Path(parsedsrc.path).name
 
-    if os.path.isdir(dst):
+    # Copy if src is a local file
-        dst = os.path.join(dst, filename)
+    if parsedsrc.scheme in ['', 'file']:
-
+        src = Path(parsedsrc.path).expanduser().resolve()
-    if urlparse(src).scheme == 'file':
+        if src != parsedsrc.path:
-        srcfile = urlparse(src).path
+            Logger.info(f'Parsed {parsedsrc.path} as {src}')
-        if os.path.exists(srcfile) and os.path.exists(dst):
+        if not src.exists():
-            if os.path.samefile(srcfile, dst):
+            raise NotFoundError(f'Source file {src} not found')
-                Logger.info(f"Using existing file {dst}")
+        if dst.exists():
+            if src.samefile(dst):
+                Logger.info(f'Using existing file {dst}')
                 return
+            Logger.info(f'Replacing existing file {dst}')
+        Logger.info(f'Copying file {src} to {dst}')
+        shutil.copyfile(src, dst)
+        return
 
-    with urlopen(src) as fsrc, open(dst, 'wb') as fdst:
+    (src, username, password) = extract_authentication(src)
-        url = fsrc.geturl()
+    auth = (username, password) if username or password else None
-        Logger.debug(f"Using URL: {url}")
 
-        if not size:
+    try:
-            with suppress(AttributeError, TypeError, ValueError):
+        with requests.get(src, stream=True, auth=auth) as fsrc, dst.open('wb') as fdst:
-                size = int(fsrc.info().get('Content-Length'))
+            fsrc.raise_for_status()
+            _download(fsrc, fdst, size)
+    except requests.RequestException as e:
+        if e.response and e.response.status_code == 404:
+            raise NotFoundError(f'URL {src} not found') from e
+        raise DownloadError(f'Could not download {src} to {dst}') from e
 
-        hostname = urlparse(url).hostname
-        sizemb = ' (%0.3f MiB)' % (size / 1024.0 / 1024) if size else ''
-        Logger.info(f'Downloading {filename} from {hostname}{sizemb}')
 
-        if not all((Logger.isEnabledFor(logging.INFO),
+def _download(fsrc, fdst, size):
-                    sys.stderr.isatty(), size)):
+    """ helper method to download src to dst using requests library. """
-            shutil.copyfileobj(fsrc, fdst)
+    url = fsrc.url
-            return
+    Logger.debug(f'Using URL: {url}')
 
-        blocksize = 4096
+    if not size:
-        XTRALEN = len('[] 99%')
+        with suppress(AttributeError, TypeError, ValueError):
-        downloaded = 0
+            size = int(fsrc.headers.get('Content-Length'))
-        bar_width = 60
-        term_width = os.get_terminal_size(sys.stderr.fileno())[0]
-        if term_width < bar_width + XTRALEN + 1:
-            bar_width = term_width - XTRALEN - 1
 
-        try:
+    parsed = urlparse(url)
-            while True:
+    filename = Path(parsed.path).name
-                block = fsrc.read(blocksize)
+    hostname = parsed.hostname
-                if not block:
+    sizemb = ' (%0.3f MiB)' % (size / 1024.0 / 1024) if size else ''
-                    break
+    Logger.info(f'Downloading {filename} from {hostname}{sizemb}')
-                fdst.write(block)
+
-                downloaded += len(block)
+    if not all((Logger.isEnabledFor(logging.INFO), sys.stderr.isatty(), size)):
-                pct = float(downloaded) / size
+        shutil.copyfileobj(fsrc.raw, fdst)
-                bar = ('=' * int(pct * bar_width))[:-1] + '>'
+        return
-                fmt = '\r[{bar:<%d}]{pct:>3}%%\r' % bar_width
+
-                sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100)))
+    blocksize = 4096
-                sys.stderr.flush()
+    XTRALEN = len('[] 99%')
-        finally:
+    downloaded = 0
-            sys.stderr.write('\r' + ' ' * (term_width - 1) + '\r')
+    bar_width = 60
-            if downloaded < size:
+    term_width = os.get_terminal_size(sys.stderr.fileno())[0]
-                Logger.error('Partial download: %0.3f MiB of %0.3f MiB' %
+    if term_width < bar_width + XTRALEN + 1:
-                             (downloaded / 1024.0 / 1024,
+        bar_width = term_width - XTRALEN - 1
-                              size / 1024.0 / 1024))
+
+    try:
+        while True:
+            block = fsrc.raw.read(blocksize)
+            if not block:
+                break
+            fdst.write(block)
+            downloaded += len(block)
+            pct = float(downloaded) / size
+            bar = ('=' * int(pct * bar_width))[:-1] + '>'
+            fmt = '\r[{bar:<%d}]{pct:>3}%%\r' % bar_width
+            sys.stderr.write(fmt.format(bar=bar, pct=int(pct * 100)))
+            sys.stderr.flush()
+    finally:
+        sys.stderr.write('\r' + ' ' * (term_width - 1) + '\r')
+        if downloaded < size:
+            Logger.error('Partial download: %0.3f MiB of %0.3f MiB' %
+                         (downloaded / 1024.0 / 1024,
+                          size / 1024.0 / 1024))
 
 
 def _download_text(src, binary):