Imported 2.603

No reason for CPC update specified.

debian/changelog

@@ -1,3 +1,13 @@
+livecd-rootfs (2.603) eoan; urgency=medium
+
+  [ Tobias Koch ]
+  * Add retry logic to snap-tool to make downloads more resilient
+
+  [ Robert C Jennings ]
+  * ubuntu-cpc: Only produce explicitly specified artifacts (LP: #1837254)
+
+ -- Tobias Koch <tobias.koch@canonical.com>  Mon, 15 Jul 2019 17:59:06 +0200
+
 livecd-rootfs (2.602) eoan; urgency=medium
 
   [ David Krauser ]

live-build/auto/build

@@ -872,3 +872,8 @@ if [ -f "config/magic-proxy.pid" ]; then
     iptables -t nat -D OUTPUT -p tcp --dport 80 -m owner ! --uid-owner daemon \
         -j REDIRECT --to 8080
 fi
+
+case $PROJECT in
+    ubuntu-cpc)
+        config/hooks.d/remove-implicit-artifacts
+esac

live-build/ubuntu-cpc/hooks.d/base/series/disk-image

@@ -1,3 +1,9 @@
 base/disk-image.binary
 base/disk-image-uefi.binary
 base/disk-image-ppc64el.binary
+provides livecd.ubuntu-cpc.ext4
+provides livecd.ubuntu-cpc.initrd-generic
+provides livecd.ubuntu-cpc.initrd-generic-lpae
+provides livecd.ubuntu-cpc.kernel-generic
+provides livecd.ubuntu-cpc.kernel-generic-lpae
+provides livecd.ubuntu-cpc.manifest

live-build/ubuntu-cpc/hooks.d/base/series/kvm

@@ -1,2 +1,4 @@
 depends disk-image
-base/kvm-image.binary
+base/kvm-image.binary
+provides livecd.ubuntu-cpc.disk-kvm.img
+provides livecd.ubuntu-cpc.disk-kvm.manifest

live-build/ubuntu-cpc/hooks.d/base/series/qcow2

@@ -1,2 +1,3 @@
 depends disk-image
 base/qcow2-image.binary
+provides livecd.ubuntu-cpc.img

live-build/ubuntu-cpc/hooks.d/base/series/root-dir

@@ -1,3 +1 @@
-# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent
-depends disk-image
 base/create-root-dir.binary

live-build/ubuntu-cpc/hooks.d/base/series/squashfs

@@ -1,4 +1,4 @@
-# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent
-depends disk-image
 depends root-dir
 base/root-squashfs.binary
+provides livecd.ubuntu-cpc.squashfs
+provides livecd.ubuntu-cpc.squashfs.manifest

live-build/ubuntu-cpc/hooks.d/base/series/tarball

@@ -1,4 +1,4 @@
-# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent
-depends disk-image
 depends root-dir
 base/root-xz.binary
+provides livecd.ubuntu-cpc.rootfs.tar.xz
+provides livecd.ubuntu-cpc.rootfs.manifest

live-build/ubuntu-cpc/hooks.d/base/series/vagrant

@@ -1,2 +1,3 @@
 depends disk-image
 base/vagrant.binary
+provides livecd.ubuntu-cpc.vagrant.box

live-build/ubuntu-cpc/hooks.d/base/series/vmdk

@@ -1,3 +1,5 @@
 depends disk-image
 base/vmdk-image.binary
 base/vmdk-ova-image.binary
+provides livecd.ubuntu-cpc.vmdk
+provides livecd.ubuntu-cpc.ova

live-build/ubuntu-cpc/hooks.d/base/series/wsl

@@ -1,2 +1,4 @@
 depends root-dir
 base/wsl.binary
+provides livecd.ubuntu-cpc.wsl.rootfs.tar.gz
+provides livecd.ubuntu-cpc.wsl.rootfs.manifest

live-build/ubuntu-cpc/hooks.d/make-hooks

@@ -31,10 +31,19 @@ to this:
     depends disk-image
     depends extra-settings
     extra/cloudB.binary
+    provides livecd.ubuntu-cpc.disk-kvm.img
+    provides livecd.ubuntu-cpc.disk-kvm.manifest
 
 Where "disk-image" and "extra-settings" may list scripts and dependencies which
 are to be processed before the script "extra/cloudB.binary" is called.
 
+The "provides" directive defines a file that the hook creates; it can be
+specified multiple times.  The field is used by this script to generate a list
+of output files created explicitly by the named image targets.  The list is
+saved to the "explicit_provides" file in the hooks output directory. In
+the case of the "all" target this list would be empty.  This list is
+consumed by the "remove-implicit-artifacts" which is run at the end of the build.
+
 ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate
 from scripts ending in ".binary". Even if you arrange them interleaved in your
 series files, the chroot scripts will be run before the binary scripts.
@@ -74,6 +83,7 @@ class MakeHooks:
         self._quiet      = quiet
         self._hooks_list = []
         self._included   = set()
+        self._provides   = []
 
     def reset(self):
         """Reset the internal state allowing instance to be reused for
@@ -120,8 +130,9 @@ class MakeHooks:
         e.g. "vmdk" or "vagrant".
         """
         self.collect_chroot_hooks()
-        self.collect_binary_hooks(image_sets)
+        self.collect_binary_hooks(image_sets, explicit_sets=True)
         self.create_symlinks()
+        self.create_explicit_provides()
 
     def collect_chroot_hooks(self):
         """Chroot hooks are numbered and not explicitly mentioned in series
@@ -139,7 +150,7 @@ class MakeHooks:
                 continue
             self._hooks_list.append(os.path.join("chroot", entry))
 
-    def collect_binary_hooks(self, image_sets):
+    def collect_binary_hooks(self, image_sets, explicit_sets=False):
         """Search the series files for the given image_sets and parse them
         and their dependencies to generate a list of hook scripts to be run
         during image build.
@@ -150,6 +161,11 @@ class MakeHooks:
 
         Populates the internal list of paths to hook scripts in the order in
         which the scripts are to be run.
+
+        If "explicit_sets" is True, the files specified on lines starting
+        with "provides" will be added to self._provides to track explicit
+        output artifacts.  This is only True for the initial images_sets
+        list, dependent image sets should set this to False.
         """
         for image_set in image_sets:
             series_file = self.find_series_file(image_set)
@@ -163,6 +179,7 @@ class MakeHooks:
                     line = line.strip()
                     if not line or line.startswith("#"):
                         continue
+
                     m = re.match(r"^\s*depends\s+(\S+.*)$", line)
                     if m:
                         include_set = m.group(1)
@@ -171,6 +188,13 @@ class MakeHooks:
                         self._included.add(include_set)
                         self.collect_binary_hooks([include_set,])
                         continue
+
+                    m = re.match(r"^\s*provides\s+(\S+.*)$", line)
+                    if m:
+                        if explicit_sets:
+                            self._provides.append(m.group(1))
+                        continue
+
                     if not line in self._hooks_list:
                         self._hooks_list.append(line)
 
@@ -195,13 +219,32 @@ class MakeHooks:
                 hook_basename = m.group("basename")
 
             linkname = ("%03d-" % counter) + hook_basename
-            linksrc  = os.path.join(self._hooks_dir, linkname)
-            linkdest = os.path.relpath(os.path.join(self._script_dir, hook),
+            linkdest = os.path.join(self._hooks_dir, linkname)
+            linksrc  = os.path.relpath(os.path.join(self._script_dir, hook),
                     self._hooks_dir)
 
             if not self._quiet:
                 print("[HOOK] %s => %s" % (linkname, hook))
-            os.symlink(linkdest, linksrc)
+            os.symlink(linksrc, linkdest)
+
+    def create_explicit_provides(self):
+        """
+        Create a file named "explicit_provides" in self._script_dir
+        listing all files named on "provides" in the series files of
+        targets explicitly named by the user.  The file is created but
+        left empty if there are no explict "provides" keywords in the
+        targets (this is the case for 'all')
+        """
+        with open(os.path.join(self._script_dir, "explicit_provides"), "w",
+                  encoding="utf-8") as fp:
+            empty = True
+            for provides in self._provides:
+                if not self._quiet:
+                    print("[PROVIDES] %s" % provides)
+                fp.write("%s\n" % provides)
+                empty = False
+            if not empty:
+                fp.write('livecd.magic-proxy.log\n')
 
     def cli(self, args):
         """Command line interface to the hooks generator."""

live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#-*- encoding: utf-8 -*-
+"""
+Remove output files not created by explicitly specified image targets
+
+This uses the 'explicit_provides' file generated by the 'make-hooks'
+script.  If the file is empty, all output will be saved.
+"""
+import glob
+import os
+import sys
+
+if __name__ == "__main__":
+    print('Running {}'.format(__file__))
+    scriptname = os.path.basename(__file__)
+    explicit = set()
+    with open('./config/hooks.d/explicit_provides', 'r',
+              encoding='utf-8') as fp:
+        for filename in fp:
+            explicit.add(filename.rstrip())
+
+    if not explicit:
+        print('{}: explicit_provides is empty. '
+              'All binary output will be included'.format(scriptname))
+        sys.exit(0)
+
+    all = set(glob.glob('livecd.ubuntu-cpc.*'))
+    implicit = all - explicit
+
+    print('{}: all artifacts considered: {}'.format(scriptname, all))
+    print('{}: explict artifacts to keep: {}'.format(scriptname, explicit))
+    print('{}: implicit artifacts to remove: {}'.format(scriptname, implicit))
+
+    for file in implicit:
+        if os.path.islink(file):
+            print('{}: unlinking {}'.format(scriptname, file))
+            os.unlink(file)
+        elif os.path.isfile(file):
+            print('{}: removing {} '
+                  '{} bytes'.format(scriptname, file, os.stat(file).st_size))
+            os.remove(file)

snap-tool

@@ -28,6 +28,7 @@ import re
 import shutil
 import subprocess
 import sys
+import time
 import urllib.error
 import urllib.request
 
@@ -50,6 +51,178 @@ class SnapAssertionError(SnapError):
     pass
 
 
+class ExpBackoffHTTPClient:
+    """This class is an abstraction layer on top of urllib with additional
+    retry logic for more reliable downloads."""
+
+    class Request:
+        """This is a convenience wrapper around urllib.request."""
+
+        def __init__(self, request, do_retry, base_interval, num_tries):
+            """
+            :param request:
+                An urllib.request.Request instance.
+            :param do_retry:
+                Whether to enable the exponential backoff and retry logic.
+            :param base_interval:
+                The initial interval to sleep after a failed attempt.
+            :param num_tries:
+                How many attempts to make.
+            """
+            self._request       = request
+            self._do_retry      = do_retry
+            self._base_interval = base_interval
+            self._num_tries     = num_tries
+            self._response      = None
+
+        def open(self):
+            """Open the connection."""
+            if not self._response:
+                self._response = self._retry_urlopen()
+
+        def close(self):
+            """Close the connection."""
+            if self._response:
+                self._response.close()
+                self._response = None
+
+        def data(self):
+            """Return the raw response body."""
+            with self:
+                return self.read()
+
+        def json(self):
+            """Return the deserialized response body interpreted as JSON."""
+            return json.loads(self.data(), encoding="utf-8")
+
+        def text(self):
+            """Return the response body as a unicode string."""
+            encoding = "utf-8"
+
+            with self:
+                content_type = self._response.getheader("Content-Type", "")
+
+                if content_type == "application/json":
+                    encoding = "utf-8"
+                else:
+                    m = re.match(r"text/\S+;\s*charset=(?P<charset>\S+)",
+                            content_type)
+                    if m:
+                        encoding=m.group("charset")
+
+                return self.read().decode(encoding)
+
+        def read(self, size=None):
+            """Read size bytes from the response. If size if not set, the
+            complete response body is read in."""
+            return self._response.read(size)
+
+        def __enter__(self):
+            """Make this class a context manager."""
+            self.open()
+            return self
+
+        def __exit__(self, type, value, traceback):
+            """Make this class a context manager."""
+            self.close()
+
+        def _retry_urlopen(self):
+            """Try to open the HTTP connection as many times as configured
+            through the constructor. Every time an error occurs, double the
+            time to wait until the next attempt."""
+            for attempt in range(self._num_tries):
+                try:
+                    return urllib.request.urlopen(self._request)
+                except Exception as e:
+                    if isinstance(e, urllib.error.HTTPError) and e.code < 500:
+                        raise
+                    if attempt >= self._num_tries - 1:
+                        raise
+                    sys.stderr.write(
+                        "WARNING: failed to open URL '{}': {}\n"
+                        .format(self._request.full_url, str(e))
+                    )
+                else:
+                    break
+
+                sleep_interval = self._base_interval * 2**attempt
+                sys.stderr.write(
+                    "Retrying HTTP request in {} seconds...\n"
+                    .format(sleep_interval)
+                )
+                time.sleep(sleep_interval)
+
+
+    def __init__(self, do_retry=True, base_interval=2, num_tries=8):
+        """
+        :param do_retry:
+            Whether to enable the retry logic.
+        :param base_interval:
+            The initial interval to sleep after a failed attempt.
+        :param num_tries:
+            How many attempts to make.
+        """
+        self._do_retry      = do_retry
+        self._base_interval = base_interval
+        self._num_tries     = num_tries if do_retry else 1
+
+    def get(self, url, headers=None):
+        """Create a GET request that can be used to retrieve the resource
+        at the given URL.
+
+        :param url:
+            An HTTP URL.
+        :param headers:
+            A dictionary of extra headers to send along.
+        :return:
+            An ExpBackoffHTTPClient.Request instance.
+        """
+        return self._prepare_request(url, headers=headers)
+
+    def post(self, url, data=None, json=None, headers=None):
+        """Create a POST request that can be used to submit data to the
+        endpoint at the given URL."""
+        return self._prepare_request(
+            url, data=data, json_data=json, headers=headers
+        )
+
+    def _prepare_request(self, url, data=None, json_data=None, headers=None):
+        """Prepare a Request instance that can be used to retrieve data from
+        and/or send data to the endpoint at the given URL.
+
+        :param url:
+            An HTTP URL.
+        :param data:
+            Raw binary data to send along in the request body.
+        :param json_data:
+            A Python data structure to be serialized and sent out in JSON
+            format.
+        :param headers:
+            A dictionary of extra headers to send along.
+        :return:
+            An ExpBackoffHTTPClient.Request instance.
+        """
+        if data is not None and json_data is not None:
+            raise ValueError(
+                "Parameters 'data' and 'json_data' are mutually exclusive."
+            )
+
+        if json_data:
+            data = json.dumps(json_data, ensure_ascii=False)
+            if headers is None:
+                headers = {}
+            headers["Content-Type"] = "application/json"
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+
+        return ExpBackoffHTTPClient.Request(
+            urllib.request.Request(url, data=data, headers=headers or {}),
+            self._do_retry,
+            self._base_interval,
+            self._num_tries
+        )
+
+
 class Snap:
     """This class provides methods to retrieve information about a snap and
     download it together with its assertions."""
@@ -115,13 +288,17 @@ class Snap:
                 "Snap-CDN": "none",
             })
 
-        request = urllib.request.Request(snap_download_url, headers=headers)
-
         if not skip_snap_download:
-            with urllib.request.urlopen(request) as response, \
-                    open(snap_filename, "wb+") as fp:
+            http_client = ExpBackoffHTTPClient()
+            response = http_client.get(snap_download_url, headers=headers)
+            with response, open(snap_filename, "wb+") as fp:
                 shutil.copyfileobj(response, fp)
 
+        if os.path.getsize(snap_filename) != snap_byte_size:
+            raise SnapError(
+                "The downloaded snap does not have the expected size."
+            )
+
         if not download_assertions:
             return
 
@@ -193,16 +370,20 @@ class Snap:
         elif self._cohort_key:
             data["actions"][0]["cohort-key"] = self._cohort_key
 
-        request_json = json.dumps(data, ensure_ascii=False).encode("utf-8")
-
         try:
-            response_dict = self._do_snapcraft_request(path, data=request_json)
+            response_dict = self._do_snapcraft_request(path, json_data=data)
         except SnapCraftError as e:
             raise SnapError("failed to get details for '{}': {}"
                     .format(self._name, str(e)))
 
         snap_data = response_dict["results"][0]
 
+        if snap_data.get("result") == "error":
+            raise SnapError(
+                "failed to get details for '{}': {}"
+                .format(self._name, snap_data.get("error", {}).get("message"))
+            )
+
         # Have "base" initialized to something meaningful.
         if self.is_core_snap():
             snap_data["snap"]["base"] = ""
@@ -296,40 +477,29 @@ class Snap:
             "Accept": "application/x.ubuntu.assertion",
         }
 
-        request = urllib.request.Request(url, headers=headers)
-
+        http_client = ExpBackoffHTTPClient()
         try:
-            with urllib.request.urlopen(request) as response:
-                body = response.read()
+            with http_client.get(url, headers=headers) as response:
+                return response.text()
         except urllib.error.HTTPError as e:
             raise SnapAssertionError(str(e))
 
-        return body.decode("utf-8")
-
-    def _do_snapcraft_request(self, path, data=None):
+    def _do_snapcraft_request(self, path, json_data=None):
         url = self._snapcraft_url + "/" + path
 
         headers = {
             "Snap-Device-Series": str(self._series),
             "Snap-Device-Architecture": self._arch,
-            "Content-Type": "application/json",
         }
 
-        request = urllib.request.Request(url, data=data, headers=headers)
-
+        http_client = ExpBackoffHTTPClient()
         try:
-            with urllib.request.urlopen(request) as response:
-                body = response.read()
+            response = http_client.post(url, json=json_data, headers=headers)
+            with response:
+                return response.json()
         except urllib.error.HTTPError as e:
             raise SnapCraftError(str(e))
 
-        try:
-            response_data = json.loads(body, encoding="utf-8")
-        except json.JSONDecodeError as e:
-            raise SnapCraftError("failed to decode response body: " + str(e))
-
-        return response_data
-
 
 class SnapCli: