From 59e1c73429ca7feb9b089fb34b9026433af702b3 Mon Sep 17 00:00:00 2001 From: CloudBuilder Date: Tue, 23 Jul 2019 10:00:09 +0000 Subject: [PATCH] Imported 2.603 No reason for CPC update specified. --- debian/changelog | 10 + live-build/auto/build | 5 + .../ubuntu-cpc/hooks.d/base/kvm-image.binary | 0 .../ubuntu-cpc/hooks.d/base/series/disk-image | 6 + live-build/ubuntu-cpc/hooks.d/base/series/kvm | 4 +- .../ubuntu-cpc/hooks.d/base/series/qcow2 | 1 + .../ubuntu-cpc/hooks.d/base/series/root-dir | 2 - .../ubuntu-cpc/hooks.d/base/series/squashfs | 4 +- .../ubuntu-cpc/hooks.d/base/series/tarball | 4 +- .../ubuntu-cpc/hooks.d/base/series/vagrant | 1 + .../ubuntu-cpc/hooks.d/base/series/vmdk | 2 + live-build/ubuntu-cpc/hooks.d/base/series/wsl | 2 + live-build/ubuntu-cpc/hooks.d/make-hooks | 53 ++++- .../hooks.d/remove-implicit-artifacts | 41 ++++ snap-tool | 222 ++++++++++++++++-- 15 files changed, 319 insertions(+), 38 deletions(-) mode change 100644 => 100755 live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary create mode 100755 live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts diff --git a/debian/changelog b/debian/changelog index 57540227..72f7cec9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +livecd-rootfs (2.603) eoan; urgency=medium + + [ Tobias Koch ] + * Add retry logic to snap-tool to make downloads more resilient + + [ Robert C Jennings ] + * ubuntu-cpc: Only produce explicitly specified artifacts (LP: #1837254) + + -- Tobias Koch Mon, 15 Jul 2019 17:59:06 +0200 + livecd-rootfs (2.602) eoan; urgency=medium [ David Krauser ] diff --git a/live-build/auto/build b/live-build/auto/build index 74630174..82abbdaf 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -872,3 +872,8 @@ if [ -f "config/magic-proxy.pid" ]; then iptables -t nat -D OUTPUT -p tcp --dport 80 -m owner ! --uid-owner daemon \ -j REDIRECT --to 8080 fi + +case $PROJECT in + ubuntu-cpc) + config/hooks.d/remove-implicit-artifacts +esac diff --git a/live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary b/live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary old mode 100644 new mode 100755 diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/disk-image b/live-build/ubuntu-cpc/hooks.d/base/series/disk-image index 355c010c..6cd49b54 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/disk-image +++ b/live-build/ubuntu-cpc/hooks.d/base/series/disk-image @@ -1,3 +1,9 @@ base/disk-image.binary base/disk-image-uefi.binary base/disk-image-ppc64el.binary +provides livecd.ubuntu-cpc.ext4 +provides livecd.ubuntu-cpc.initrd-generic +provides livecd.ubuntu-cpc.initrd-generic-lpae +provides livecd.ubuntu-cpc.kernel-generic +provides livecd.ubuntu-cpc.kernel-generic-lpae +provides livecd.ubuntu-cpc.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/kvm b/live-build/ubuntu-cpc/hooks.d/base/series/kvm index 6c5d8b8c..556ca8dd 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/kvm +++ b/live-build/ubuntu-cpc/hooks.d/base/series/kvm @@ -1,2 +1,4 @@ depends disk-image -base/kvm-image.binary \ No newline at end of file +base/kvm-image.binary +provides livecd.ubuntu-cpc.disk-kvm.img +provides livecd.ubuntu-cpc.disk-kvm.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 b/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 index cc3ced35..745adb9b 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 +++ b/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 @@ -1,2 +1,3 @@ depends disk-image base/qcow2-image.binary +provides livecd.ubuntu-cpc.img diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/root-dir b/live-build/ubuntu-cpc/hooks.d/base/series/root-dir index b41635af..b5d3b4e4 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/root-dir +++ b/live-build/ubuntu-cpc/hooks.d/base/series/root-dir @@ -1,3 +1 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image base/create-root-dir.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/squashfs b/live-build/ubuntu-cpc/hooks.d/base/series/squashfs index 6d2cb910..560fd79b 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/squashfs +++ b/live-build/ubuntu-cpc/hooks.d/base/series/squashfs @@ -1,4 +1,4 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image depends root-dir base/root-squashfs.binary +provides livecd.ubuntu-cpc.squashfs +provides livecd.ubuntu-cpc.squashfs.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/tarball b/live-build/ubuntu-cpc/hooks.d/base/series/tarball index 184046c2..c741c483 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/tarball +++ b/live-build/ubuntu-cpc/hooks.d/base/series/tarball @@ -1,4 +1,4 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image depends root-dir base/root-xz.binary +provides livecd.ubuntu-cpc.rootfs.tar.xz +provides livecd.ubuntu-cpc.rootfs.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/vagrant b/live-build/ubuntu-cpc/hooks.d/base/series/vagrant index a4eeb86f..6e5fcf39 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/vagrant +++ b/live-build/ubuntu-cpc/hooks.d/base/series/vagrant @@ -1,2 +1,3 @@ depends disk-image base/vagrant.binary +provides livecd.ubuntu-cpc.vagrant.box diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/vmdk b/live-build/ubuntu-cpc/hooks.d/base/series/vmdk index ba0acbae..c583fe96 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/vmdk +++ b/live-build/ubuntu-cpc/hooks.d/base/series/vmdk @@ -1,3 +1,5 @@ depends disk-image base/vmdk-image.binary base/vmdk-ova-image.binary +provides livecd.ubuntu-cpc.vmdk +provides livecd.ubuntu-cpc.ova diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/wsl b/live-build/ubuntu-cpc/hooks.d/base/series/wsl index 34915a31..b082f4e4 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/wsl +++ b/live-build/ubuntu-cpc/hooks.d/base/series/wsl @@ -1,2 +1,4 @@ depends root-dir base/wsl.binary +provides livecd.ubuntu-cpc.wsl.rootfs.tar.gz +provides livecd.ubuntu-cpc.wsl.rootfs.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/make-hooks b/live-build/ubuntu-cpc/hooks.d/make-hooks index 08ae45be..25c8e0dd 100755 --- a/live-build/ubuntu-cpc/hooks.d/make-hooks +++ b/live-build/ubuntu-cpc/hooks.d/make-hooks @@ -31,10 +31,19 @@ to this: depends disk-image depends extra-settings extra/cloudB.binary + provides livecd.ubuntu-cpc.disk-kvm.img + provides livecd.ubuntu-cpc.disk-kvm.manifest Where "disk-image" and "extra-settings" may list scripts and dependencies which are to be processed before the script "extra/cloudB.binary" is called. +The "provides" directive defines a file that the hook creates; it can be +specified multiple times. The field is used by this script to generate a list +of output files created explicitly by the named image targets. The list is +saved to the "explicit_provides" file in the hooks output directory. In +the case of the "all" target this list would be empty. This list is +consumed by the "remove-implicit-artifacts" which is run at the end of the build. + ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate from scripts ending in ".binary". Even if you arrange them interleaved in your series files, the chroot scripts will be run before the binary scripts. @@ -74,6 +83,7 @@ class MakeHooks: self._quiet = quiet self._hooks_list = [] self._included = set() + self._provides = [] def reset(self): """Reset the internal state allowing instance to be reused for @@ -120,8 +130,9 @@ class MakeHooks: e.g. "vmdk" or "vagrant". """ self.collect_chroot_hooks() - self.collect_binary_hooks(image_sets) + self.collect_binary_hooks(image_sets, explicit_sets=True) self.create_symlinks() + self.create_explicit_provides() def collect_chroot_hooks(self): """Chroot hooks are numbered and not explicitly mentioned in series @@ -139,7 +150,7 @@ class MakeHooks: continue self._hooks_list.append(os.path.join("chroot", entry)) - def collect_binary_hooks(self, image_sets): + def collect_binary_hooks(self, image_sets, explicit_sets=False): """Search the series files for the given image_sets and parse them and their dependencies to generate a list of hook scripts to be run during image build. @@ -150,6 +161,11 @@ class MakeHooks: Populates the internal list of paths to hook scripts in the order in which the scripts are to be run. + + If "explicit_sets" is True, the files specified on lines starting + with "provides" will be added to self._provides to track explicit + output artifacts. This is only True for the initial images_sets + list, dependent image sets should set this to False. """ for image_set in image_sets: series_file = self.find_series_file(image_set) @@ -163,6 +179,7 @@ class MakeHooks: line = line.strip() if not line or line.startswith("#"): continue + m = re.match(r"^\s*depends\s+(\S+.*)$", line) if m: include_set = m.group(1) @@ -171,6 +188,13 @@ class MakeHooks: self._included.add(include_set) self.collect_binary_hooks([include_set,]) continue + + m = re.match(r"^\s*provides\s+(\S+.*)$", line) + if m: + if explicit_sets: + self._provides.append(m.group(1)) + continue + if not line in self._hooks_list: self._hooks_list.append(line) @@ -195,13 +219,32 @@ class MakeHooks: hook_basename = m.group("basename") linkname = ("%03d-" % counter) + hook_basename - linksrc = os.path.join(self._hooks_dir, linkname) - linkdest = os.path.relpath(os.path.join(self._script_dir, hook), + linkdest = os.path.join(self._hooks_dir, linkname) + linksrc = os.path.relpath(os.path.join(self._script_dir, hook), self._hooks_dir) if not self._quiet: print("[HOOK] %s => %s" % (linkname, hook)) - os.symlink(linkdest, linksrc) + os.symlink(linksrc, linkdest) + + def create_explicit_provides(self): + """ + Create a file named "explicit_provides" in self._script_dir + listing all files named on "provides" in the series files of + targets explicitly named by the user. The file is created but + left empty if there are no explict "provides" keywords in the + targets (this is the case for 'all') + """ + with open(os.path.join(self._script_dir, "explicit_provides"), "w", + encoding="utf-8") as fp: + empty = True + for provides in self._provides: + if not self._quiet: + print("[PROVIDES] %s" % provides) + fp.write("%s\n" % provides) + empty = False + if not empty: + fp.write('livecd.magic-proxy.log\n') def cli(self, args): """Command line interface to the hooks generator.""" diff --git a/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts b/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts new file mode 100755 index 00000000..074df632 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/remove-implicit-artifacts @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +#-*- encoding: utf-8 -*- +""" +Remove output files not created by explicitly specified image targets + +This uses the 'explicit_provides' file generated by the 'make-hooks' +script. If the file is empty, all output will be saved. +""" +import glob +import os +import sys + +if __name__ == "__main__": + print('Running {}'.format(__file__)) + scriptname = os.path.basename(__file__) + explicit = set() + with open('./config/hooks.d/explicit_provides', 'r', + encoding='utf-8') as fp: + for filename in fp: + explicit.add(filename.rstrip()) + + if not explicit: + print('{}: explicit_provides is empty. ' + 'All binary output will be included'.format(scriptname)) + sys.exit(0) + + all = set(glob.glob('livecd.ubuntu-cpc.*')) + implicit = all - explicit + + print('{}: all artifacts considered: {}'.format(scriptname, all)) + print('{}: explict artifacts to keep: {}'.format(scriptname, explicit)) + print('{}: implicit artifacts to remove: {}'.format(scriptname, implicit)) + + for file in implicit: + if os.path.islink(file): + print('{}: unlinking {}'.format(scriptname, file)) + os.unlink(file) + elif os.path.isfile(file): + print('{}: removing {} ' + '{} bytes'.format(scriptname, file, os.stat(file).st_size)) + os.remove(file) diff --git a/snap-tool b/snap-tool index 44979c17..cc4aa529 100755 --- a/snap-tool +++ b/snap-tool @@ -28,6 +28,7 @@ import re import shutil import subprocess import sys +import time import urllib.error import urllib.request @@ -50,6 +51,178 @@ class SnapAssertionError(SnapError): pass +class ExpBackoffHTTPClient: + """This class is an abstraction layer on top of urllib with additional + retry logic for more reliable downloads.""" + + class Request: + """This is a convenience wrapper around urllib.request.""" + + def __init__(self, request, do_retry, base_interval, num_tries): + """ + :param request: + An urllib.request.Request instance. + :param do_retry: + Whether to enable the exponential backoff and retry logic. + :param base_interval: + The initial interval to sleep after a failed attempt. + :param num_tries: + How many attempts to make. + """ + self._request = request + self._do_retry = do_retry + self._base_interval = base_interval + self._num_tries = num_tries + self._response = None + + def open(self): + """Open the connection.""" + if not self._response: + self._response = self._retry_urlopen() + + def close(self): + """Close the connection.""" + if self._response: + self._response.close() + self._response = None + + def data(self): + """Return the raw response body.""" + with self: + return self.read() + + def json(self): + """Return the deserialized response body interpreted as JSON.""" + return json.loads(self.data(), encoding="utf-8") + + def text(self): + """Return the response body as a unicode string.""" + encoding = "utf-8" + + with self: + content_type = self._response.getheader("Content-Type", "") + + if content_type == "application/json": + encoding = "utf-8" + else: + m = re.match(r"text/\S+;\s*charset=(?P\S+)", + content_type) + if m: + encoding=m.group("charset") + + return self.read().decode(encoding) + + def read(self, size=None): + """Read size bytes from the response. If size if not set, the + complete response body is read in.""" + return self._response.read(size) + + def __enter__(self): + """Make this class a context manager.""" + self.open() + return self + + def __exit__(self, type, value, traceback): + """Make this class a context manager.""" + self.close() + + def _retry_urlopen(self): + """Try to open the HTTP connection as many times as configured + through the constructor. Every time an error occurs, double the + time to wait until the next attempt.""" + for attempt in range(self._num_tries): + try: + return urllib.request.urlopen(self._request) + except Exception as e: + if isinstance(e, urllib.error.HTTPError) and e.code < 500: + raise + if attempt >= self._num_tries - 1: + raise + sys.stderr.write( + "WARNING: failed to open URL '{}': {}\n" + .format(self._request.full_url, str(e)) + ) + else: + break + + sleep_interval = self._base_interval * 2**attempt + sys.stderr.write( + "Retrying HTTP request in {} seconds...\n" + .format(sleep_interval) + ) + time.sleep(sleep_interval) + + + def __init__(self, do_retry=True, base_interval=2, num_tries=8): + """ + :param do_retry: + Whether to enable the retry logic. + :param base_interval: + The initial interval to sleep after a failed attempt. + :param num_tries: + How many attempts to make. + """ + self._do_retry = do_retry + self._base_interval = base_interval + self._num_tries = num_tries if do_retry else 1 + + def get(self, url, headers=None): + """Create a GET request that can be used to retrieve the resource + at the given URL. + + :param url: + An HTTP URL. + :param headers: + A dictionary of extra headers to send along. + :return: + An ExpBackoffHTTPClient.Request instance. + """ + return self._prepare_request(url, headers=headers) + + def post(self, url, data=None, json=None, headers=None): + """Create a POST request that can be used to submit data to the + endpoint at the given URL.""" + return self._prepare_request( + url, data=data, json_data=json, headers=headers + ) + + def _prepare_request(self, url, data=None, json_data=None, headers=None): + """Prepare a Request instance that can be used to retrieve data from + and/or send data to the endpoint at the given URL. + + :param url: + An HTTP URL. + :param data: + Raw binary data to send along in the request body. + :param json_data: + A Python data structure to be serialized and sent out in JSON + format. + :param headers: + A dictionary of extra headers to send along. + :return: + An ExpBackoffHTTPClient.Request instance. + """ + if data is not None and json_data is not None: + raise ValueError( + "Parameters 'data' and 'json_data' are mutually exclusive." + ) + + if json_data: + data = json.dumps(json_data, ensure_ascii=False) + if headers is None: + headers = {} + headers["Content-Type"] = "application/json" + if isinstance(data, str): + data = data.encode("utf-8") + + return ExpBackoffHTTPClient.Request( + urllib.request.Request(url, data=data, headers=headers or {}), + self._do_retry, + self._base_interval, + self._num_tries + ) + + class Snap: """This class provides methods to retrieve information about a snap and download it together with its assertions.""" @@ -115,13 +288,17 @@ class Snap: "Snap-CDN": "none", }) - request = urllib.request.Request(snap_download_url, headers=headers) - if not skip_snap_download: - with urllib.request.urlopen(request) as response, \ - open(snap_filename, "wb+") as fp: + http_client = ExpBackoffHTTPClient() + response = http_client.get(snap_download_url, headers=headers) + with response, open(snap_filename, "wb+") as fp: shutil.copyfileobj(response, fp) + if os.path.getsize(snap_filename) != snap_byte_size: + raise SnapError( + "The downloaded snap does not have the expected size." + ) + if not download_assertions: return @@ -193,16 +370,20 @@ class Snap: elif self._cohort_key: data["actions"][0]["cohort-key"] = self._cohort_key - request_json = json.dumps(data, ensure_ascii=False).encode("utf-8") - try: - response_dict = self._do_snapcraft_request(path, data=request_json) + response_dict = self._do_snapcraft_request(path, json_data=data) except SnapCraftError as e: raise SnapError("failed to get details for '{}': {}" .format(self._name, str(e))) snap_data = response_dict["results"][0] + if snap_data.get("result") == "error": + raise SnapError( + "failed to get details for '{}': {}" + .format(self._name, snap_data.get("error", {}).get("message")) + ) + # Have "base" initialized to something meaningful. if self.is_core_snap(): snap_data["snap"]["base"] = "" @@ -296,40 +477,29 @@ class Snap: "Accept": "application/x.ubuntu.assertion", } - request = urllib.request.Request(url, headers=headers) - + http_client = ExpBackoffHTTPClient() try: - with urllib.request.urlopen(request) as response: - body = response.read() + with http_client.get(url, headers=headers) as response: + return response.text() except urllib.error.HTTPError as e: raise SnapAssertionError(str(e)) - return body.decode("utf-8") - - def _do_snapcraft_request(self, path, data=None): + def _do_snapcraft_request(self, path, json_data=None): url = self._snapcraft_url + "/" + path headers = { "Snap-Device-Series": str(self._series), "Snap-Device-Architecture": self._arch, - "Content-Type": "application/json", } - request = urllib.request.Request(url, data=data, headers=headers) - + http_client = ExpBackoffHTTPClient() try: - with urllib.request.urlopen(request) as response: - body = response.read() + response = http_client.post(url, json=json_data, headers=headers) + with response: + return response.json() except urllib.error.HTTPError as e: raise SnapCraftError(str(e)) - try: - response_data = json.loads(body, encoding="utf-8") - except json.JSONDecodeError as e: - raise SnapCraftError("failed to decode response body: " + str(e)) - - return response_data - class SnapCli: