From 0d56c46d244b784ca0f54139f73efb2bb74cc3a8 Mon Sep 17 00:00:00 2001 From: Robert C Jennings Date: Fri, 19 Jul 2019 05:12:20 -0500 Subject: [PATCH] ubuntu-cpc: Only produce explicitly named artifacts (LP: #1837254) In parallel builds where a list of image targets are provided the build may produce binaries that are not part of the named set of targets but are created by series dependencies. These implicitly created binaries may be generated by multiple builds but are unused as our convention for the ubuntu-cpc project is to only consume binaries from the explicitly named image targets; this avoid overwriting the same object by multiple parallel builds. This patch adds support for a 'provides' keyword for series files. It can be specified multiple times per series file. The field is used by the make-hooks script to generate a list of output files created explicitly by the named image targets. The list is saved to the "explicit_provides" file in the hooks output directory. In the case of the "all" target this list would be empty. This list is consumed by the "final.binary" hook file. This patch adds support for optional final.binary hooks in hooks.d/base and/or hooks.d/extra. These final.binary hooks are always included as the last hook(s) if either exist with the hook in "extra" running last. The base/final.binary hook includes logic to parse the "explicit_provides" file generated by the make-hooks script and remove any binary output not explicitly specified. Some series files named unnecessary dependencies, specifically disk-image, to keep output of implicit artifacts consistent between parallel builds. These unnecessary dependencies are removed in this patch. --- debian/changelog | 6 ++ .../ubuntu-cpc/hooks.d/base/final.binary | 47 ++++++++++++++ .../ubuntu-cpc/hooks.d/base/series/disk-image | 7 +++ .../ubuntu-cpc/hooks.d/base/series/qcow2 | 1 + .../ubuntu-cpc/hooks.d/base/series/root-dir | 2 - .../ubuntu-cpc/hooks.d/base/series/squashfs | 4 +- .../ubuntu-cpc/hooks.d/base/series/tarball | 4 +- .../ubuntu-cpc/hooks.d/base/series/vagrant | 1 + .../ubuntu-cpc/hooks.d/base/series/vmdk | 2 + live-build/ubuntu-cpc/hooks.d/base/series/wsl | 2 + live-build/ubuntu-cpc/hooks.d/make-hooks | 63 +++++++++++++++++-- 11 files changed, 128 insertions(+), 11 deletions(-) create mode 100755 live-build/ubuntu-cpc/hooks.d/base/final.binary diff --git a/debian/changelog b/debian/changelog index 8210033d..7dcccb56 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +livecd-rootfs (2.578.8) UNRELEASED; urgency=medium + + * ubuntu-cpc: Only produce explicitly specified artifacts (LP: #1837254) + + -- Robert C Jennings Mon, 26 Aug 2019 16:25:24 -0500 + livecd-rootfs (2.578.7) disco; urgency=medium [ Tobias Koch ] diff --git a/live-build/ubuntu-cpc/hooks.d/base/final.binary b/live-build/ubuntu-cpc/hooks.d/base/final.binary new file mode 100755 index 00000000..d1d3803f --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/final.binary @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +#-*- encoding: utf-8 -*- +""" +The final hook is run after all other binary hooks. +""" +import glob +import os + + +def remove_implicit_files(): + """ + Remove output files not created by explicitly specified image targets + + This uses the 'explicit_provides' file generated by the 'make-hooks' + script. If the file is empty, all output will be saved. + """ + explicit = set() + with open('./config/hooks/explicit_provides', 'r', encoding='utf-8') as fp: + for filename in fp: + explicit.add(filename.rstrip()) + + if not explicit: + print('remove_implicit_files: explicit_provides is empty ' + 'all binary output will be included') + quit() + + all = set(glob.glob('livecd.ubuntu-cpc.*')) + implicit = all - explicit + + print('remove_implicit_files: all artifacts considered: {}'.format(all)) + print('remove_implicit_files: explict artifacts to keep: ' + '{}'.format(explicit)) + print('remove_implicit_files: implicit artifacts to remove: ' + '{}'.format(implicit)) + + for file in implicit: + print('remove_implicit_files: removing {} ' + '{} bytes'.format(file, os.stat(file).st_size)) + if os.path.islink(file): + os.unlink(file) + elif os.path.isfile(file): + os.remove(file) + + +if __name__ == "__main__": + print('Running {}'.format(__file__)) + remove_implicit_files() diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/disk-image b/live-build/ubuntu-cpc/hooks.d/base/series/disk-image index 355c010c..4eb5b166 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/disk-image +++ b/live-build/ubuntu-cpc/hooks.d/base/series/disk-image @@ -1,3 +1,10 @@ base/disk-image.binary base/disk-image-uefi.binary base/disk-image-ppc64el.binary +provides livecd.ubuntu-cpc.ext4 +provides livecd.ubuntu-cpc.initrd-generic +provides livecd.ubuntu-cpc.initrd-generic-lpae +provides livecd.ubuntu-cpc.kernel-generic +provides livecd.ubuntu-cpc.kernel-generic-lpae +provides livecd.ubuntu-cpc.kernel-kvm +provides livecd.ubuntu-cpc.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 b/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 index cc3ced35..745adb9b 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 +++ b/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 @@ -1,2 +1,3 @@ depends disk-image base/qcow2-image.binary +provides livecd.ubuntu-cpc.img diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/root-dir b/live-build/ubuntu-cpc/hooks.d/base/series/root-dir index b41635af..b5d3b4e4 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/root-dir +++ b/live-build/ubuntu-cpc/hooks.d/base/series/root-dir @@ -1,3 +1 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image base/create-root-dir.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/squashfs b/live-build/ubuntu-cpc/hooks.d/base/series/squashfs index 6d2cb910..560fd79b 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/squashfs +++ b/live-build/ubuntu-cpc/hooks.d/base/series/squashfs @@ -1,4 +1,4 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image depends root-dir base/root-squashfs.binary +provides livecd.ubuntu-cpc.squashfs +provides livecd.ubuntu-cpc.squashfs.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/tarball b/live-build/ubuntu-cpc/hooks.d/base/series/tarball index 184046c2..c741c483 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/tarball +++ b/live-build/ubuntu-cpc/hooks.d/base/series/tarball @@ -1,4 +1,4 @@ -# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent -depends disk-image depends root-dir base/root-xz.binary +provides livecd.ubuntu-cpc.rootfs.tar.xz +provides livecd.ubuntu-cpc.rootfs.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/vagrant b/live-build/ubuntu-cpc/hooks.d/base/series/vagrant index a4eeb86f..6e5fcf39 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/vagrant +++ b/live-build/ubuntu-cpc/hooks.d/base/series/vagrant @@ -1,2 +1,3 @@ depends disk-image base/vagrant.binary +provides livecd.ubuntu-cpc.vagrant.box diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/vmdk b/live-build/ubuntu-cpc/hooks.d/base/series/vmdk index ba0acbae..c583fe96 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/vmdk +++ b/live-build/ubuntu-cpc/hooks.d/base/series/vmdk @@ -1,3 +1,5 @@ depends disk-image base/vmdk-image.binary base/vmdk-ova-image.binary +provides livecd.ubuntu-cpc.vmdk +provides livecd.ubuntu-cpc.ova diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/wsl b/live-build/ubuntu-cpc/hooks.d/base/series/wsl index 34915a31..b082f4e4 100644 --- a/live-build/ubuntu-cpc/hooks.d/base/series/wsl +++ b/live-build/ubuntu-cpc/hooks.d/base/series/wsl @@ -1,2 +1,4 @@ depends root-dir base/wsl.binary +provides livecd.ubuntu-cpc.wsl.rootfs.tar.gz +provides livecd.ubuntu-cpc.wsl.rootfs.manifest diff --git a/live-build/ubuntu-cpc/hooks.d/make-hooks b/live-build/ubuntu-cpc/hooks.d/make-hooks index 08ae45be..360e0cbe 100755 --- a/live-build/ubuntu-cpc/hooks.d/make-hooks +++ b/live-build/ubuntu-cpc/hooks.d/make-hooks @@ -31,10 +31,23 @@ to this: depends disk-image depends extra-settings extra/cloudB.binary + provides livecd.ubuntu-cpc.disk-kvm.img + provides livecd.ubuntu-cpc.disk-kvm.manifest Where "disk-image" and "extra-settings" may list scripts and dependencies which are to be processed before the script "extra/cloudB.binary" is called. +The "provides" directive defines a file that the hook creates; it can be +specified multiple times. The field is used by this script to generate a list +of output files created explicitly by the named image targets. The list is +saved to the "explicit_provides" file in the hooks output directory. In +the case of the "all" target this list would be empty. This list is +consumed by the "final.binary" hook file. + +The final.binary hook is always included as the last hook(s) if it exists, +it should not be specified in series files. It can be included from "base" +and/or "extra" directories with the final hook in "exta" running last. + ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate from scripts ending in ".binary". Even if you arrange them interleaved in your series files, the chroot scripts will be run before the binary scripts. @@ -74,6 +87,7 @@ class MakeHooks: self._quiet = quiet self._hooks_list = [] self._included = set() + self._provides = [] def reset(self): """Reset the internal state allowing instance to be reused for @@ -120,8 +134,9 @@ class MakeHooks: e.g. "vmdk" or "vagrant". """ self.collect_chroot_hooks() - self.collect_binary_hooks(image_sets) + self.collect_binary_hooks(image_sets, explicit_sets=True) self.create_symlinks() + self.create_explicit_provides() def collect_chroot_hooks(self): """Chroot hooks are numbered and not explicitly mentioned in series @@ -139,7 +154,7 @@ class MakeHooks: continue self._hooks_list.append(os.path.join("chroot", entry)) - def collect_binary_hooks(self, image_sets): + def collect_binary_hooks(self, image_sets, explicit_sets=False): """Search the series files for the given image_sets and parse them and their dependencies to generate a list of hook scripts to be run during image build. @@ -150,6 +165,11 @@ class MakeHooks: Populates the internal list of paths to hook scripts in the order in which the scripts are to be run. + + If "explicit_sets" is True, the files specified on lines starting + with "provides" will be added to self._provides to track explicit + output artifacts. This is only True for the initial images_sets + list, dependent image sets should set this to False. """ for image_set in image_sets: series_file = self.find_series_file(image_set) @@ -163,6 +183,7 @@ class MakeHooks: line = line.strip() if not line or line.startswith("#"): continue + m = re.match(r"^\s*depends\s+(\S+.*)$", line) if m: include_set = m.group(1) @@ -171,6 +192,13 @@ class MakeHooks: self._included.add(include_set) self.collect_binary_hooks([include_set,]) continue + + m = re.match(r"^\s*provides\s+(\S+.*)$", line) + if m: + if explicit_sets: + self._provides.append(m.group(1)) + continue + if not line in self._hooks_list: self._hooks_list.append(line) @@ -187,6 +215,12 @@ class MakeHooks: sys.stderr.write("WARNING: Hooks directory exists and is not empty.\n") os.makedirs(self._hooks_dir, exist_ok=True) + # Always add final.binary hooks if they exist + for subdir in ["base", "extra"]: + final_hook = os.path.join(self._script_dir, subdir, "final.binary") + if os.path.exists(final_hook) and os.path.isfile(final_hook): + self._hooks_list.append("base/final.binary") + for counter, hook in enumerate(self._hooks_list, start=1): hook_basename = os.path.basename(hook) @@ -195,13 +229,32 @@ class MakeHooks: hook_basename = m.group("basename") linkname = ("%03d-" % counter) + hook_basename - linksrc = os.path.join(self._hooks_dir, linkname) - linkdest = os.path.relpath(os.path.join(self._script_dir, hook), + linkdest = os.path.join(self._hooks_dir, linkname) + linksrc = os.path.relpath(os.path.join(self._script_dir, hook), self._hooks_dir) if not self._quiet: print("[HOOK] %s => %s" % (linkname, hook)) - os.symlink(linkdest, linksrc) + os.symlink(linksrc, linkdest) + + def create_explicit_provides(self): + """ + Create a file named "explicit_provides" in self._hooks_dir + listing all files named on "provides" in the series files of + targets explicitly named by the user. The file is created but + left empty if there are no explict "provides" keywords in the + targets (this is the case for 'all') + """ + with open(os.path.join(self._hooks_dir, "explicit_provides"), "w", + encoding="utf-8") as fp: + empty = True + for provides in self._provides: + if not self._quiet: + print("[PROVIDES] %s" % provides) + fp.write("%s\n" % provides) + empty = False + if not empty: + fp.write('livecd.magic-proxy.log\n') def cli(self, args): """Command line interface to the hooks generator."""