ubuntu-cpc: Only produce explicitly named artifacts (LP: #1837254)

In parallel builds where a list of image targets are provided the build
may produce binaries that are not part of the named set of targets but
are created by series dependencies.  These implicitly created binaries
may be generated by multiple builds but are unused as our convention for
the ubuntu-cpc project is to only consume binaries from the explicitly
named image targets; this avoid overwriting the same object by multiple
parallel builds.

This patch adds support for a 'provides' keyword for series files. It can
be specified multiple times per series file.  The field is used by the
make-hooks script to generate a list of output files created explicitly by
the named image targets.  The list is saved to the "explicit_provides"
file in the hooks output directory. In the case of the "all" target
this list would be empty.  This list is consumed by the "final.binary"
hook file.

This patch adds support for optional final.binary hooks in hooks.d/base
and/or hooks.d/extra.  These final.binary hooks are always included as
the last hook(s) if either exist with the hook in "extra" running last.

The base/final.binary hook includes logic to parse the "explicit_provides"
file generated by the make-hooks script and remove any binary output not
explicitly specified.

Some series files named unnecessary dependencies, specifically
disk-image, to keep output of implicit artifacts consistent between
parallel builds.  These unnecessary dependencies are removed in this
patch.

debian/changelog

@@ -1,3 +1,9 @@
+livecd-rootfs (2.408.51) UNRELEASED; urgency=medium
+
+  * ubuntu-cpc: Only produce explicitly specified artifacts (LP: #1837254)
+
+ -- Robert C Jennings <robert.jennings@canonical.com>  Mon, 26 Aug 2019 16:32:41 -0500
+
 livecd-rootfs (2.408.50) xenial; urgency=medium
 
   * Actually, do not depend on snapd on powerpc as well.  Snaps are not

live-build/ubuntu-cpc/hooks.d/base/final.binary

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+#-*- encoding: utf-8 -*-
+"""
+The final hook is run after all other binary hooks.
+"""
+import glob
+import os
+
+
+def remove_implicit_files():
+    """
+    Remove output files not created by explicitly specified image targets
+
+    This uses the 'explicit_provides' file generated by the 'make-hooks'
+    script.  If the file is empty, all output will be saved.
+    """
+    explicit = set()
+    with open('./config/hooks/explicit_provides', 'r', encoding='utf-8') as fp:
+        for filename in fp:
+            explicit.add(filename.rstrip())
+
+    if not explicit:
+        print('remove_implicit_files: explicit_provides is empty '
+              'all binary output will be included')
+        quit()
+
+    all = set(glob.glob('livecd.ubuntu-cpc.*'))
+    implicit = all - explicit
+
+    print('remove_implicit_files: all artifacts considered: {}'.format(all))
+    print('remove_implicit_files: explict artifacts to keep: '
+          '{}'.format(explicit))
+    print('remove_implicit_files: implicit artifacts to remove: '
+          '{}'.format(implicit))
+
+    for file in implicit:
+        print('remove_implicit_files: removing {} '
+              '{} bytes'.format(file, os.stat(file).st_size))
+        if os.path.islink(file):
+            os.unlink(file)
+        elif os.path.isfile(file):
+            os.remove(file)
+
+
+if __name__ == "__main__":
+    print('Running {}'.format(__file__))
+    remove_implicit_files()

live-build/ubuntu-cpc/hooks.d/base/series/disk-image

@@ -1,3 +1,12 @@
 base/disk-image.binary
 base/disk-image-uefi.binary
 base/disk-image-ppc64el.binary
+provides livecd.ubuntu-cpc.ext4
+provides livecd.ubuntu-cpc.initrd-generic
+provides livecd.ubuntu-cpc.initrd-generic-lpae
+provides livecd.ubuntu-cpc.initrd-powerpc64-smp
+provides livecd.ubuntu-cpc.kernel-generic
+provides livecd.ubuntu-cpc.kernel-generic-lpae
+provides livecd.ubuntu-cpc.kernel-powerpc64-smp
+provides livecd.ubuntu-cpc.kernel-kvm
+provides livecd.ubuntu-cpc.manifest

live-build/ubuntu-cpc/hooks.d/base/series/qcow2

@@ -1,2 +1,5 @@
 depends disk-image
 base/qcow2-image.binary
+provides livecd.ubuntu-cpc.disk1.img
+provides livecd.ubuntu-cpc.uefi1.img
+provides livecd.ubuntu-cpc.disk1.img.xz

live-build/ubuntu-cpc/hooks.d/base/series/root-dir

@@ -1,3 +1 @@
-# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent
-depends disk-image
 base/root-tarball.binary

live-build/ubuntu-cpc/hooks.d/base/series/squashfs

@@ -1,4 +1,4 @@
-# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent
-depends disk-image
 depends root-dir
 base/root-squashfs.binary
+provides livecd.ubuntu-cpc.squashfs
+provides livecd.ubuntu-cpc.squashfs.manifest

live-build/ubuntu-cpc/hooks.d/base/series/tarball

@@ -1,4 +1,5 @@
-# Include disk-image to ensure livecd.ubuntu-cpc.ext4 is consistent
-depends disk-image
 depends root-dir
 base/root-xz.binary
+provides livecd.ubuntu-cpc.rootfs.tar.gz
+provides livecd.ubuntu-cpc.rootfs.tar.xz
+provides livecd.ubuntu-cpc.rootfs.manifest

live-build/ubuntu-cpc/hooks.d/base/series/vagrant

@@ -1,2 +1,3 @@
 depends disk-image
 base/vagrant.binary
+provides livecd.ubuntu-cpc.vagrant.box

live-build/ubuntu-cpc/hooks.d/base/series/vmdk

@@ -1,3 +1,6 @@
 depends disk-image
 base/vmdk-image.binary
 base/vmdk-ova-image.binary
+provides livecd.ubuntu-cpc.disk1.vmdk
+provides livecd.ubuntu-cpc.uefi.vmdk
+provides livecd.ubuntu-cpc.ova

live-build/ubuntu-cpc/hooks.d/base/series/wsl

@@ -1,2 +1,4 @@
 depends root-dir
 base/wsl-gz.binary
+provides livecd.ubuntu-cpc.wsl.rootfs.tar.gz
+provides livecd.ubuntu-cpc.wsl.rootfs.manifest

live-build/ubuntu-cpc/hooks.d/make-hooks

@@ -31,10 +31,23 @@ to this:
     depends disk-image
     depends extra-settings
     extra/cloudB.binary
+    provides livecd.ubuntu-cpc.disk-kvm.img
+    provides livecd.ubuntu-cpc.disk-kvm.manifest
 
 Where "disk-image" and "extra-settings" may list scripts and dependencies which
 are to be processed before the script "extra/cloudB.binary" is called.
 
+The "provides" directive defines a file that the hook creates; it can be
+specified multiple times.  The field is used by this script to generate a list
+of output files created explicitly by the named image targets.  The list is
+saved to the "explicit_provides" file in the hooks output directory. In
+the case of the "all" target this list would be empty.  This list is
+consumed by the "final.binary" hook file.
+
+The final.binary hook is always included as the last hook(s) if it exists,
+it should not be specified in series files.  It can be included from "base"
+and/or "extra" directories with the final hook in "exta" running last.
+
 ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate
 from scripts ending in ".binary". Even if you arrange them interleaved in your
 series files, the chroot scripts will be run before the binary scripts.
@@ -74,6 +87,7 @@ class MakeHooks:
         self._quiet      = quiet
         self._hooks_list = []
         self._included   = set()
+        self._provides   = []
 
     def reset(self):
         """Reset the internal state allowing instance to be reused for
@@ -120,8 +134,9 @@ class MakeHooks:
         e.g. "vmdk" or "vagrant".
         """
         self.collect_chroot_hooks()
-        self.collect_binary_hooks(image_sets)
+        self.collect_binary_hooks(image_sets, explicit_sets=True)
         self.create_symlinks()
+        self.create_explicit_provides()
 
     def collect_chroot_hooks(self):
         """Chroot hooks are numbered and not explicitly mentioned in series
@@ -139,7 +154,7 @@ class MakeHooks:
                 continue
             self._hooks_list.append(os.path.join("chroot", entry))
 
-    def collect_binary_hooks(self, image_sets):
+    def collect_binary_hooks(self, image_sets, explicit_sets=False):
         """Search the series files for the given image_sets and parse them
         and their dependencies to generate a list of hook scripts to be run
         during image build.
@@ -150,6 +165,11 @@ class MakeHooks:
 
         Populates the internal list of paths to hook scripts in the order in
         which the scripts are to be run.
+
+        If "explicit_sets" is True, the files specified on lines starting
+        with "provides" will be added to self._provides to track explicit
+        output artifacts.  This is only True for the initial images_sets
+        list, dependent image sets should set this to False.
         """
         for image_set in image_sets:
             series_file = self.find_series_file(image_set)
@@ -163,6 +183,7 @@ class MakeHooks:
                     line = line.strip()
                     if not line or line.startswith("#"):
                         continue
+
                     m = re.match(r"^\s*depends\s+(\S+.*)$", line)
                     if m:
                         include_set = m.group(1)
@@ -171,6 +192,13 @@ class MakeHooks:
                         self._included.add(include_set)
                         self.collect_binary_hooks([include_set,])
                         continue
+
+                    m = re.match(r"^\s*provides\s+(\S+.*)$", line)
+                    if m:
+                        if explicit_sets:
+                            self._provides.append(m.group(1))
+                        continue
+
                     if not line in self._hooks_list:
                         self._hooks_list.append(line)
 
@@ -187,6 +215,12 @@ class MakeHooks:
             sys.stderr.write("WARNING: Hooks directory exists and is not empty.\n")
         os.makedirs(self._hooks_dir, exist_ok=True)
 
+        # Always add final.binary hooks if they exist
+        for subdir in ["base", "extra"]:
+            final_hook = os.path.join(self._script_dir, subdir, "final.binary")
+            if os.path.exists(final_hook) and os.path.isfile(final_hook):
+                self._hooks_list.append("base/final.binary")
+
         for counter, hook in enumerate(self._hooks_list, start=1):
             hook_basename = os.path.basename(hook)
 
@@ -195,13 +229,32 @@ class MakeHooks:
                 hook_basename = m.group("basename")
 
             linkname = ("%03d-" % counter) + hook_basename
-            linksrc  = os.path.join(self._hooks_dir, linkname)
+            linkdest = os.path.join(self._hooks_dir, linkname)
-            linkdest = os.path.relpath(os.path.join(self._script_dir, hook),
+            linksrc  = os.path.relpath(os.path.join(self._script_dir, hook),
                     self._hooks_dir)
 
             if not self._quiet:
                 print("[HOOK] %s => %s" % (linkname, hook))
-            os.symlink(linkdest, linksrc)
+            os.symlink(linksrc, linkdest)
+
+    def create_explicit_provides(self):
+        """
+        Create a file named "explicit_provides" in self._hooks_dir
+        listing all files named on "provides" in the series files of
+        targets explicitly named by the user.  The file is created but
+        left empty if there are no explict "provides" keywords in the
+        targets (this is the case for 'all')
+        """
+        with open(os.path.join(self._hooks_dir, "explicit_provides"), "w",
+                  encoding="utf-8") as fp:
+            empty = True
+            for provides in self._provides:
+                if not self._quiet:
+                    print("[PROVIDES] %s" % provides)
+                fp.write("%s\n" % provides)
+                empty = False
+            if not empty:
+                fp.write('livecd.magic-proxy.log\n')
 
     def cli(self, args):
         """Command line interface to the hooks generator."""