Imported 2.549

No reason for CPC update specified.

debian/changelog

@@ -1,3 +1,11 @@
+livecd-rootfs (2.549) disco; urgency=medium
+
+  [ Tobias Koch ]
+  * Use series files with dependency handling to generate hook symlinks
+    dynamically
+
+ -- Steve Langasek <steve.langasek@ubuntu.com>  Mon, 10 Dec 2018 14:15:34 -0800
+
 livecd-rootfs (2.548) disco; urgency=medium
 
   * Key netplan delegation to NetworkManager on presence of

live-build/auto/config

@@ -1026,6 +1026,19 @@ EOF
 	ubuntu-touch:*|ubuntu-touch-custom:*|ubuntu-core:system-image|ubuntu-desktop-next:system-image|ubuntu-cpc:*|ubuntu-server:live)
 		cp -af /usr/share/livecd-rootfs/live-build/${PROJECT}/* \
 			config/
+
+		if [ "$PROJECT" = "ubuntu-cpc" ]; then
+			case ${IMAGE_TARGETS:-} in
+				"")
+					config/hooks.d/make-hooks --hooks-dir config/hooks all
+					;;
+				*)
+					config/hooks.d/make-hooks --hooks-dir config/hooks \
+						"$IMAGE_TARGETS"
+					;;
+			esac
+		fi
+
 		if [ "$IMAGEFORMAT" = none ]; then
 			rm -f config/hooks/*.binary*
 		fi

live-build/ubuntu-cpc/README.cpc.md

@@ -0,0 +1,68 @@
+# TL;DR
+
+In order to generate the hooks for a specific image target set, call the
+`make-hooks` script, located in `hooks.d` as
+
+    ./make-hooks --hooks-dir ../hooks <image_set>
+
+where `image_set` is the name of a series file (e.g. "vagrant") without leading
+path components. Do *not* check in the `hooks` folder, it is automatically
+generated by `auto/config` during Live Build runs.
+
+
+# Hook placement and ordering
+
+Scripts live in subfolders below the `hooks.d` folder. Currently the folders
+`chroot` and `base` exist. The folder with the name `extra` is reserved for
+private scripts, which are not included in the source of livecd-rootfs. The
+scripts are not numbered, instead the order of their execution depends on the
+order in which they are listed in a *series* file.
+
+Series files are placed in subfolders `hooks.d/base/series` or
+`hooks.d/extra/series`. Each series file contains a list of scripts to be
+executed. Empty lines and lines starting with a `#` are ignored.
+
+Series files in `extra/series` override files in `base/series` with the same
+name. For example, if a series file `base/series/cloudA` exists and a series
+file `extra/series/cloudA`, then the latter will be preferred.
+
+A series file in `extra/series` may also list scripts that are located in the
+`chroot` and `base` folders. In addition, series files can *depend* on other
+series files. For example, the series files for most custom images look similar
+to this:
+
+    depends disk-image
+    depends extra-settings
+    extra/cloudB.binary
+
+Where `disk-image` and `extra-settings` may list scripts and dependencies which
+are to be processed before the script `extra/cloudB.binary` is called.
+
+ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate
+from scripts ending in ".binary". Even if you arrange them interleaved in your
+series files, the chroot scripts will be run before the binary scripts.
+
+# Image set selection for Live Build
+
+During a Live Build, enumerated symbolic links are generated based on the
+contents of one or more series files. The series files are selected according
+to the contents of the `IMAGE_TARGETS` environment variable. For example, in
+order to trigger the build of `squashfs` and `vagrant`, list them in the
+`IMAGE_TARGETS` variable as `squashfs,vagrant`. The separator can be a comma,
+a semi-colon or whitespace.
+
+The generation of the symbolic links is triggered from the `auto/config` script,
+from where the contents of the `IMAGE_TARGETS` environment variable are passed
+on to the `make-hooks` script.
+
+
+# Symlink generation
+
+Since Live Build itself does not know about series files, a traditional `hooks`
+folder is generated using the `make-hooks` script. The script takes as arguments
+the names of the series files to be processed.
+
+The script parses the series files and generates enumerated symbolic links for
+all entries. Per default, these are placed into a directory named `hooks` next
+to the `hooks.d` directory. This can be changed using the `--hooks-dir`
+parameter.

live-build/ubuntu-cpc/hooks.d/base/disk-image.binary

@@ -12,6 +12,8 @@ BOOTPART_END=
 BOOT_MOUNTPOINT=
 ROOTPART_START=1
 
+my_d=$(dirname $(readlink -f ${0}))
+
 case $ARCH:$SUBARCH in
 	ppc64el:*|powerpc:*)
 		echo "POWER disk images are handled separately"
@@ -97,7 +99,7 @@ case $ARCH:$SUBARCH in
 		# not the best place for this, but neither flash-kernel nor
 		# u-boot have provisions for installing u-boot via maintainer
 		# script
-		config/hooks/raspi2/mkknlimg --dtok \
+		${my_d}/raspi2/mkknlimg --dtok \
 			mountpoint/usr/lib/u-boot/rpi_2/u-boot.bin \
 			mountpoint/boot/firmware/uboot.bin
 		;;

live-build/ubuntu-cpc/hooks.d/base/qcow2-image.binary

@@ -1,14 +1,5 @@
 #!/bin/bash -ex
 
-case $IMAGE_TARGETS in
-	""|*qcow2*)
-		;;
-	*)
-		echo "Skipping qcow2 image build"
-		exit 0
-		;;
-esac
-
 case $ARCH:$SUBARCH in
 	# Not sure if any other cloud images use subarch for something that
 	# should take qcow2 format, so only skipping this on raspi2 for now.

live-build/ubuntu-cpc/hooks.d/base/root-squashfs.binary

@@ -3,15 +3,6 @@
 #
 # Generate a squashfs root and manifest
 
-case $IMAGE_TARGETS in
-	""|*squashfs*)
-		;;
-	*)
-		echo "Skipping squashfs build"
-		exit 0
-		;;
-esac
-
 if [ -n "$SUBARCH" ]; then
 	echo "Skipping rootfs build for subarch flavor build"
 	exit 0

live-build/ubuntu-cpc/hooks.d/base/series/all

@@ -0,0 +1 @@
+base

live-build/ubuntu-cpc/hooks.d/base/series/base

@@ -0,0 +1,7 @@
+depends root-dir
+depends tarball
+depends squashfs
+depends disk-image
+depends qcow2
+depends vmdk
+depends vagrant

live-build/ubuntu-cpc/hooks.d/base/series/disk-image

@@ -0,0 +1,3 @@
+base/disk-image.binary
+base/disk-image-uefi.binary
+base/disk-image-ppc64el.binary

live-build/ubuntu-cpc/hooks.d/base/series/qcow2

@@ -0,0 +1,2 @@
+depends disk-image
+base/qcow2-image.binary

live-build/ubuntu-cpc/hooks.d/base/series/root-dir

@@ -0,0 +1 @@
+base/create-root-dir.binary

live-build/ubuntu-cpc/hooks.d/base/series/squashfs

@@ -0,0 +1,2 @@
+depends root-dir
+base/root-squashfs.binary

live-build/ubuntu-cpc/hooks.d/base/series/tarball

@@ -0,0 +1,2 @@
+depends root-dir
+base/root-xz.binary

live-build/ubuntu-cpc/hooks.d/base/series/vagrant

@@ -0,0 +1,2 @@
+depends disk-image
+base/vagrant.binary

live-build/ubuntu-cpc/hooks.d/base/series/vmdk

@@ -0,0 +1,3 @@
+depends disk-image
+base/vmdk-image.binary
+base/vmdk-ova-image.binary

live-build/ubuntu-cpc/hooks.d/base/vagrant.binary

@@ -24,15 +24,6 @@ case ${SUBPROJECT:-} in
         ;;
 esac
 
-case $IMAGE_TARGETS in
-	""|*vagrant*)
-		;;
-	*)
-		echo "Skipping Vagrant image build"
-		exit 0
-		;;
-esac
-
 cur_d=${PWD}
 my_d=$(dirname $(readlink -f ${0}))
 

live-build/ubuntu-cpc/hooks.d/base/vmdk-image.binary

@@ -18,15 +18,6 @@ case $ARCH in
                      exit 0;;
 esac
 
-case ${IMAGE_TARGETS:-} in
-	""|*vmdk*)
-		;;
-	*)
-		echo "Skipping VMDK image build"
-		exit 0
-		;;
-esac
-
 . config/functions
 
 if [ -e binary/boot/disk-uefi.ext4 ]; then

live-build/ubuntu-cpc/hooks.d/base/vmdk-ova-image.binary

@@ -32,15 +32,6 @@ case $ARCH in
             exit 0;;
 esac
 
-case ${IMAGE_TARGETS:-} in
-	""|*vmdk*)
-		;;
-	*)
-		echo "Skipping OVA image build"
-		exit 0
-		;;
-esac
-
 cur_d=${PWD}
 my_d=$(dirname $(readlink -f ${0}))
 

live-build/ubuntu-cpc/hooks.d/make-hooks

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+#-*- encoding: utf-8 -*-
+"""
+This script parses a series file and its dependencies and generates a hooks
+folder containing symbolic links to the scripts that need to be invoked for
+a given image target set.
+
+For example, if you wish to build the image target sets "vmdk" and "vagrant",
+you would call this script as
+
+./make-hooks --hooks-dir hooks vmdk vagrant
+
+Scripts live in subfolders below the "hooks.d" folder. Currently the folders
+"chroot" and "base" exist. The folder with the name "extra" is reserved for
+private scripts, which are not included in the source of livecd-rootfs. The
+scripts are not numbered, instead the order of their execution depends on the
+order in which they are listed in a series file.
+
+Series files are placed into the subfolders "base/series" or "extra/series".
+Each series file contains a list of scripts to be executed. Empty lines and
+lines starting with a '#' are ignored. Series files in "extra/series" override
+files in "base/series" with the same name. For example, if a series file
+"base/series/cloudA" exists and a series file "extra/series/cloudA", then the
+latter will be preferred.
+
+A series file in "extra/series" may also list scripts that are located in the
+"chroot" and "base" folders. In addition, series files can depend on other
+series files. For example, the series files for most custom images look similar
+to this:
+
+    depends disk-image
+    depends extra-settings
+    extra/cloudB.binary
+
+Where "disk-image" and "extra-settings" may list scripts and dependencies which
+are to be processed before the script "extra/cloudB.binary" is called.
+
+ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate
+from scripts ending in ".binary". Even if you arrange them interleaved in your
+series files, the chroot scripts will be run before the binary scripts.
+"""
+
+import argparse
+import os
+import re
+import shutil
+import sys
+import yaml
+
+SCRIPT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(sys.argv[0])))
+HOOKS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "hooks"))
+
+EXIT_OK = 0
+EXIT_ERR = 1
+
+class MakeHooksError(Exception):
+    pass
+
+class MakeHooks:
+    """This class provides series file parsing and symlink generator
+    functionality."""
+
+    def __init__(self, hooks_dir=None, quiet=False):
+        """The hooks_dir parameter can be used to specify the path to the
+        directory, into which the hook symlinks to the actual script files
+        should be placed.
+
+        If quiet is set to True, info messages during symlink creation will
+        be suppressed. Use this if your build is not private, but you would
+        like to hide which scripts are being run.
+        """
+        self._script_dir = SCRIPT_DIR
+        self._hooks_dir  = hooks_dir or HOOKS_DIR
+        self._quiet      = quiet
+        self._hooks_list = []
+        self._included   = set()
+
+    def reset(self):
+        """Reset the internal state allowing instance to be reused for
+        another run."""
+        self._hooks_list.clear()
+        self._included.clear()
+
+    def print_usage(self):
+        print(
+            "CPC live build hook generator script                               \n"
+            "                                                                   \n"
+            "Usage: ./make-hooks.sh [OPTIONS] <image_set>                       \n"
+            "                                                                   \n"
+            "Options:                                                           \n"
+            "                                                                   \n"
+            " --help, -h              Show this message and exit.               \n"
+            " --quiet, -q             Only show warnings and error messages.    \n"
+            " --hooks-dir, -d <dir>   The directory where to write the symlinks.\n"
+        )
+
+    def find_series_file(self, image_set):
+        """Search for the series file requested in the image_set parameter.
+
+        The image_set parameter should be a string containing the name of an
+        image target set represented by a series file. First the "extra/series"
+        folder is searched followed by the "base/series" folder.
+        
+        When a file with the given name is found, the search stops and the
+        full path to the file is returned.
+        """
+        for subdir in ["extra", "base"]:
+            series_file = os.path.join(self._script_dir, subdir, "series",
+                    image_set)
+            if os.path.isfile(series_file):
+                return series_file
+        return None
+
+    def make_hooks(self, image_sets):
+        """Entry point for parsing series files and their dependencies and
+        for generating the symlinks in the hooks folder.
+
+        The image_sets parameter must be an iterable containing the names of
+        the series files representing the corresponding image target sets,
+        e.g. "vmdk" or "vagrant".
+        """
+        self.collect_chroot_hooks()
+        self.collect_binary_hooks(image_sets)
+        self.create_symlinks()
+
+    def collect_chroot_hooks(self):
+        """Chroot hooks are numbered and not explicitly mentioned in series
+        files. Collect them, sort them and add them to the internal list of
+        paths to hook sripts.
+        """
+        chroot_hooks_dir = os.path.join(self._script_dir, "chroot")
+
+        chroot_entries = os.listdir(chroot_hooks_dir)
+        chroot_entries.sort()
+
+        for entry in chroot_entries:
+            if not (entry.endswith(".chroot_early") or 
+                    entry.endswith(".chroot")):
+                continue
+            self._hooks_list.append(os.path.join("chroot", entry))
+
+    def collect_binary_hooks(self, image_sets):
+        """Search the series files for the given image_sets and parse them
+        and their dependencies to generate a list of hook scripts to be run
+        during image build.
+
+        The image_sets parameter must be an iterable containing the names of
+        the series files representing the corresponding image target sets,
+        e.g. "vmdk" or "vagrant".
+
+        Populates the internal list of paths to hook scripts in the order in
+        which the scripts are to be run.
+        """
+        for image_set in image_sets:
+            series_file = self.find_series_file(image_set)
+
+            if not series_file:
+                raise MakeHooksError(
+                    "Series file for image set '%s' not found." % image_set)
+
+            with open(series_file, "r", encoding="utf-8") as fp:
+                for line in fp:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    m = re.match(r"^\s*depends\s+(\S+.*)$", line)
+                    if m:
+                        include_set = m.group(1)
+                        if include_set in self._included:
+                            continue
+                        self._included.add(include_set)
+                        self.collect_binary_hooks([include_set,])
+                        continue
+                    if not line in self._hooks_list:
+                        self._hooks_list.append(line)
+
+    def create_symlinks(self):
+        """Once the internal list of hooks scripts has been populated by a
+        call to collect_?_hooks, this method is used to populate the hooks
+        folder with enumerated symbolic links to the hooks scripts. If the
+        folder does not exist, it will be created. If it exists, it must be
+        empty or a MakeHooksError will be thrown.
+        """
+        if os.path.isdir(self._hooks_dir) and os.listdir(self._hooks_dir):
+            # Only print a warning, because directory might have been created
+            # by auto/config voodoo.
+            sys.stderr.write("WARNING: Hooks directory exists and is not empty.\n")
+        os.makedirs(self._hooks_dir, exist_ok=True)
+
+        for counter, hook in enumerate(self._hooks_list, start=1):
+            hook_basename = os.path.basename(hook)
+
+            m = re.match(r"^\d+-(?:\d+-)?(?P<basename>.*)$", hook_basename)
+            if m:
+                hook_basename = m.group("basename")
+
+            linkname = ("%03d-" % counter) + hook_basename
+            linksrc  = os.path.join(self._hooks_dir, linkname)
+            linkdest = os.path.relpath(os.path.join(self._script_dir, hook),
+                    self._hooks_dir)
+
+            if not self._quiet:
+                print("[HOOK] %s => %s" % (linkname, hook))
+            os.symlink(linkdest, linksrc)
+
+    def cli(self, args):
+        """Command line interface to the hooks generator."""
+        parser = argparse.ArgumentParser()
+
+        parser.add_argument("-q", "--quiet", dest="quiet", type=bool,
+                help="Only show warnings and error messages.")
+        parser.add_argument("-d", "--hooks-dir", dest="hooks_dir", type=str,
+                help="The directory where to create the symlinks.")
+        parser.add_argument("image_target", nargs="+", type=str,
+                help="")
+
+        self.reset()
+        options = parser.parse_args(args)
+
+        # Copy options to object attributes.
+        for key, value in vars(options).items():
+            if value and hasattr(self, "_" + key):
+                setattr(self, "_" + key, value)
+
+        # Take remaining command line arguments, sanitize and turn into list.
+        image_sets = re.sub(r";|,", " ", " ".join(options.image_target))\
+                .split()
+
+        self.make_hooks(image_sets)
+
+
+if __name__ == "__main__":
+    try:
+        MakeHooks().cli(sys.argv[1:])
+    except MakeHooksError as e:
+        sys.stderr.write("%s: %s\n" % (os.path.basename(sys.argv[0]), str(e)))
+        sys.exit(EXIT_ERR)

live-build/ubuntu-cpc/hooks/999-extras.binary

@@ -1,16 +0,0 @@
-#!/bin/bash
-# Execute extra binary hooks.
-
-my_dir=$(dirname $(readlink -f ${0}))
-extra_d=${my_dir}/extra
-
-if [ ! -d ${my_dir}/extra ]; then
-    exit 0
-fi
-
-export IMAGE_STR="# CLOUD_IMG: This file was created/modified by the Cloud Image build process"
-export CLOUD_IMG_STR="$IMAGE_STR"
-export FS_LABEL="cloudimg-rootfs"
-
-# Cleaner execution
-/bin/run-parts --exit-on-error --regex ".*\.binary" "${extra_d}"