#!/usr/bin/env python3 #-*- encoding: utf-8 -*- """ This script parses a series file and its dependencies and generates a hooks folder containing symbolic links to the scripts that need to be invoked for a given image target set. For example, if you wish to build the image target sets "vmdk" and "vagrant", you would call this script as ./make-hooks --hooks-dir hooks vmdk vagrant Scripts live in subfolders below the "hooks.d" folder. Currently the folders "chroot" and "base" exist. The folder with the name "extra" is reserved for private scripts, which are not included in the source of livecd-rootfs. The scripts are not numbered, instead the order of their execution depends on the order in which they are listed in a series file. Series files are placed into the subfolders "base/series" or "extra/series". Each series file contains a list of scripts to be executed. Empty lines and lines starting with a '#' are ignored. Series files in "extra/series" override files in "base/series" with the same name. For example, if a series file "base/series/cloudA" exists and a series file "extra/series/cloudA", then the latter will be preferred. A series file in "extra/series" may also list scripts that are located in the "chroot" and "base" folders. In addition, series files can depend on other series files. For example, the series files for most custom images look similar to this: depends disk-image depends extra-settings extra/cloudB.binary provides livecd.ubuntu-cpc.disk-kvm.img provides livecd.ubuntu-cpc.disk-kvm.manifest Where "disk-image" and "extra-settings" may list scripts and dependencies which are to be processed before the script "extra/cloudB.binary" is called. The "provides" directive defines a file that the hook creates; it can be specified multiple times. The field is used by this script to generate a list of output files created explicitly by the named image targets. The list is saved to the "explicit_provides" file in the hooks output directory. In the case of the "all" target this list would be empty. This list is consumed by the "remove-implicit-artifacts" which is run at the end of the build. ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate from scripts ending in ".binary". Even if you arrange them interleaved in your series files, the chroot scripts will be run before the binary scripts. """ import argparse import os import re import shutil import sys import yaml SCRIPT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(sys.argv[0]))) HOOKS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "hooks")) EXIT_OK = 0 EXIT_ERR = 1 class MakeHooksError(Exception): pass class MakeHooks: """This class provides series file parsing and symlink generator functionality.""" def __init__(self, hooks_dir=None, quiet=False): """The hooks_dir parameter can be used to specify the path to the directory, into which the hook symlinks to the actual script files should be placed. If quiet is set to True, info messages during symlink creation will be suppressed. Use this if your build is not private, but you would like to hide which scripts are being run. """ self._script_dir = SCRIPT_DIR self._hooks_dir = hooks_dir or HOOKS_DIR self._quiet = quiet self._hooks_list = [] self._included = set() self._provides = [] def reset(self): """Reset the internal state allowing instance to be reused for another run.""" self._hooks_list.clear() self._included.clear() def print_usage(self): print( "CPC live build hook generator script \n" " \n" "Usage: ./make-hooks.sh [OPTIONS] \n" " \n" "Options: \n" " \n" " --help, -h Show this message and exit. \n" " --quiet, -q Only show warnings and error messages. \n" " --hooks-dir, -d The directory where to write the symlinks.\n" ) def find_series_file(self, image_set): """Search for the series file requested in the image_set parameter. The image_set parameter should be a string containing the name of an image target set represented by a series file. First the "extra/series" folder is searched followed by the "base/series" folder. When a file with the given name is found, the search stops and the full path to the file is returned. """ for subdir in ["extra", "base"]: series_file = os.path.join(self._script_dir, subdir, "series", image_set) if os.path.isfile(series_file): return series_file return None def make_hooks(self, image_sets): """Entry point for parsing series files and their dependencies and for generating the symlinks in the hooks folder. The image_sets parameter must be an iterable containing the names of the series files representing the corresponding image target sets, e.g. "vmdk" or "vagrant". """ self.collect_chroot_hooks() self.collect_binary_hooks(image_sets, explicit_sets=True) self.create_symlinks() self.create_explicit_provides() def collect_chroot_hooks(self): """Chroot hooks are numbered and not explicitly mentioned in series files. Collect them, sort them and add them to the internal list of paths to hook sripts. """ chroot_hooks_dir = os.path.join(self._script_dir, "chroot") chroot_entries = os.listdir(chroot_hooks_dir) chroot_entries.sort() for entry in chroot_entries: if not (entry.endswith(".chroot_early") or entry.endswith(".chroot")): continue self._hooks_list.append(os.path.join("chroot", entry)) def collect_binary_hooks(self, image_sets, explicit_sets=False): """Search the series files for the given image_sets and parse them and their dependencies to generate a list of hook scripts to be run during image build. The image_sets parameter must be an iterable containing the names of the series files representing the corresponding image target sets, e.g. "vmdk" or "vagrant". Populates the internal list of paths to hook scripts in the order in which the scripts are to be run. If "explicit_sets" is True, the files specified on lines starting with "provides" will be added to self._provides to track explicit output artifacts. This is only True for the initial images_sets list, dependent image sets should set this to False. """ for image_set in image_sets: series_file = self.find_series_file(image_set) if not series_file: raise MakeHooksError( "Series file for image set '%s' not found." % image_set) with open(series_file, "r", encoding="utf-8") as fp: for line in fp: line = line.strip() if not line or line.startswith("#"): continue m = re.match(r"^\s*depends\s+(\S+.*)$", line) if m: include_set = m.group(1) if include_set in self._included: continue self._included.add(include_set) self.collect_binary_hooks([include_set,]) continue m = re.match(r"^\s*provides\s+(\S+.*)$", line) if m: if explicit_sets: self._provides.append(m.group(1)) continue if not line in self._hooks_list: self._hooks_list.append(line) def create_symlinks(self): """Once the internal list of hooks scripts has been populated by a call to collect_?_hooks, this method is used to populate the hooks folder with enumerated symbolic links to the hooks scripts. If the folder does not exist, it will be created. If it exists, it must be empty or a MakeHooksError will be thrown. """ if os.path.isdir(self._hooks_dir) and os.listdir(self._hooks_dir): # Only print a warning, because directory might have been created # by auto/config voodoo. sys.stderr.write("WARNING: Hooks directory exists and is not empty.\n") os.makedirs(self._hooks_dir, exist_ok=True) for counter, hook in enumerate(self._hooks_list, start=1): hook_basename = os.path.basename(hook) m = re.match(r"^\d+-(?:\d+-)?(?P.*)$", hook_basename) if m: hook_basename = m.group("basename") linkname = ("%03d-" % counter) + hook_basename linkdest = os.path.join(self._hooks_dir, linkname) linksrc = os.path.relpath(os.path.join(self._script_dir, hook), self._hooks_dir) if not self._quiet: print("[HOOK] %s => %s" % (linkname, hook)) os.symlink(linksrc, linkdest) def create_explicit_provides(self): """ Create a file named "explicit_provides" in self._script_dir listing all files named on "provides" in the series files of targets explicitly named by the user. The file is created but left empty if there are no explict "provides" keywords in the targets (this is the case for 'all') """ with open(os.path.join(self._script_dir, "explicit_provides"), "w", encoding="utf-8") as fp: empty = True for provides in self._provides: if not self._quiet: print("[PROVIDES] %s" % provides) fp.write("%s\n" % provides) empty = False if not empty: fp.write('livecd.magic-proxy.log\n') def cli(self, args): """Command line interface to the hooks generator.""" parser = argparse.ArgumentParser() parser.add_argument("-q", "--quiet", dest="quiet", type=bool, help="Only show warnings and error messages.") parser.add_argument("-d", "--hooks-dir", dest="hooks_dir", type=str, help="The directory where to create the symlinks.") parser.add_argument("image_target", nargs="+", type=str, help="") self.reset() options = parser.parse_args(args) # Copy options to object attributes. for key, value in vars(options).items(): if value and hasattr(self, "_" + key): setattr(self, "_" + key, value) # Take remaining command line arguments, sanitize and turn into list. image_sets = re.sub(r";|,", " ", " ".join(options.image_target))\ .split() self.make_hooks(image_sets) if __name__ == "__main__": try: MakeHooks().cli(sys.argv[1:]) except MakeHooksError as e: sys.stderr.write("%s: %s\n" % (os.path.basename(sys.argv[0]), str(e))) sys.exit(EXIT_ERR)