mirror of
https://git.launchpad.net/livecd-rootfs
synced 2025-03-30 20:31:14 +00:00
f7cd30f59a
In parallel builds where a list of image targets are provided the build may produce binaries that are not part of the named set of targets but are created by series dependencies. These implicitly created binaries may be generated by multiple builds but are unused as our convention for the ubuntu-cpc project is to only consume binaries from the explicitly named image targets; this avoid overwriting the same object by multiple parallel builds. This patch adds support for a 'provides' keyword for series files. It can be specified multiple times per series file. The field is used by the make-hooks script to generate a list of output files created explicitly by the named image targets. The list is saved to the "explicit_provides" file in the hooks output directory. In the case of the "all" target this list would be empty. This list is consumed by the "final.binary" hook file. This patch adds support for optional final.binary hooks in hooks.d/base and/or hooks.d/extra. These final.binary hooks are always included as the last hook(s) if either exist with the hook in "extra" running last. The base/final.binary hook includes logic to parse the "explicit_provides" file generated by the make-hooks script and remove any binary output not explicitly specified. Some series files named unnecessary dependencies, specifically disk-image, to keep output of implicit artifacts consistent between parallel builds. These unnecessary dependencies are removed in this patch.
291 lines
12 KiB
Python
Executable File
291 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#-*- encoding: utf-8 -*-
|
|
"""
|
|
This script parses a series file and its dependencies and generates a hooks
|
|
folder containing symbolic links to the scripts that need to be invoked for
|
|
a given image target set.
|
|
|
|
For example, if you wish to build the image target sets "vmdk" and "vagrant",
|
|
you would call this script as
|
|
|
|
./make-hooks --hooks-dir hooks vmdk vagrant
|
|
|
|
Scripts live in subfolders below the "hooks.d" folder. Currently the folders
|
|
"chroot" and "base" exist. The folder with the name "extra" is reserved for
|
|
private scripts, which are not included in the source of livecd-rootfs. The
|
|
scripts are not numbered, instead the order of their execution depends on the
|
|
order in which they are listed in a series file.
|
|
|
|
Series files are placed into the subfolders "base/series" or "extra/series".
|
|
Each series file contains a list of scripts to be executed. Empty lines and
|
|
lines starting with a '#' are ignored. Series files in "extra/series" override
|
|
files in "base/series" with the same name. For example, if a series file
|
|
"base/series/cloudA" exists and a series file "extra/series/cloudA", then the
|
|
latter will be preferred.
|
|
|
|
A series file in "extra/series" may also list scripts that are located in the
|
|
"chroot" and "base" folders. In addition, series files can depend on other
|
|
series files. For example, the series files for most custom images look similar
|
|
to this:
|
|
|
|
depends disk-image
|
|
depends extra-settings
|
|
extra/cloudB.binary
|
|
provides livecd.ubuntu-cpc.disk-kvm.img
|
|
provides livecd.ubuntu-cpc.disk-kvm.manifest
|
|
|
|
Where "disk-image" and "extra-settings" may list scripts and dependencies which
|
|
are to be processed before the script "extra/cloudB.binary" is called.
|
|
|
|
The "provides" directive defines a file that the hook creates; it can be
|
|
specified multiple times. The field is used by this script to generate a list
|
|
of output files created explicitly by the named image targets. The list is
|
|
saved to the "explicit_provides" file in the hooks output directory. In
|
|
the case of the "all" target this list would be empty. This list is
|
|
consumed by the "final.binary" hook file.
|
|
|
|
The final.binary hook is always included as the last hook(s) if it exists,
|
|
it should not be specified in series files. It can be included from "base"
|
|
and/or "extra" directories with the final hook in "exta" running last.
|
|
|
|
ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate
|
|
from scripts ending in ".binary". Even if you arrange them interleaved in your
|
|
series files, the chroot scripts will be run before the binary scripts.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import yaml
|
|
|
|
SCRIPT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(sys.argv[0])))
|
|
HOOKS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "hooks"))
|
|
|
|
EXIT_OK = 0
|
|
EXIT_ERR = 1
|
|
|
|
class MakeHooksError(Exception):
|
|
pass
|
|
|
|
class MakeHooks:
|
|
"""This class provides series file parsing and symlink generator
|
|
functionality."""
|
|
|
|
def __init__(self, hooks_dir=None, quiet=False):
|
|
"""The hooks_dir parameter can be used to specify the path to the
|
|
directory, into which the hook symlinks to the actual script files
|
|
should be placed.
|
|
|
|
If quiet is set to True, info messages during symlink creation will
|
|
be suppressed. Use this if your build is not private, but you would
|
|
like to hide which scripts are being run.
|
|
"""
|
|
self._script_dir = SCRIPT_DIR
|
|
self._hooks_dir = hooks_dir or HOOKS_DIR
|
|
self._quiet = quiet
|
|
self._hooks_list = []
|
|
self._included = set()
|
|
self._provides = []
|
|
|
|
def reset(self):
|
|
"""Reset the internal state allowing instance to be reused for
|
|
another run."""
|
|
self._hooks_list.clear()
|
|
self._included.clear()
|
|
|
|
def print_usage(self):
|
|
print(
|
|
"CPC live build hook generator script \n"
|
|
" \n"
|
|
"Usage: ./make-hooks.sh [OPTIONS] <image_set> \n"
|
|
" \n"
|
|
"Options: \n"
|
|
" \n"
|
|
" --help, -h Show this message and exit. \n"
|
|
" --quiet, -q Only show warnings and error messages. \n"
|
|
" --hooks-dir, -d <dir> The directory where to write the symlinks.\n"
|
|
)
|
|
|
|
def find_series_file(self, image_set):
|
|
"""Search for the series file requested in the image_set parameter.
|
|
|
|
The image_set parameter should be a string containing the name of an
|
|
image target set represented by a series file. First the "extra/series"
|
|
folder is searched followed by the "base/series" folder.
|
|
|
|
When a file with the given name is found, the search stops and the
|
|
full path to the file is returned.
|
|
"""
|
|
for subdir in ["extra", "base"]:
|
|
series_file = os.path.join(self._script_dir, subdir, "series",
|
|
image_set)
|
|
if os.path.isfile(series_file):
|
|
return series_file
|
|
return None
|
|
|
|
def make_hooks(self, image_sets):
|
|
"""Entry point for parsing series files and their dependencies and
|
|
for generating the symlinks in the hooks folder.
|
|
|
|
The image_sets parameter must be an iterable containing the names of
|
|
the series files representing the corresponding image target sets,
|
|
e.g. "vmdk" or "vagrant".
|
|
"""
|
|
self.collect_chroot_hooks()
|
|
self.collect_binary_hooks(image_sets, explicit_sets=True)
|
|
self.create_symlinks()
|
|
self.create_explicit_provides()
|
|
|
|
def collect_chroot_hooks(self):
|
|
"""Chroot hooks are numbered and not explicitly mentioned in series
|
|
files. Collect them, sort them and add them to the internal list of
|
|
paths to hook sripts.
|
|
"""
|
|
chroot_hooks_dir = os.path.join(self._script_dir, "chroot")
|
|
|
|
chroot_entries = os.listdir(chroot_hooks_dir)
|
|
chroot_entries.sort()
|
|
|
|
for entry in chroot_entries:
|
|
if not (entry.endswith(".chroot_early") or
|
|
entry.endswith(".chroot")):
|
|
continue
|
|
self._hooks_list.append(os.path.join("chroot", entry))
|
|
|
|
def collect_binary_hooks(self, image_sets, explicit_sets=False):
|
|
"""Search the series files for the given image_sets and parse them
|
|
and their dependencies to generate a list of hook scripts to be run
|
|
during image build.
|
|
|
|
The image_sets parameter must be an iterable containing the names of
|
|
the series files representing the corresponding image target sets,
|
|
e.g. "vmdk" or "vagrant".
|
|
|
|
Populates the internal list of paths to hook scripts in the order in
|
|
which the scripts are to be run.
|
|
|
|
If "explicit_sets" is True, the files specified on lines starting
|
|
with "provides" will be added to self._provides to track explicit
|
|
output artifacts. This is only True for the initial images_sets
|
|
list, dependent image sets should set this to False.
|
|
"""
|
|
for image_set in image_sets:
|
|
series_file = self.find_series_file(image_set)
|
|
|
|
if not series_file:
|
|
raise MakeHooksError(
|
|
"Series file for image set '%s' not found." % image_set)
|
|
|
|
with open(series_file, "r", encoding="utf-8") as fp:
|
|
for line in fp:
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
|
|
m = re.match(r"^\s*depends\s+(\S+.*)$", line)
|
|
if m:
|
|
include_set = m.group(1)
|
|
if include_set in self._included:
|
|
continue
|
|
self._included.add(include_set)
|
|
self.collect_binary_hooks([include_set,])
|
|
continue
|
|
|
|
m = re.match(r"^\s*provides\s+(\S+.*)$", line)
|
|
if m:
|
|
if explicit_sets:
|
|
self._provides.append(m.group(1))
|
|
continue
|
|
|
|
if not line in self._hooks_list:
|
|
self._hooks_list.append(line)
|
|
|
|
def create_symlinks(self):
|
|
"""Once the internal list of hooks scripts has been populated by a
|
|
call to collect_?_hooks, this method is used to populate the hooks
|
|
folder with enumerated symbolic links to the hooks scripts. If the
|
|
folder does not exist, it will be created. If it exists, it must be
|
|
empty or a MakeHooksError will be thrown.
|
|
"""
|
|
if os.path.isdir(self._hooks_dir) and os.listdir(self._hooks_dir):
|
|
# Only print a warning, because directory might have been created
|
|
# by auto/config voodoo.
|
|
sys.stderr.write("WARNING: Hooks directory exists and is not empty.\n")
|
|
os.makedirs(self._hooks_dir, exist_ok=True)
|
|
|
|
# Always add final.binary hooks if they exist
|
|
for subdir in ["base", "extra"]:
|
|
final_hook = os.path.join(self._script_dir, subdir, "final.binary")
|
|
if os.path.exists(final_hook) and os.path.isfile(final_hook):
|
|
self._hooks_list.append("base/final.binary")
|
|
|
|
for counter, hook in enumerate(self._hooks_list, start=1):
|
|
hook_basename = os.path.basename(hook)
|
|
|
|
m = re.match(r"^\d+-(?:\d+-)?(?P<basename>.*)$", hook_basename)
|
|
if m:
|
|
hook_basename = m.group("basename")
|
|
|
|
linkname = ("%03d-" % counter) + hook_basename
|
|
linkdest = os.path.join(self._hooks_dir, linkname)
|
|
linksrc = os.path.relpath(os.path.join(self._script_dir, hook),
|
|
self._hooks_dir)
|
|
|
|
if not self._quiet:
|
|
print("[HOOK] %s => %s" % (linkname, hook))
|
|
os.symlink(linksrc, linkdest)
|
|
|
|
def create_explicit_provides(self):
|
|
"""
|
|
Create a file named "explicit_provides" in self._hooks_dir
|
|
listing all files named on "provides" in the series files of
|
|
targets explicitly named by the user. The file is created but
|
|
left empty if there are no explict "provides" keywords in the
|
|
targets (this is the case for 'all')
|
|
"""
|
|
with open(os.path.join(self._hooks_dir, "explicit_provides"), "w",
|
|
encoding="utf-8") as fp:
|
|
empty = True
|
|
for provides in self._provides:
|
|
if not self._quiet:
|
|
print("[PROVIDES] %s" % provides)
|
|
fp.write("%s\n" % provides)
|
|
empty = False
|
|
if not empty:
|
|
fp.write('livecd.magic-proxy.log\n')
|
|
|
|
def cli(self, args):
|
|
"""Command line interface to the hooks generator."""
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument("-q", "--quiet", dest="quiet", type=bool,
|
|
help="Only show warnings and error messages.")
|
|
parser.add_argument("-d", "--hooks-dir", dest="hooks_dir", type=str,
|
|
help="The directory where to create the symlinks.")
|
|
parser.add_argument("image_target", nargs="+", type=str,
|
|
help="")
|
|
|
|
self.reset()
|
|
options = parser.parse_args(args)
|
|
|
|
# Copy options to object attributes.
|
|
for key, value in vars(options).items():
|
|
if value and hasattr(self, "_" + key):
|
|
setattr(self, "_" + key, value)
|
|
|
|
# Take remaining command line arguments, sanitize and turn into list.
|
|
image_sets = re.sub(r";|,", " ", " ".join(options.image_target))\
|
|
.split()
|
|
|
|
self.make_hooks(image_sets)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
MakeHooks().cli(sys.argv[1:])
|
|
except MakeHooksError as e:
|
|
sys.stderr.write("%s: %s\n" % (os.path.basename(sys.argv[0]), str(e)))
|
|
sys.exit(EXIT_ERR)
|