From b3fdc4e615e0cfc5ed7e6607ef4397fa5c7a24e8 Mon Sep 17 00:00:00 2001 From: "michael.hudson@canonical.com" Date: Wed, 14 Jan 2026 16:55:49 +1300 Subject: [PATCH] Add isobuild tool to build installer ISOs This adds a new tool, isobuild, which replaces the ISO-building functionality previously provided by live-build and cdimage. It is invoked from auto/build when MAKE_ISO=yes. The tool supports: - Layered desktop images (Ubuntu Desktop, flavors) - Non-layered images (Kubuntu, Ubuntu Unity) - Images with package pools (most installers) - Images without pools (Ubuntu Core Installer) The isobuild command has several subcommands: - init: Initialize the ISO build directory structure - setup-apt: Configure APT for package pool generation - generate-pool: Create the package pool from a seed - generate-sources: Generate cdrom.sources for the installed system - add-live-filesystem: Add squashfs and kernel/initrd to the ISO - make-bootable: Add GRUB and other boot infrastructure - make-iso: Generate the final ISO image auto/config is updated to: - Set MAKE_ISO=yes for relevant image types - Set POOL_SEED_NAME for images that need a package pool - Invoke gen-iso-ids to compute ISO metadata auto/build is updated to: - Remove old live-build ISO handling code - Invoke isobuild at appropriate points in the build lb_binary_layered is updated to create squashfs files with cdrom.sources included for use in the ISO. --- live-build/auto/build | 43 ++- live-build/auto/config | 29 ++ live-build/functions | 7 + live-build/isobuild | 221 ++++++++++++++++ live-build/isobuilder/__init__.py | 1 + live-build/isobuilder/apt_state.py | 110 ++++++++ live-build/isobuilder/builder.py | 365 ++++++++++++++++++++++++++ live-build/isobuilder/gpg_key.py | 58 ++++ live-build/isobuilder/pool_builder.py | 166 ++++++++++++ live-build/lb_binary_layered | 14 + 10 files changed, 1007 insertions(+), 7 deletions(-) create mode 100755 live-build/isobuild create mode 100644 live-build/isobuilder/__init__.py create mode 100644 live-build/isobuilder/apt_state.py create mode 100644 live-build/isobuilder/builder.py create mode 100644 live-build/isobuilder/gpg_key.py create mode 100644 live-build/isobuilder/pool_builder.py diff --git a/live-build/auto/build b/live-build/auto/build index fd9c8943..4a8adfb7 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -208,6 +208,22 @@ EOF undivert_update_initramfs undivert_grub chroot fi + if [ "${MAKE_ISO}" = yes ]; then + isobuild init --disk-info "$(cat config/iso-ids/disk-info)" --series "${LB_DISTRIBUTION}" --arch "${ARCH}" + # Determine which chroot directory has the apt configuration to use. + # Layered builds (PASSES set) create overlay directories named + # "overlay.base", "overlay.live", etc. - we use the first one (base). + # Single-pass builds use the "chroot" directory directly. + if [ "${PASSES}" ]; then + CHROOT="overlay.$(set -- $PASSES; echo $1)" + else + CHROOT=chroot + fi + isobuild setup-apt --chroot $CHROOT + if [ -n "${POOL_SEED_NAME}" ]; then + isobuild generate-pool --package-list-file "config/germinate-output/${POOL_SEED_NAME}" + fi + fi if [ -d chroot/etc/apt/preferences.d.save ]; then # https://mastodon.social/@scream@botsin.space @@ -427,13 +443,6 @@ if [ -e config/manifest-minimal-remove ]; then cp config/manifest-minimal-remove "$PREFIX.manifest-minimal-remove" fi -for ISO in binary.iso binary.hybrid.iso; do - [ -e "$ISO" ] || continue - ln "$ISO" "$PREFIX.iso" - chmod 644 "$PREFIX.iso" - break -done - if [ -e "binary/$INITFS/filesystem.dir" ]; then (cd "binary/$INITFS/filesystem.dir/" && tar -c --sort=name --xattrs *) | \ gzip -9 --rsyncable > "$PREFIX.rootfs.tar.gz" @@ -558,3 +567,23 @@ case $PROJECT in ubuntu-cpc) config/hooks.d/remove-implicit-artifacts esac + +if [ "${MAKE_ISO}" = "yes" ]; then + # Link build artifacts with "for-iso." prefix for isobuild to consume. + # Layered builds create squashfs via lb_binary_layered (which already + # creates for-iso.*.squashfs files). Single-pass builds only have + # ${PREFIX}.squashfs, so we link it here. + if [ -z "$PASSES" ]; then + ln ${PREFIX}.squashfs for-iso.filesystem.squashfs + fi + # Link kernel and initrd files. The ${thing#${PREFIX}} expansion strips + # the PREFIX, so "livecd.ubuntu-server.kernel-generic" becomes + # "for-iso.kernel-generic". + for thing in ${PREFIX}.kernel-* ${PREFIX}.initrd-*; do + ln $thing for-iso${thing#${PREFIX}} + done + isobuild add-live-filesystem --artifact-prefix for-iso. + isobuild make-bootable --project "${PROJECT}" --capproject "$(cat config/iso-ids/capproject)" \ + ${SUBARCH:+--subarch "${SUBARCH}"} + isobuild make-iso --volid "$(cat config/iso-ids/vol-id)" --dest ${PREFIX}.iso +fi diff --git a/live-build/auto/config b/live-build/auto/config index 79015586..fb44849c 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -663,10 +663,23 @@ if ! [ -e config/germinate-output/structure ]; then -s $FLAVOUR.$SUITE $GERMINATE_ARG -a ${ARCH_VARIANT:-$ARCH}) fi +# ISO build configuration. These defaults are overridden per-project below. +# +# MAKE_ISO: Set to "yes" to generate an installer ISO at the end of the build. +# This triggers isobuild to run in auto/build. +MAKE_ISO=no +# POOL_SEED_NAME: The germinate output file defining packages for the ISO's +# package pool (repository). Different flavors use different seeds: +# - "ship-live" for most desktop images +# - "server-ship-live" for Ubuntu Server (includes server-specific packages) +# - "" (empty) for images without a pool, like Ubuntu Core Installer +POOL_SEED_NAME=ship-live + # Common functionality for layered desktop images common_layered_desktop_image() { touch config/universe-enabled PASSES_TO_LAYERS="true" + MAKE_ISO=yes if [ -n "$HAS_MINIMAL" ]; then if [ -z "$MINIMAL_TASKS" ]; then @@ -897,6 +910,7 @@ case $PROJECT in add_task install minimal standard add_task install kubuntu-desktop LIVE_TASK='kubuntu-live' + MAKE_ISO=yes add_chroot_hook remove-gnome-icon-cache ;; @@ -923,6 +937,7 @@ case $PROJECT in ubuntu-unity) add_task install minimal standard ${PROJECT}-desktop LIVE_TASK=${PROJECT}-live + MAKE_ISO=yes ;; lubuntu) @@ -997,6 +1012,8 @@ case $PROJECT in live) OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal" PASSES_TO_LAYERS=true + MAKE_ISO=yes + POOL_SEED_NAME=server-ship-live add_task ubuntu-server-minimal server-minimal add_package ubuntu-server-minimal lxd-installer add_task ubuntu-server-minimal.ubuntu-server minimal standard server @@ -1129,6 +1146,8 @@ case $PROJECT in fi OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal" PASSES_TO_LAYERS=true + MAKE_ISO=yes + POOL_SEED_NAME= add_task base server-minimal server add_task base.live server-live add_package base.live linux-image-generic @@ -1384,6 +1403,8 @@ echo "IMAGEFORMAT=\"$IMAGEFORMAT\"" >> config/chroot if [ -n "$PASSES" ]; then echo "PASSES=\"$PASSES\"" >> config/common fi +echo "MAKE_ISO=\"$MAKE_ISO\"" >> config/common +echo "POOL_SEED_NAME=\"$POOL_SEED_NAME\"" >> config/common if [ -n "$NO_SQUASHFS_PASSES" ]; then echo "NO_SQUASHFS_PASSES=\"$NO_SQUASHFS_PASSES\"" >> config/common fi @@ -1677,3 +1698,11 @@ apt-get -y download $PREINSTALL_POOL EOF fi fi + +if [ "${MAKE_ISO}" = "yes" ]; then + # XXX should pass --official here. + /usr/share/livecd-rootfs/live-build/gen-iso-ids \ + --project $PROJECT ${SUBPROJECT:+--subproject $SUBPROJECT} \ + --arch $ARCH ${SUBARCH:+--subarch $SUBARCH} ${NOW+--serial $NOW} \ + --output-dir config/iso-ids/ +fi diff --git a/live-build/functions b/live-build/functions index 20759cfc..955a3179 100644 --- a/live-build/functions +++ b/live-build/functions @@ -1444,3 +1444,10 @@ gpt_root_partition_uuid() { echo "${ROOTFS_PARTITION_TYPE}" } + +# Wrapper for the isobuild tool. Sets PYTHONPATH so the isobuilder module +# is importable, and uses config/iso-dir as the standard working directory +# for ISO metadata and intermediate files. +isobuild () { + PYTHONPATH=/usr/share/livecd-rootfs/live-build/ /usr/share/livecd-rootfs/live-build/isobuild --workdir config/iso-dir "$@" +} diff --git a/live-build/isobuild b/live-build/isobuild new file mode 100755 index 00000000..1f839d4f --- /dev/null +++ b/live-build/isobuild @@ -0,0 +1,221 @@ +#!/usr/bin/python3 + +# Building an ISO requires knowing: +# +# * The architecture and series we are building for +# * The address of the mirror to pull packages from the pool from and the +# components of that mirror to use +# * The list of packages to include in the pool +# * Where the squashfs files that contain the rootfs and other metadata layers +# are +# * Where to put the final ISO +# * All the bits of information that end up in .disk/info on the ISO and in the +# "volume ID" for the ISO +# +# It's not completely trivial to come up with a nice feeling interface between +# livecd-rootfs and this tool. There are about 13 parameters that are needed to +# build the ISO and having a tool take 13 arguments seems a bit overwhelming. In +# addition some steps need to run before the layers are made into squashfs files +# and some after. It felt nicer to have a tool with a few subcommands (7, in the +# end) and taking arguments relevant to each step: +# +# $ isobuild --work-dir "" init --disk-id "" --series "" --arch "" +# +# Set up the work-dir for later steps. Create the skeleton file layout of the +# ISO, populate .disk/info etc, create the gpg key referred to above. Store +# series and arch somewhere that later steps can refer to. +# +# $ isobuild --work-dir "" setup-apt --chroot "" +# +# Set up aptfor use by later steps, using the configuration from the passed +# chroot. +# +# $ isobuild --work-dir "" generate-pool --package-list-file "" +# +# Create the pool from the passed germinate output file. +# +# $ isobuild --work-dir "" generate-sources --mountpoint "" +# +# Generate an apt deb822 source for the pool, assuming it is mounted at the +# passed mountpoint, and output it on stdout. +# +# $ isobuild --work-dir "" add-live-filesystem --artifact-prefix "" +# +# Copy the relevant artifacts to the casper directory (and extract the uuids +# from the initrds) +# +# $ isobuild --work-dir "" make-bootable --project "" --capitalized-project "" +# --subarch "" +# +# Set up the bootloader etc so that the ISO can boot (for this clones debian-cd +# and run the tools/boot/$series-$arch script but those should be folded into +# isobuild fairly promptly IMO). +# +# $ isobuild --work-dir "" make-iso --vol-id "" --dest "" +# +# Generate the checksum file and run xorriso to build the final ISO. + + +import pathlib +import shlex + +import click + +from isobuilder.builder import ISOBuilder + + +@click.group() +@click.option( + "--workdir", + type=click.Path(file_okay=False, resolve_path=True, path_type=pathlib.Path), + required=True, + help="working directory", +) +@click.pass_context +def main(ctxt, workdir): + ctxt.obj = ISOBuilder(workdir) + cwd = pathlib.Path().cwd() + if workdir.is_relative_to(cwd): + workdir = workdir.relative_to(cwd) + ctxt.obj.logger.log(f"isobuild starting, workdir: {workdir}") + + +def subcommand(f): + """Decorator that converts a function into a Click subcommand with logging. + + This decorator: + 1. Converts function name from snake_case to kebab-case for the CLI + 2. Wraps the function to log the subcommand name and all parameters + 3. Registers it as a Click command under the main command group + 4. Extracts the ISOBuilder instance from the context and passes it as first arg + """ + name = f.__name__.replace("_", "-") + + def wrapped(ctxt, **kw): + # Build a log message showing the subcommand and all its parameters. + # We use ctxt.params (Click's resolved parameters) rather than **kw + # because ctxt.params includes path resolution and type conversion. + # Paths are converted to relative form to keep logs readable and avoid + # exposing full filesystem paths in build artifacts. + msg = f"subcommand {name}" + cwd = pathlib.Path().cwd() + for k, v in sorted(ctxt.params.items()): + if isinstance(v, pathlib.Path): + if v.is_relative_to(cwd): + v = v.relative_to(cwd) + v = shlex.quote(str(v)) + msg += f" {k}={v}" + with ctxt.obj.logger.logged(msg): + f(ctxt.obj, **kw) + + return main.command(name=name)(click.pass_context(wrapped)) + + +@click.option( + "--disk-info", + type=str, + required=True, + help="contents of .disk/info", +) +@click.option( + "--series", + type=str, + required=True, + help="series being built", +) +@click.option( + "--arch", + type=str, + required=True, + help="architecture being built", +) +@subcommand +def init(builder, disk_info, series, arch): + builder.init(disk_info, series, arch) + + +@click.option( + "--chroot", + type=click.Path( + file_okay=False, resolve_path=True, path_type=pathlib.Path, exists=True + ), + required=True, +) +@subcommand +def setup_apt(builder, chroot: pathlib.Path): + builder.setup_apt(chroot) + + +@click.pass_obj +@click.option( + "--package-list-file", + type=click.Path( + dir_okay=False, exists=True, resolve_path=True, path_type=pathlib.Path + ), + required=True, +) +@subcommand +def generate_pool(builder, package_list_file: pathlib.Path): + builder.generate_pool(package_list_file) + + +@click.option( + "--mountpoint", + type=str, + required=True, +) +@subcommand +def generate_sources(builder, mountpoint: str): + builder.generate_sources(mountpoint) + + +@click.option( + "--artifact-prefix", + type=click.Path(dir_okay=False, resolve_path=True, path_type=pathlib.Path), + required=True, +) +@subcommand +def add_live_filesystem(builder, artifact_prefix: pathlib.Path): + builder.add_live_filesystem(artifact_prefix) + + +@click.option( + "--project", + type=str, + required=True, +) +@click.option("--capproject", type=str, required=True) +@click.option( + "--subarch", + type=str, + default="", +) +@subcommand +def make_bootable(builder, project: str, capproject: str | None, subarch: str): + # capproject is the "capitalized project name" used in GRUB menu entries, + # e.g. "Ubuntu" or "Kubuntu". It should come from gen-iso-ids (which uses + # project_to_capproject_map for proper formatting like "Ubuntu-MATE"), but + # we provide a simple .capitalize() fallback for cases where the caller + # doesn't have the pre-computed value. + if capproject is None: + capproject = project.capitalize() + builder.make_bootable(project, capproject, subarch) + + +@click.option( + "--dest", + type=click.Path(dir_okay=False, resolve_path=True, path_type=pathlib.Path), + required=True, +) +@click.option( + "--volid", + type=str, + default=None, +) +@subcommand +def make_iso(builder, dest: pathlib.Path, volid: str | None): + builder.make_iso(dest, volid) + + +if __name__ == "__main__": + main() diff --git a/live-build/isobuilder/__init__.py b/live-build/isobuilder/__init__.py new file mode 100644 index 00000000..792d6005 --- /dev/null +++ b/live-build/isobuilder/__init__.py @@ -0,0 +1 @@ +# diff --git a/live-build/isobuilder/apt_state.py b/live-build/isobuilder/apt_state.py new file mode 100644 index 00000000..4fde19e2 --- /dev/null +++ b/live-build/isobuilder/apt_state.py @@ -0,0 +1,110 @@ +import dataclasses +import os +import pathlib +import shutil +import subprocess +from typing import Iterator + + +@dataclasses.dataclass +class PackageInfo: + package: str + filename: str + architecture: str + version: str + + @property + def spec(self) -> str: + return f"{self.package}:{self.architecture}={self.version}" + + +def check_proc(proc, ok_codes=(0,)) -> None: + proc.wait() + if proc.returncode not in ok_codes: + raise Exception(f"{proc} failed") + + +class AptStateManager: + """Maintain and use an apt state directory to access package info and debs.""" + + def __init__(self, logger, series: str, apt_dir: pathlib.Path): + self.logger = logger + self.series = series + self.apt_root = apt_dir.joinpath("root") + self.apt_conf_path = apt_dir.joinpath("apt.conf") + + def _apt_env(self) -> dict[str, str]: + return dict(os.environ, APT_CONFIG=str(self.apt_conf_path)) + + def setup(self, chroot: pathlib.Path): + """Set up the manager by copying the apt configuration from `chroot`.""" + for path in "etc/apt", "var/lib/apt": + tgt = self.apt_root.joinpath(path) + tgt.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(chroot.joinpath(path), tgt) + self.apt_conf_path.write_text(f'Dir "{self.apt_root}/"; \n') + with self.logger.logged("updating apt indices"): + self.logger.run(["apt-get", "update"], env=self._apt_env()) + + def show(self, pkgs: list[str]) -> Iterator[PackageInfo]: + """Return information about the binary packages named by `pkgs`. + + Parses apt-cache output, which uses RFC822-like format: field names + followed by ": " and values, with multi-line values indented with + leading whitespace. We skip continuation lines (starting with space) + since PackageInfo only needs single-line fields. + + The `fields` set (derived from PackageInfo's dataclass fields) acts as + a filter - we only extract fields we care about, ignoring others like + Description. This provides some forward-compatibility if apt-cache + output format changes. + """ + proc = subprocess.Popen( + ["apt-cache", "-o", "APT::Cache::AllVersions=0", "show"] + pkgs, + stdout=subprocess.PIPE, + encoding="utf-8", + env=self._apt_env(), + ) + assert proc.stdout is not None + fields = {f.name for f in dataclasses.fields(PackageInfo)} + params: dict[str, str] = {} + for line in proc.stdout: + if line == "\n": + yield PackageInfo(**params) + params = {} + continue + if line.startswith(" "): + continue + field, value = line.split(": ", 1) + field = field.lower() + if field in fields: + params[field] = value.strip() + check_proc(proc) + if params: + yield PackageInfo(**params) + + def download(self, rootdir: pathlib.Path, pkg_info: PackageInfo): + """Download the package specified by `pkg_info` under `rootdir`. + + The package is saved to the same path under `rootdir` as it is + at in the archive it comes from. + """ + target_dir = rootdir.joinpath(pkg_info.filename).parent + target_dir.mkdir(parents=True, exist_ok=True) + self.logger.run( + ["apt-get", "download", pkg_info.spec], + cwd=target_dir, + check=True, + env=self._apt_env(), + ) + + def in_release_path(self) -> pathlib.Path: + """Return the path to the InRelease file. + + This assumes exactly one InRelease file matches the pattern. + Will raise ValueError if there are 0 or multiple matches. + """ + [path] = self.apt_root.joinpath("var/lib/apt/lists").glob( + f"*_dists_{self.series}_InRelease" + ) + return path diff --git a/live-build/isobuilder/builder.py b/live-build/isobuilder/builder.py new file mode 100644 index 00000000..53149643 --- /dev/null +++ b/live-build/isobuilder/builder.py @@ -0,0 +1,365 @@ +import contextlib +import json +import os +import pathlib +import shlex +import shutil +import subprocess +import sys + +from isobuilder.apt_state import AptStateManager +from isobuilder.gpg_key import EphemeralGPGKey +from isobuilder.pool_builder import PoolBuilder + +# Constants +PACKAGE_BATCH_SIZE = 200 +MAX_CMD_DISPLAY_LENGTH = 80 + + +def package_list_packages(package_list_file: pathlib.Path) -> list[str]: + # Parse germinate output to extract package names. Germinate is Ubuntu's + # package dependency resolver that outputs dependency trees for seeds (like + # "ship-live" or "server-ship-live"). + # + # Germinate output format has 2 header lines at the start and 2 footer lines + # at the end (showing statistics), so we skip them with [2:-2]. + # Each data line starts with the package name followed by whitespace and + # dependency info. This format is stable but if germinate ever changes its + # header/footer count, this will break silently. + lines = package_list_file.read_text().splitlines()[2:-2] + return [line.split(None, 1)[0] for line in lines] + + +def make_sources_text( + series: str, gpg_key: EphemeralGPGKey, components: list[str], mountpoint: str +) -> str: + """Generate a deb822-format apt source file for the ISO's package pool. + + deb822 is the modern apt sources format (see sources.list(5) and deb822(5)). + It uses RFC822-style fields where multi-line values must be indented with a + leading space, and empty lines within a value are represented as " ." + (space-dot). This format is required for inline GPG keys in the Signed-By + field. + """ + key = gpg_key.export_public() + quoted_key = [] + for line in key.splitlines(): + if not line: + quoted_key.append(" .") + else: + quoted_key.append(" " + line) + return f"""\ +Types: deb +URIs: file://{mountpoint} +Suites: {series} +Components: {" ".join(components)} +Check-Date: no +Signed-By: +""" + "\n".join( + quoted_key + ) + + +class Logger: + + def __init__(self): + self._indent = "" + + def log(self, msg): + print(self._indent + msg, file=sys.stderr) + + @contextlib.contextmanager + def logged(self, msg, done_msg=None): + self.log(msg) + self._indent += " " + try: + yield + finally: + self._indent = self._indent[:-2] + if done_msg is not None: + self.log(done_msg) + + def msg_for_cmd(self, cmd, cwd=None) -> str: + if cwd is None: + _cwd = pathlib.Path().cwd() + else: + _cwd = cwd + fmted_cmd = [] + for arg in cmd: + if isinstance(arg, pathlib.Path): + if arg.is_relative_to(_cwd): + arg = arg.relative_to(_cwd) + arg = str(arg) + fmted_cmd.append(shlex.quote(arg)) + fmted_cmd_str = " ".join(fmted_cmd) + if len(fmted_cmd_str) > MAX_CMD_DISPLAY_LENGTH: + fmted_cmd_str = fmted_cmd_str[:MAX_CMD_DISPLAY_LENGTH] + "..." + msg = f"running `{fmted_cmd_str}`" + if cwd is not None: + msg += f" in {cwd}" + return msg + + def run(self, cmd: list[str | pathlib.Path], *args, check=True, **kw): + with self.logged(self.msg_for_cmd(cmd, kw.get("cwd"))): + return subprocess.run(cmd, *args, check=check, **kw) + + +class ISOBuilder: + + def __init__(self, workdir: pathlib.Path): + self.workdir = workdir + self.logger = Logger() + self.iso_root = workdir.joinpath("iso-root") + self._series = self._arch = self._gpg_key = self._apt_state = None + + # UTILITY STUFF + + def _read_config(self): + with self.workdir.joinpath("config.json").open() as fp: + data = json.load(fp) + self._series = data["series"] + self._arch = data["arch"] + + @property + def arch(self): + if self._arch is None: + self._read_config() + return self._arch + + @property + def series(self): + if self._series is None: + self._read_config() + return self._series + + @property + def gpg_key(self): + if self._gpg_key is None: + self._gpg_key = EphemeralGPGKey( + self.logger, self.workdir.joinpath("gpg-home") + ) + return self._gpg_key + + @property + def apt_state(self): + if self._apt_state is None: + self._apt_state = AptStateManager( + self.logger, self.series, self.workdir.joinpath("apt-state") + ) + return self._apt_state + + # COMMANDS + + def init(self, disk_info: str, series: str, arch: str): + self.logger.log("creating directories") + self.workdir.mkdir(exist_ok=True) + self.iso_root.mkdir() + dot_disk = self.iso_root.joinpath(".disk") + dot_disk.mkdir() + + self.logger.log("saving config") + with self.workdir.joinpath("config.json").open("w") as fp: + json.dump({"arch": arch, "series": series}, fp) + + self.logger.log("populating .disk") + dot_disk.joinpath("base_installable").touch() + dot_disk.joinpath("cd_type").write_text("full_cd/single\n") + dot_disk.joinpath("info").write_text(disk_info) + self.iso_root.joinpath("casper").mkdir() + + self.gpg_key.create() + + def setup_apt(self, chroot: pathlib.Path): + self.apt_state.setup(chroot) + + def generate_pool(self, package_list_file: pathlib.Path): + # do we need any of the symlinks we create here?? + self.logger.log("creating pool skeleton") + self.iso_root.joinpath("ubuntu").symlink_to(".") + if self.arch not in ("amd64", "i386"): + self.iso_root.joinpath("ubuntu-ports").symlink_to(".") + self.iso_root.joinpath("dists", self.series).mkdir(parents=True) + + builder = PoolBuilder( + self.logger, + series=self.series, + rootdir=self.iso_root, + apt_state=self.apt_state, + ) + pkgs = package_list_packages(package_list_file) + # XXX include 32-bit deps of 32-bit packages if needed here + with self.logger.logged("adding packages"): + for i in range(0, len(pkgs), PACKAGE_BATCH_SIZE): + builder.add_packages( + self.apt_state.show(pkgs[i : i + PACKAGE_BATCH_SIZE]) + ) + builder.make_packages() + release_file = builder.make_release() + self.gpg_key.sign(release_file) + for name in "stable", "unstable": + self.iso_root.joinpath("dists", name).symlink_to(self.series) + + def generate_sources(self, mountpoint: str): + components = [p.name for p in self.iso_root.joinpath("pool").iterdir()] + print( + make_sources_text( + self.series, self.gpg_key, mountpoint=mountpoint, components=components + ) + ) + + def _extract_casper_uuids(self): + # Extract UUID files from initrd images for casper (the live boot system). + # Each initrd contains a conf/uuid.conf with a unique identifier that + # casper uses at boot time to locate the correct root filesystem. These + # UUIDs must be placed in .disk/casper-uuid- on the ISO so casper + # can verify it's booting from the right media. + with self.logger.logged("extracting casper uuids"): + casper_dir = self.iso_root.joinpath("casper") + prefix = "filesystem.initrd-" + dot_disk = self.iso_root.joinpath(".disk") + for initrd in casper_dir.glob(f"{prefix}*"): + initrddir = self.workdir.joinpath("initrd") + with self.logger.logged( + f"unpacking {initrd.name} ...", done_msg="... done" + ): + self.logger.run(["unmkinitramfs", initrd, initrddir]) + # unmkinitramfs can produce different directory structures: + # - Platforms with early firmware: subdirs like "main/" or "early/" + # containing conf/uuid.conf + # - Other platforms: conf/uuid.conf directly in the root + # Try to find uuid.conf in both locations. The [uuid_conf] = confs + # unpacking asserts exactly one match; multiple matches would + # indicate an unexpected initrd structure. + confs = list(initrddir.glob("*/conf/uuid.conf")) + if confs: + [uuid_conf] = confs + elif initrddir.joinpath("conf/uuid.conf").exists(): + uuid_conf = initrddir.joinpath("conf/uuid.conf") + else: + raise Exception("uuid.conf not found") + self.logger.log(f"found {uuid_conf.relative_to(initrddir)}") + uuid_conf.rename( + dot_disk.joinpath("casper-uuid-" + initrd.name[len(prefix) :]) + ) + shutil.rmtree(initrddir) + + def add_live_filesystem(self, artifact_prefix: pathlib.Path): + # Link build artifacts into the ISO's casper directory. We use hardlinks + # (not copies) for filesystem efficiency - they reference the same inode. + # + # Artifacts come from the layered build with names like "for-iso.base.squashfs" + # and need to be renamed for casper. The prefix is stripped, so: + # for-iso.base.squashfs -> base.squashfs + # for-iso.kernel-generic -> filesystem.kernel-generic + # + # Kernel and initrd get the extra "filesystem." prefix because casper + # expects names like filesystem.kernel-* and filesystem.initrd-*. + casper_dir = self.iso_root.joinpath("casper") + artifact_dir = artifact_prefix.parent + filename_prefix = artifact_prefix.name + + def link(src, target_name): + target = casper_dir.joinpath(target_name) + self.logger.log( + f"creating link from $ISOROOT/casper/{target_name} to $src/{src.name}" + ) + target.hardlink_to(src) + + with self.logger.logged( + f"linking artifacts from {casper_dir} to {artifact_dir}" + ): + for ext in "squashfs", "squashfs.gpg", "size", "manifest", "yaml": + for path in artifact_dir.glob(f"{filename_prefix}*.{ext}"): + newname = path.name[len(filename_prefix) :] + link(path, newname) + for item in "kernel", "initrd": + for path in artifact_dir.glob(f"{filename_prefix}{item}-*"): + newname = "filesystem." + path.name[len(filename_prefix) :] + link(path, newname) + self._extract_casper_uuids() + + def make_bootable(self, project: str, capproject: str, subarch: str): + # debian-cd is Ubuntu's CD/ISO image build system. It contains + # architecture and series-specific boot configuration scripts that set up + # GRUB, syslinux, EFI boot, etc. The tools/boot/$series/boot-$arch script + # knows how to make an ISO bootable for each architecture. + # + # TODO: The boot configuration logic should eventually be ported directly + # into isobuilder to avoid this external dependency and git clone. + debian_cd_dir = self.workdir.joinpath("debian-cd") + with self.logger.logged("cloning debian-cd"): + self.logger.run( + [ + "git", + "clone", + "--depth=1", + "https://git.launchpad.net/~ubuntu-cdimage/debian-cd/+git/ubuntu", + debian_cd_dir, + ], + ) + # Override apt-selection to use our ISO's apt configuration instead of + # debian-cd's default. This ensures the boot scripts query packages from + # the correct repository (our pool) when determining boot requirements. + apt_selection = debian_cd_dir.joinpath("tools/apt-selection") + with self.logger.logged("overwriting apt-selection"): + apt_selection.write_text( + "#!/bin/sh\n" f"APT_CONFIG={self.apt_state.apt_conf_path} apt-get $@\n" + ) + env = dict( + os.environ, + BASEDIR=str(debian_cd_dir), + DIST=self.series, + PROJECT=project, + CAPPROJECT=capproject, + SUBARCH=subarch, + ) + tool_name = f"tools/boot/{self.series}/boot-{self.arch}" + with self.logger.logged(f"running {tool_name} ...", done_msg="... done"): + self.logger.run( + [ + debian_cd_dir.joinpath(tool_name), + "1", + self.iso_root, + ], + env=env, + ) + + def checksum(self): + # Generate md5sum.txt for ISO integrity verification. + # - Symlinks are excluded because their targets are already checksummed + # - Files are sorted for deterministic, reproducible output across builds + # - Paths use "./" prefix and we run md5sum from iso_root so the output + # matches what users get when they verify with "md5sum -c" from the ISO + all_files = [] + for dirpath, dirnames, filenames in self.iso_root.walk(): + filepaths = [dirpath.joinpath(filename) for filename in filenames] + all_files.extend( + "./" + str(filepath.relative_to(self.iso_root)) + for filepath in filepaths + if not filepath.is_symlink() + ) + self.iso_root.joinpath("md5sum.txt").write_bytes( + self.logger.run( + ["md5sum"] + sorted(all_files), + cwd=self.iso_root, + stdout=subprocess.PIPE, + ).stdout + ) + + def make_iso(self, dest: pathlib.Path, volid: str | None): + # 1.mkisofs_opts is generated by debian-cd's make_bootable step. The "1" + # refers to "pass 1" of the build (a legacy naming convention). It contains + # architecture-specific xorriso options for boot sectors, EFI images, etc. + mkisofs_opts = shlex.split(self.workdir.joinpath("1.mkisofs_opts").read_text()) + self.checksum() + # xorriso with "-as mkisofs" runs in mkisofs compatibility mode. + # -r enables Rock Ridge extensions for Unix metadata (permissions, symlinks). + # -iso-level 3 (amd64 only) allows files >4GB which some amd64 ISOs need. + cmd: list[str | pathlib.Path] = ["xorriso", "-as", "mkisofs", "-r"] + if self.arch == "amd64": + cmd.extend(["-iso-level", "3"]) + if volid: + cmd += ["-V", volid] + cmd += mkisofs_opts + [self.iso_root, "-o", dest] + with self.logger.logged("running xorriso"): + self.logger.run(cmd, cwd=self.workdir, check=True) diff --git a/live-build/isobuilder/gpg_key.py b/live-build/isobuilder/gpg_key.py new file mode 100644 index 00000000..d35e8df6 --- /dev/null +++ b/live-build/isobuilder/gpg_key.py @@ -0,0 +1,58 @@ +import pathlib +import subprocess + +key_conf = """\ +%no-protection +Key-Type: eddsa +Key-Curve: Ed25519 +Key-Usage: sign +Name-Real: Ubuntu ISO One-Time Signing Key +Name-Email: noone@nowhere.invalid +Expire-Date: 0 +""" + + +class EphemeralGPGKey: + + def __init__(self, logger, gpghome): + self.logger = logger + self.gpghome = gpghome + + def _run_gpg(self, cmd, **kwargs): + return self.logger.run( + ["gpg", "--homedir", self.gpghome] + cmd, check=True, **kwargs + ) + + def create(self): + with self.logger.logged("creating gpg key ...", done_msg="... done"): + self.gpghome.mkdir(mode=0o700) + self._run_gpg( + ["--gen-key", "--batch"], + input=key_conf, + text=True, + ) + + def sign(self, path: pathlib.Path): + with self.logger.logged(f"signing {path}"): + with path.open("rb") as inp: + with pathlib.Path(str(path) + ".gpg").open("wb") as outp: + self._run_gpg( + [ + "--no-options", + "--batch", + "--no-tty", + "--armour", + "--digest-algo", + "SHA512", + "--detach-sign", + ], + stdin=inp, + stdout=outp, + ) + + def export_public(self) -> str: + return self._run_gpg( + ["--export", "--armor"], + stdout=subprocess.PIPE, + text=True, + ).stdout diff --git a/live-build/isobuilder/pool_builder.py b/live-build/isobuilder/pool_builder.py new file mode 100644 index 00000000..ce6815ff --- /dev/null +++ b/live-build/isobuilder/pool_builder.py @@ -0,0 +1,166 @@ +import pathlib +import subprocess +import tempfile + +from isobuilder.apt_state import AptStateManager, PackageInfo + + +generate_template = """ +Dir::ArchiveDir "{root}"; +Dir::CacheDir "{scratch}/apt-ftparchive-db"; + +TreeDefault::Contents " "; + +Tree "dists/{series}" {{ + FileList "{scratch}/filelist_$(SECTION)"; + Sections "{components}"; + Architectures "{arches}"; +}} +""" + + +class PoolBuilder: + + def __init__( + self, logger, series: str, apt_state: AptStateManager, rootdir: pathlib.Path + ): + self.logger = logger + self.series = series + self.apt_state = apt_state + self.rootdir = rootdir + self.arches: set[str] = set() + self._present_components: set[str] = set() + + def add_packages(self, pkglist: list[PackageInfo]): + for pkg_info in pkglist: + if pkg_info.architecture != "all": + self.arches.add(pkg_info.architecture) + self.apt_state.download(self.rootdir, pkg_info) + + def make_packages(self) -> None: + with self.logger.logged("making Packages files"): + with tempfile.TemporaryDirectory() as tmpdir: + scratchdir = pathlib.Path(tmpdir) + with self.logger.logged("scanning for packages"): + for component in ["main", "restricted", "universe", "multiverse"]: + if not self.rootdir.joinpath("pool", component).is_dir(): + continue + self._present_components.add(component) + for arch in self.arches: + self.rootdir.joinpath( + "dists", self.series, component, f"binary-{arch}" + ).mkdir(parents=True) + proc = self.logger.run( + ["find", f"pool/{component}"], + stdout=subprocess.PIPE, + cwd=self.rootdir, + encoding="utf-8", + check=True, + ) + scratchdir.joinpath(f"filelist_{component}").write_text( + "\n".join(sorted(proc.stdout.splitlines())) + ) + with self.logger.logged("writing apt-ftparchive config"): + scratchdir.joinpath("apt-ftparchive-db").mkdir() + generate_path = scratchdir.joinpath("generate-binary") + generate_path.write_text( + generate_template.format( + arches=" ".join(self.arches), + series=self.series, + root=self.rootdir.resolve(), + scratch=scratchdir.resolve(), + components=" ".join(self._present_components), + ) + ) + with self.logger.logged("running apt-ftparchive generate"): + self.logger.run( + [ + "apt-ftparchive", + "--no-contents", + "--no-md5", + "--no-sha1", + "--no-sha512", + "generate", + generate_path, + ], + check=True, + ) + + def make_release(self) -> pathlib.Path: + # Build the Release file by merging metadata from the mirror with + # checksums for our pool. We can't just use apt-ftparchive's Release + # output directly because: + # 1. apt-ftparchive doesn't know about Origin, Label, Suite, Version, + # Codename, etc. - these come from the mirror and maintain package + # provenance + # 2. We keep the mirror's Date (when packages were released) rather than + # apt-ftparchive's Date (when we ran the command) + # 3. We need to override Architectures/Components to match our pool + # + # There may be a cleaner way (apt-get indextargets?) but this works. + with self.logger.logged("making Release file"): + in_release = self.apt_state.in_release_path() + cp_mirror_release = self.logger.run( + ["gpg", "--verify", "--output", "-", in_release], + stdout=subprocess.PIPE, + encoding="utf-8", + check=False, + ) + if cp_mirror_release.returncode not in (0, 2): + # gpg returns code 2 when the public key the InRelease is + # signed with is not available, which is most of the time. + raise Exception("gpg failed") + mirror_release_lines = cp_mirror_release.stdout.splitlines() + release_dir = self.rootdir.joinpath("dists", self.series) + af_release_lines = self.logger.run( + [ + "apt-ftparchive", + "--no-contents", + "--no-md5", + "--no-sha1", + "--no-sha512", + "release", + ".", + ], + stdout=subprocess.PIPE, + encoding="utf-8", + cwd=release_dir, + check=True, + ).stdout.splitlines() + # Build the final Release file by merging mirror metadata with pool + # checksums. + # Strategy: + # 1. Take metadata fields (Suite, Origin, etc.) from the mirror's InRelease + # 2. Override Architectures and Components to match what's actually in our + # pool + # 3. Skip the mirror's checksum sections (MD5Sum, SHA256, etc.) because they + # don't apply to our pool + # 4. Skip Acquire-By-Hash since we don't use it + # 5. Append checksums from apt-ftparchive (but not the Date field) + release_lines = [] + skipping = False + for line in mirror_release_lines: + if line.startswith("Architectures:"): + line = "Architectures: " + " ".join(sorted(self.arches)) + elif line.startswith("Components:"): + line = "Components: " + " ".join(sorted(self._present_components)) + elif line.startswith("MD5") or line.startswith("SHA"): + # Start of a checksum section - skip this and indented lines below + # it + skipping = True + elif not line.startswith(" "): + # Non-indented line means we've left the checksum section if we were + # in one. + skipping = False + if line.startswith("Acquire-By-Hash"): + continue + if not skipping: + release_lines.append(line) + # Append checksums from apt-ftparchive, but skip its Date field + # (we want to keep the Date from the mirror release) + for line in af_release_lines: + if not line.startswith("Date"): + release_lines.append(line) + release_path = release_dir.joinpath("Release") + release_path.write_text("\n".join(release_lines)) + return release_path diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index f12e3512..efcbfdb7 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -184,6 +184,20 @@ build_layered_squashfs () { fi create_squashfs "${overlay_dir}" ${squashfs_f} + # Create a "for-iso" variant of the squashfs for ISO builds. For + # the root layer (the base system) when building with a pool, we + # need to include cdrom.sources so casper can access the ISO's + # package repository. This requires regenerating the squashfs with + # that file included, then removing it (so it doesn't pollute the + # regular squashfs). Non-root layers (desktop environment, etc.) + # and builds without pools can just hardlink to the regular squashfs. + if [ -n "${POOL_SEED_NAME}" ] && $(is_root_layer $pass); then + isobuild generate-sources --mountpoint=/cdrom > ${overlay_dir}/etc/apt/sources.list.d/cdrom.sources + create_squashfs "${overlay_dir}" ${PWD}/for-iso.${pass}.squashfs + rm ${overlay_dir}/etc/apt/sources.list.d/cdrom.sources + else + ln ${squashfs_f} ${PWD}/for-iso.${pass}.squashfs + fi if [ -f config/$pass.catalog-in.yaml ]; then echo "Expanding catalog entry template for $pass"