Add isobuild tool to build installer ISOs

This adds a new tool, isobuild, which replaces the ISO-building
functionality previously provided by live-build and cdimage. It is
invoked from auto/build when MAKE_ISO=yes.

The tool supports:
 - Layered desktop images (Ubuntu Desktop, flavors)
 - Non-layered images (Kubuntu, Ubuntu Unity)
 - Images with package pools (most installers)
 - Images without pools (Ubuntu Core Installer)

The isobuild command has several subcommands:
 - init: Initialize the ISO build directory structure
 - setup-apt: Configure APT for package pool generation
 - generate-pool: Create the package pool from a seed
 - generate-sources: Generate cdrom.sources for the installed system
 - add-live-filesystem: Add squashfs and kernel/initrd to the ISO
 - make-bootable: Add GRUB and other boot infrastructure
 - make-iso: Generate the final ISO image

auto/config is updated to:
 - Set MAKE_ISO=yes for relevant image types
 - Set POOL_SEED_NAME for images that need a package pool
 - Invoke gen-iso-ids to compute ISO metadata

auto/build is updated to:
 - Remove old live-build ISO handling code
 - Invoke isobuild at appropriate points in the build

lb_binary_layered is updated to create squashfs files with
cdrom.sources included for use in the ISO.

live-build/auto/build

@@ -208,6 +208,22 @@ EOF
 		undivert_update_initramfs
 		undivert_grub chroot
 	fi
+	if [ "${MAKE_ISO}" = yes ]; then
+		isobuild init --disk-info "$(cat config/iso-ids/disk-info)" --series "${LB_DISTRIBUTION}" --arch "${ARCH}"
+		# Determine which chroot directory has the apt configuration to use.
+		# Layered builds (PASSES set) create overlay directories named
+		# "overlay.base", "overlay.live", etc. - we use the first one (base).
+		# Single-pass builds use the "chroot" directory directly.
+		if [ "${PASSES}" ]; then
+			CHROOT="overlay.$(set -- $PASSES; echo $1)"
+		else
+			CHROOT=chroot
+		fi
+		isobuild setup-apt --chroot $CHROOT
+		if [ -n "${POOL_SEED_NAME}" ]; then
+			isobuild generate-pool --package-list-file "config/germinate-output/${POOL_SEED_NAME}"
+		fi
+	fi
 
 	if [ -d chroot/etc/apt/preferences.d.save ]; then
 		# https://mastodon.social/@scream@botsin.space
@@ -427,13 +443,6 @@ if [ -e config/manifest-minimal-remove ]; then
         cp config/manifest-minimal-remove  "$PREFIX.manifest-minimal-remove"
 fi
 
-for ISO in binary.iso binary.hybrid.iso; do
-	[ -e "$ISO" ] || continue
-	ln "$ISO" "$PREFIX.iso"
-	chmod 644 "$PREFIX.iso"
-	break
-done
-
 if [ -e "binary/$INITFS/filesystem.dir" ]; then
 	(cd "binary/$INITFS/filesystem.dir/" && tar -c --sort=name --xattrs *) | \
 		gzip -9 --rsyncable > "$PREFIX.rootfs.tar.gz"
@@ -558,3 +567,23 @@ case $PROJECT in
     ubuntu-cpc)
         config/hooks.d/remove-implicit-artifacts
 esac
+
+if [ "${MAKE_ISO}" = "yes" ]; then
+	# Link build artifacts with "for-iso." prefix for isobuild to consume.
+	# Layered builds create squashfs via lb_binary_layered (which already
+	# creates for-iso.*.squashfs files). Single-pass builds only have
+	# ${PREFIX}.squashfs, so we link it here.
+	if [ -z "$PASSES" ]; then
+		ln ${PREFIX}.squashfs for-iso.filesystem.squashfs
+	fi
+	# Link kernel and initrd files. The ${thing#${PREFIX}} expansion strips
+	# the PREFIX, so "livecd.ubuntu-server.kernel-generic" becomes
+	# "for-iso.kernel-generic".
+	for thing in ${PREFIX}.kernel-* ${PREFIX}.initrd-*; do
+		ln $thing for-iso${thing#${PREFIX}}
+	done
+	isobuild add-live-filesystem --artifact-prefix for-iso.
+	isobuild make-bootable --project "${PROJECT}" --capproject "$(cat config/iso-ids/capproject)" \
+		${SUBARCH:+--subarch "${SUBARCH}"}
+	isobuild make-iso --volid "$(cat config/iso-ids/vol-id)" --dest ${PREFIX}.iso
+fi

live-build/auto/config

@@ -663,10 +663,23 @@ if ! [ -e config/germinate-output/structure ]; then
                 -s $FLAVOUR.$SUITE $GERMINATE_ARG -a ${ARCH_VARIANT:-$ARCH})
 fi
 
+# ISO build configuration. These defaults are overridden per-project below.
+#
+# MAKE_ISO: Set to "yes" to generate an installer ISO at the end of the build.
+# This triggers isobuild to run in auto/build.
+MAKE_ISO=no
+# POOL_SEED_NAME: The germinate output file defining packages for the ISO's
+# package pool (repository). Different flavors use different seeds:
+# - "ship-live" for most desktop images
+# - "server-ship-live" for Ubuntu Server (includes server-specific packages)
+# - "" (empty) for images without a pool, like Ubuntu Core Installer
+POOL_SEED_NAME=ship-live
+
 # Common functionality for layered desktop images
 common_layered_desktop_image() {
 	touch config/universe-enabled
 	PASSES_TO_LAYERS="true"
+	MAKE_ISO=yes
 
 	if [ -n "$HAS_MINIMAL" ]; then
 		if [ -z "$MINIMAL_TASKS" ]; then
@@ -897,6 +910,7 @@ case $PROJECT in
 		add_task install minimal standard
 		add_task install kubuntu-desktop
 		LIVE_TASK='kubuntu-live'
+		MAKE_ISO=yes
 		add_chroot_hook remove-gnome-icon-cache
 		;;
 
@@ -923,6 +937,7 @@ case $PROJECT in
 	ubuntu-unity)
 		add_task install minimal standard ${PROJECT}-desktop
 		LIVE_TASK=${PROJECT}-live
+		MAKE_ISO=yes
 		;;
 
 	lubuntu)
@@ -997,6 +1012,8 @@ case $PROJECT in
 			live)
 				OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal"
 				PASSES_TO_LAYERS=true
+				MAKE_ISO=yes
+				POOL_SEED_NAME=server-ship-live
 				add_task ubuntu-server-minimal server-minimal
 				add_package ubuntu-server-minimal lxd-installer
 				add_task ubuntu-server-minimal.ubuntu-server minimal standard server
@@ -1129,6 +1146,8 @@ case $PROJECT in
 	    fi
 	    OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal"
 	    PASSES_TO_LAYERS=true
+	    MAKE_ISO=yes
+	    POOL_SEED_NAME=
 	    add_task base server-minimal server
 	    add_task base.live server-live
 	    add_package base.live linux-image-generic
@@ -1384,6 +1403,8 @@ echo "IMAGEFORMAT=\"$IMAGEFORMAT\"" >> config/chroot
 if [ -n "$PASSES" ]; then
 	echo "PASSES=\"$PASSES\"" >> config/common
 fi
+echo "MAKE_ISO=\"$MAKE_ISO\"" >> config/common
+echo "POOL_SEED_NAME=\"$POOL_SEED_NAME\"" >> config/common
 if [ -n "$NO_SQUASHFS_PASSES" ]; then
     echo "NO_SQUASHFS_PASSES=\"$NO_SQUASHFS_PASSES\"" >> config/common
 fi
@@ -1677,3 +1698,11 @@ apt-get -y download $PREINSTALL_POOL
 EOF
 	fi
 fi
+
+if [ "${MAKE_ISO}" = "yes" ]; then
+	# XXX should pass --official here.
+	/usr/share/livecd-rootfs/live-build/gen-iso-ids \
+		--project $PROJECT ${SUBPROJECT:+--subproject $SUBPROJECT} \
+		--arch $ARCH ${SUBARCH:+--subarch $SUBARCH} ${NOW+--serial $NOW} \
+		--output-dir config/iso-ids/
+fi

live-build/functions

@@ -1444,3 +1444,10 @@ gpt_root_partition_uuid() {
 
     echo "${ROOTFS_PARTITION_TYPE}"
 }
+
+# Wrapper for the isobuild tool. Sets PYTHONPATH so the isobuilder module
+# is importable, and uses config/iso-dir as the standard working directory
+# for ISO metadata and intermediate files.
+isobuild () {
+    PYTHONPATH=/usr/share/livecd-rootfs/live-build/ /usr/share/livecd-rootfs/live-build/isobuild --workdir config/iso-dir "$@"
+}

live-build/isobuild

@@ -0,0 +1,221 @@
+#!/usr/bin/python3
+
+# Building an ISO requires knowing:
+#
+# * The architecture and series we are building for
+# * The address of the mirror to pull packages from the pool from and the
+#   components of that mirror to use
+# * The list of packages to include in the pool
+# * Where the squashfs files that contain the rootfs and other metadata layers
+#   are
+# * Where to put the final ISO
+# * All the bits of information that end up in .disk/info on the ISO and in the
+#   "volume ID" for the ISO
+#
+# It's not completely trivial to come up with a nice feeling interface between
+# livecd-rootfs and this tool. There are about 13 parameters that are needed to
+# build the ISO and having a tool take 13 arguments seems a bit overwhelming. In
+# addition some steps need to run before the layers are made into squashfs files
+# and some after. It felt nicer to have a tool with a few subcommands (7, in the
+# end) and taking arguments relevant to each step:
+#
+# $ isobuild --work-dir "" init --disk-id "" --series "" --arch ""
+#
+# Set up the work-dir for later steps. Create the skeleton file layout of the
+# ISO, populate .disk/info etc, create the gpg key referred to above. Store
+# series and arch somewhere that later steps can refer to.
+#
+# $ isobuild --work-dir "" setup-apt --chroot ""
+#
+# Set up aptfor use by later steps, using the configuration from the passed
+# chroot.
+#
+# $ isobuild --work-dir "" generate-pool --package-list-file ""
+#
+# Create the pool from the passed germinate output file.
+#
+# $ isobuild --work-dir "" generate-sources --mountpoint ""
+#
+# Generate an apt deb822 source for the pool, assuming it is mounted at the
+# passed mountpoint, and output it on stdout.
+#
+# $ isobuild --work-dir "" add-live-filesystem --artifact-prefix ""
+#
+# Copy the relevant artifacts to the casper directory (and extract the uuids
+# from the initrds)
+#
+# $ isobuild --work-dir "" make-bootable --project "" --capitalized-project ""
+#            --subarch ""
+#
+# Set up the bootloader etc so that the ISO can boot (for this clones debian-cd
+# and run the tools/boot/$series-$arch script but those should be folded into
+# isobuild fairly promptly IMO).
+#
+# $ isobuild --work-dir "" make-iso --vol-id "" --dest ""
+#
+# Generate the checksum file and run xorriso to build the final ISO.
+
+
+import pathlib
+import shlex
+
+import click
+
+from isobuilder.builder import ISOBuilder
+
+
+@click.group()
+@click.option(
+    "--workdir",
+    type=click.Path(file_okay=False, resolve_path=True, path_type=pathlib.Path),
+    required=True,
+    help="working directory",
+)
+@click.pass_context
+def main(ctxt, workdir):
+    ctxt.obj = ISOBuilder(workdir)
+    cwd = pathlib.Path().cwd()
+    if workdir.is_relative_to(cwd):
+        workdir = workdir.relative_to(cwd)
+    ctxt.obj.logger.log(f"isobuild starting, workdir: {workdir}")
+
+
+def subcommand(f):
+    """Decorator that converts a function into a Click subcommand with logging.
+
+    This decorator:
+    1. Converts function name from snake_case to kebab-case for the CLI
+    2. Wraps the function to log the subcommand name and all parameters
+    3. Registers it as a Click command under the main command group
+    4. Extracts the ISOBuilder instance from the context and passes it as first arg
+    """
+    name = f.__name__.replace("_", "-")
+
+    def wrapped(ctxt, **kw):
+        # Build a log message showing the subcommand and all its parameters.
+        # We use ctxt.params (Click's resolved parameters) rather than **kw
+        # because ctxt.params includes path resolution and type conversion.
+        # Paths are converted to relative form to keep logs readable and avoid
+        # exposing full filesystem paths in build artifacts.
+        msg = f"subcommand {name}"
+        cwd = pathlib.Path().cwd()
+        for k, v in sorted(ctxt.params.items()):
+            if isinstance(v, pathlib.Path):
+                if v.is_relative_to(cwd):
+                    v = v.relative_to(cwd)
+            v = shlex.quote(str(v))
+            msg += f" {k}={v}"
+        with ctxt.obj.logger.logged(msg):
+            f(ctxt.obj, **kw)
+
+    return main.command(name=name)(click.pass_context(wrapped))
+
+
+@click.option(
+    "--disk-info",
+    type=str,
+    required=True,
+    help="contents of .disk/info",
+)
+@click.option(
+    "--series",
+    type=str,
+    required=True,
+    help="series being built",
+)
+@click.option(
+    "--arch",
+    type=str,
+    required=True,
+    help="architecture being built",
+)
+@subcommand
+def init(builder, disk_info, series, arch):
+    builder.init(disk_info, series, arch)
+
+
+@click.option(
+    "--chroot",
+    type=click.Path(
+        file_okay=False, resolve_path=True, path_type=pathlib.Path, exists=True
+    ),
+    required=True,
+)
+@subcommand
+def setup_apt(builder, chroot: pathlib.Path):
+    builder.setup_apt(chroot)
+
+
+@click.pass_obj
+@click.option(
+    "--package-list-file",
+    type=click.Path(
+        dir_okay=False, exists=True, resolve_path=True, path_type=pathlib.Path
+    ),
+    required=True,
+)
+@subcommand
+def generate_pool(builder, package_list_file: pathlib.Path):
+    builder.generate_pool(package_list_file)
+
+
+@click.option(
+    "--mountpoint",
+    type=str,
+    required=True,
+)
+@subcommand
+def generate_sources(builder, mountpoint: str):
+    builder.generate_sources(mountpoint)
+
+
+@click.option(
+    "--artifact-prefix",
+    type=click.Path(dir_okay=False, resolve_path=True, path_type=pathlib.Path),
+    required=True,
+)
+@subcommand
+def add_live_filesystem(builder, artifact_prefix: pathlib.Path):
+    builder.add_live_filesystem(artifact_prefix)
+
+
+@click.option(
+    "--project",
+    type=str,
+    required=True,
+)
+@click.option("--capproject", type=str, required=True)
+@click.option(
+    "--subarch",
+    type=str,
+    default="",
+)
+@subcommand
+def make_bootable(builder, project: str, capproject: str | None, subarch: str):
+    # capproject is the "capitalized project name" used in GRUB menu entries,
+    # e.g. "Ubuntu" or "Kubuntu". It should come from gen-iso-ids (which uses
+    # project_to_capproject_map for proper formatting like "Ubuntu-MATE"), but
+    # we provide a simple .capitalize() fallback for cases where the caller
+    # doesn't have the pre-computed value.
+    if capproject is None:
+        capproject = project.capitalize()
+    builder.make_bootable(project, capproject, subarch)
+
+
+@click.option(
+    "--dest",
+    type=click.Path(dir_okay=False, resolve_path=True, path_type=pathlib.Path),
+    required=True,
+)
+@click.option(
+    "--volid",
+    type=str,
+    default=None,
+)
+@subcommand
+def make_iso(builder, dest: pathlib.Path, volid: str | None):
+    builder.make_iso(dest, volid)
+
+
+if __name__ == "__main__":
+    main()

live-build/isobuilder/__init__.py

@@ -0,0 +1 @@
+#

live-build/isobuilder/apt_state.py

@@ -0,0 +1,110 @@
+import dataclasses
+import os
+import pathlib
+import shutil
+import subprocess
+from typing import Iterator
+
+
+@dataclasses.dataclass
+class PackageInfo:
+    package: str
+    filename: str
+    architecture: str
+    version: str
+
+    @property
+    def spec(self) -> str:
+        return f"{self.package}:{self.architecture}={self.version}"
+
+
+def check_proc(proc, ok_codes=(0,)) -> None:
+    proc.wait()
+    if proc.returncode not in ok_codes:
+        raise Exception(f"{proc} failed")
+
+
+class AptStateManager:
+    """Maintain and use an apt state directory to access package info and debs."""
+
+    def __init__(self, logger, series: str, apt_dir: pathlib.Path):
+        self.logger = logger
+        self.series = series
+        self.apt_root = apt_dir.joinpath("root")
+        self.apt_conf_path = apt_dir.joinpath("apt.conf")
+
+    def _apt_env(self) -> dict[str, str]:
+        return dict(os.environ, APT_CONFIG=str(self.apt_conf_path))
+
+    def setup(self, chroot: pathlib.Path):
+        """Set up the manager by copying the apt configuration from `chroot`."""
+        for path in "etc/apt", "var/lib/apt":
+            tgt = self.apt_root.joinpath(path)
+            tgt.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copytree(chroot.joinpath(path), tgt)
+        self.apt_conf_path.write_text(f'Dir "{self.apt_root}/"; \n')
+        with self.logger.logged("updating apt indices"):
+            self.logger.run(["apt-get", "update"], env=self._apt_env())
+
+    def show(self, pkgs: list[str]) -> Iterator[PackageInfo]:
+        """Return information about the binary packages named by `pkgs`.
+
+        Parses apt-cache output, which uses RFC822-like format: field names
+        followed by ": " and values, with multi-line values indented with
+        leading whitespace. We skip continuation lines (starting with space)
+        since PackageInfo only needs single-line fields.
+
+        The `fields` set (derived from PackageInfo's dataclass fields) acts as
+        a filter - we only extract fields we care about, ignoring others like
+        Description. This provides some forward-compatibility if apt-cache
+        output format changes.
+        """
+        proc = subprocess.Popen(
+            ["apt-cache", "-o", "APT::Cache::AllVersions=0", "show"] + pkgs,
+            stdout=subprocess.PIPE,
+            encoding="utf-8",
+            env=self._apt_env(),
+        )
+        assert proc.stdout is not None
+        fields = {f.name for f in dataclasses.fields(PackageInfo)}
+        params: dict[str, str] = {}
+        for line in proc.stdout:
+            if line == "\n":
+                yield PackageInfo(**params)
+                params = {}
+                continue
+            if line.startswith(" "):
+                continue
+            field, value = line.split(": ", 1)
+            field = field.lower()
+            if field in fields:
+                params[field] = value.strip()
+        check_proc(proc)
+        if params:
+            yield PackageInfo(**params)
+
+    def download(self, rootdir: pathlib.Path, pkg_info: PackageInfo):
+        """Download the package specified by `pkg_info` under `rootdir`.
+
+        The package is saved to the same path under `rootdir` as it is
+        at in the archive it comes from.
+        """
+        target_dir = rootdir.joinpath(pkg_info.filename).parent
+        target_dir.mkdir(parents=True, exist_ok=True)
+        self.logger.run(
+            ["apt-get", "download", pkg_info.spec],
+            cwd=target_dir,
+            check=True,
+            env=self._apt_env(),
+        )
+
+    def in_release_path(self) -> pathlib.Path:
+        """Return the path to the InRelease file.
+
+        This assumes exactly one InRelease file matches the pattern.
+        Will raise ValueError if there are 0 or multiple matches.
+        """
+        [path] = self.apt_root.joinpath("var/lib/apt/lists").glob(
+            f"*_dists_{self.series}_InRelease"
+        )
+        return path

live-build/isobuilder/builder.py

@@ -0,0 +1,365 @@
+import contextlib
+import json
+import os
+import pathlib
+import shlex
+import shutil
+import subprocess
+import sys
+
+from isobuilder.apt_state import AptStateManager
+from isobuilder.gpg_key import EphemeralGPGKey
+from isobuilder.pool_builder import PoolBuilder
+
+# Constants
+PACKAGE_BATCH_SIZE = 200
+MAX_CMD_DISPLAY_LENGTH = 80
+
+
+def package_list_packages(package_list_file: pathlib.Path) -> list[str]:
+    # Parse germinate output to extract package names. Germinate is Ubuntu's
+    # package dependency resolver that outputs dependency trees for seeds (like
+    # "ship-live" or "server-ship-live").
+    #
+    # Germinate output format has 2 header lines at the start and 2 footer lines
+    # at the end (showing statistics), so we skip them with [2:-2].
+    # Each data line starts with the package name followed by whitespace and
+    # dependency info. This format is stable but if germinate ever changes its
+    # header/footer count, this will break silently.
+    lines = package_list_file.read_text().splitlines()[2:-2]
+    return [line.split(None, 1)[0] for line in lines]
+
+
+def make_sources_text(
+    series: str, gpg_key: EphemeralGPGKey, components: list[str], mountpoint: str
+) -> str:
+    """Generate a deb822-format apt source file for the ISO's package pool.
+
+    deb822 is the modern apt sources format (see sources.list(5) and deb822(5)).
+    It uses RFC822-style fields where multi-line values must be indented with a
+    leading space, and empty lines within a value are represented as " ."
+    (space-dot). This format is required for inline GPG keys in the Signed-By
+    field.
+    """
+    key = gpg_key.export_public()
+    quoted_key = []
+    for line in key.splitlines():
+        if not line:
+            quoted_key.append(" .")
+        else:
+            quoted_key.append(" " + line)
+    return f"""\
+Types: deb
+URIs: file://{mountpoint}
+Suites: {series}
+Components: {" ".join(components)}
+Check-Date: no
+Signed-By:
+""" + "\n".join(
+        quoted_key
+    )
+
+
+class Logger:
+
+    def __init__(self):
+        self._indent = ""
+
+    def log(self, msg):
+        print(self._indent + msg, file=sys.stderr)
+
+    @contextlib.contextmanager
+    def logged(self, msg, done_msg=None):
+        self.log(msg)
+        self._indent += "  "
+        try:
+            yield
+        finally:
+            self._indent = self._indent[:-2]
+        if done_msg is not None:
+            self.log(done_msg)
+
+    def msg_for_cmd(self, cmd, cwd=None) -> str:
+        if cwd is None:
+            _cwd = pathlib.Path().cwd()
+        else:
+            _cwd = cwd
+        fmted_cmd = []
+        for arg in cmd:
+            if isinstance(arg, pathlib.Path):
+                if arg.is_relative_to(_cwd):
+                    arg = arg.relative_to(_cwd)
+                arg = str(arg)
+            fmted_cmd.append(shlex.quote(arg))
+        fmted_cmd_str = " ".join(fmted_cmd)
+        if len(fmted_cmd_str) > MAX_CMD_DISPLAY_LENGTH:
+            fmted_cmd_str = fmted_cmd_str[:MAX_CMD_DISPLAY_LENGTH] + "..."
+        msg = f"running `{fmted_cmd_str}`"
+        if cwd is not None:
+            msg += f" in {cwd}"
+        return msg
+
+    def run(self, cmd: list[str | pathlib.Path], *args, check=True, **kw):
+        with self.logged(self.msg_for_cmd(cmd, kw.get("cwd"))):
+            return subprocess.run(cmd, *args, check=check, **kw)
+
+
+class ISOBuilder:
+
+    def __init__(self, workdir: pathlib.Path):
+        self.workdir = workdir
+        self.logger = Logger()
+        self.iso_root = workdir.joinpath("iso-root")
+        self._series = self._arch = self._gpg_key = self._apt_state = None
+
+    # UTILITY STUFF
+
+    def _read_config(self):
+        with self.workdir.joinpath("config.json").open() as fp:
+            data = json.load(fp)
+        self._series = data["series"]
+        self._arch = data["arch"]
+
+    @property
+    def arch(self):
+        if self._arch is None:
+            self._read_config()
+        return self._arch
+
+    @property
+    def series(self):
+        if self._series is None:
+            self._read_config()
+        return self._series
+
+    @property
+    def gpg_key(self):
+        if self._gpg_key is None:
+            self._gpg_key = EphemeralGPGKey(
+                self.logger, self.workdir.joinpath("gpg-home")
+            )
+        return self._gpg_key
+
+    @property
+    def apt_state(self):
+        if self._apt_state is None:
+            self._apt_state = AptStateManager(
+                self.logger, self.series, self.workdir.joinpath("apt-state")
+            )
+        return self._apt_state
+
+    # COMMANDS
+
+    def init(self, disk_info: str, series: str, arch: str):
+        self.logger.log("creating directories")
+        self.workdir.mkdir(exist_ok=True)
+        self.iso_root.mkdir()
+        dot_disk = self.iso_root.joinpath(".disk")
+        dot_disk.mkdir()
+
+        self.logger.log("saving config")
+        with self.workdir.joinpath("config.json").open("w") as fp:
+            json.dump({"arch": arch, "series": series}, fp)
+
+        self.logger.log("populating .disk")
+        dot_disk.joinpath("base_installable").touch()
+        dot_disk.joinpath("cd_type").write_text("full_cd/single\n")
+        dot_disk.joinpath("info").write_text(disk_info)
+        self.iso_root.joinpath("casper").mkdir()
+
+        self.gpg_key.create()
+
+    def setup_apt(self, chroot: pathlib.Path):
+        self.apt_state.setup(chroot)
+
+    def generate_pool(self, package_list_file: pathlib.Path):
+        # do we need any of the symlinks we create here??
+        self.logger.log("creating pool skeleton")
+        self.iso_root.joinpath("ubuntu").symlink_to(".")
+        if self.arch not in ("amd64", "i386"):
+            self.iso_root.joinpath("ubuntu-ports").symlink_to(".")
+        self.iso_root.joinpath("dists", self.series).mkdir(parents=True)
+
+        builder = PoolBuilder(
+            self.logger,
+            series=self.series,
+            rootdir=self.iso_root,
+            apt_state=self.apt_state,
+        )
+        pkgs = package_list_packages(package_list_file)
+        # XXX include 32-bit deps of 32-bit packages if needed here
+        with self.logger.logged("adding packages"):
+            for i in range(0, len(pkgs), PACKAGE_BATCH_SIZE):
+                builder.add_packages(
+                    self.apt_state.show(pkgs[i : i + PACKAGE_BATCH_SIZE])
+                )
+        builder.make_packages()
+        release_file = builder.make_release()
+        self.gpg_key.sign(release_file)
+        for name in "stable", "unstable":
+            self.iso_root.joinpath("dists", name).symlink_to(self.series)
+
+    def generate_sources(self, mountpoint: str):
+        components = [p.name for p in self.iso_root.joinpath("pool").iterdir()]
+        print(
+            make_sources_text(
+                self.series, self.gpg_key, mountpoint=mountpoint, components=components
+            )
+        )
+
+    def _extract_casper_uuids(self):
+        # Extract UUID files from initrd images for casper (the live boot system).
+        # Each initrd contains a conf/uuid.conf with a unique identifier that
+        # casper uses at boot time to locate the correct root filesystem. These
+        # UUIDs must be placed in .disk/casper-uuid-<flavor> on the ISO so casper
+        # can verify it's booting from the right media.
+        with self.logger.logged("extracting casper uuids"):
+            casper_dir = self.iso_root.joinpath("casper")
+            prefix = "filesystem.initrd-"
+            dot_disk = self.iso_root.joinpath(".disk")
+            for initrd in casper_dir.glob(f"{prefix}*"):
+                initrddir = self.workdir.joinpath("initrd")
+                with self.logger.logged(
+                    f"unpacking {initrd.name} ...", done_msg="... done"
+                ):
+                    self.logger.run(["unmkinitramfs", initrd, initrddir])
+                # unmkinitramfs can produce different directory structures:
+                # - Platforms with early firmware: subdirs like "main/" or "early/"
+                #   containing conf/uuid.conf
+                # - Other platforms: conf/uuid.conf directly in the root
+                # Try to find uuid.conf in both locations. The [uuid_conf] = confs
+                # unpacking asserts exactly one match; multiple matches would
+                # indicate an unexpected initrd structure.
+                confs = list(initrddir.glob("*/conf/uuid.conf"))
+                if confs:
+                    [uuid_conf] = confs
+                elif initrddir.joinpath("conf/uuid.conf").exists():
+                    uuid_conf = initrddir.joinpath("conf/uuid.conf")
+                else:
+                    raise Exception("uuid.conf not found")
+                self.logger.log(f"found {uuid_conf.relative_to(initrddir)}")
+                uuid_conf.rename(
+                    dot_disk.joinpath("casper-uuid-" + initrd.name[len(prefix) :])
+                )
+                shutil.rmtree(initrddir)
+
+    def add_live_filesystem(self, artifact_prefix: pathlib.Path):
+        # Link build artifacts into the ISO's casper directory. We use hardlinks
+        # (not copies) for filesystem efficiency - they reference the same inode.
+        #
+        # Artifacts come from the layered build with names like "for-iso.base.squashfs"
+        # and need to be renamed for casper. The prefix is stripped, so:
+        #   for-iso.base.squashfs -> base.squashfs
+        #   for-iso.kernel-generic -> filesystem.kernel-generic
+        #
+        # Kernel and initrd get the extra "filesystem." prefix because casper
+        # expects names like filesystem.kernel-* and filesystem.initrd-*.
+        casper_dir = self.iso_root.joinpath("casper")
+        artifact_dir = artifact_prefix.parent
+        filename_prefix = artifact_prefix.name
+
+        def link(src, target_name):
+            target = casper_dir.joinpath(target_name)
+            self.logger.log(
+                f"creating link from $ISOROOT/casper/{target_name} to $src/{src.name}"
+            )
+            target.hardlink_to(src)
+
+        with self.logger.logged(
+            f"linking artifacts from {casper_dir} to {artifact_dir}"
+        ):
+            for ext in "squashfs", "squashfs.gpg", "size", "manifest", "yaml":
+                for path in artifact_dir.glob(f"{filename_prefix}*.{ext}"):
+                    newname = path.name[len(filename_prefix) :]
+                    link(path, newname)
+            for item in "kernel", "initrd":
+                for path in artifact_dir.glob(f"{filename_prefix}{item}-*"):
+                    newname = "filesystem." + path.name[len(filename_prefix) :]
+                    link(path, newname)
+        self._extract_casper_uuids()
+
+    def make_bootable(self, project: str, capproject: str, subarch: str):
+        # debian-cd is Ubuntu's CD/ISO image build system. It contains
+        # architecture and series-specific boot configuration scripts that set up
+        # GRUB, syslinux, EFI boot, etc. The tools/boot/$series/boot-$arch script
+        # knows how to make an ISO bootable for each architecture.
+        #
+        # TODO: The boot configuration logic should eventually be ported directly
+        # into isobuilder to avoid this external dependency and git clone.
+        debian_cd_dir = self.workdir.joinpath("debian-cd")
+        with self.logger.logged("cloning debian-cd"):
+            self.logger.run(
+                [
+                    "git",
+                    "clone",
+                    "--depth=1",
+                    "https://git.launchpad.net/~ubuntu-cdimage/debian-cd/+git/ubuntu",
+                    debian_cd_dir,
+                ],
+            )
+        # Override apt-selection to use our ISO's apt configuration instead of
+        # debian-cd's default. This ensures the boot scripts query packages from
+        # the correct repository (our pool) when determining boot requirements.
+        apt_selection = debian_cd_dir.joinpath("tools/apt-selection")
+        with self.logger.logged("overwriting apt-selection"):
+            apt_selection.write_text(
+                "#!/bin/sh\n" f"APT_CONFIG={self.apt_state.apt_conf_path} apt-get $@\n"
+            )
+        env = dict(
+            os.environ,
+            BASEDIR=str(debian_cd_dir),
+            DIST=self.series,
+            PROJECT=project,
+            CAPPROJECT=capproject,
+            SUBARCH=subarch,
+        )
+        tool_name = f"tools/boot/{self.series}/boot-{self.arch}"
+        with self.logger.logged(f"running {tool_name} ...", done_msg="... done"):
+            self.logger.run(
+                [
+                    debian_cd_dir.joinpath(tool_name),
+                    "1",
+                    self.iso_root,
+                ],
+                env=env,
+            )
+
+    def checksum(self):
+        # Generate md5sum.txt for ISO integrity verification.
+        # - Symlinks are excluded because their targets are already checksummed
+        # - Files are sorted for deterministic, reproducible output across builds
+        # - Paths use "./" prefix and we run md5sum from iso_root so the output
+        #   matches what users get when they verify with "md5sum -c" from the ISO
+        all_files = []
+        for dirpath, dirnames, filenames in self.iso_root.walk():
+            filepaths = [dirpath.joinpath(filename) for filename in filenames]
+            all_files.extend(
+                "./" + str(filepath.relative_to(self.iso_root))
+                for filepath in filepaths
+                if not filepath.is_symlink()
+            )
+        self.iso_root.joinpath("md5sum.txt").write_bytes(
+            self.logger.run(
+                ["md5sum"] + sorted(all_files),
+                cwd=self.iso_root,
+                stdout=subprocess.PIPE,
+            ).stdout
+        )
+
+    def make_iso(self, dest: pathlib.Path, volid: str | None):
+        # 1.mkisofs_opts is generated by debian-cd's make_bootable step. The "1"
+        # refers to "pass 1" of the build (a legacy naming convention). It contains
+        # architecture-specific xorriso options for boot sectors, EFI images, etc.
+        mkisofs_opts = shlex.split(self.workdir.joinpath("1.mkisofs_opts").read_text())
+        self.checksum()
+        # xorriso with "-as mkisofs" runs in mkisofs compatibility mode.
+        # -r enables Rock Ridge extensions for Unix metadata (permissions, symlinks).
+        # -iso-level 3 (amd64 only) allows files >4GB which some amd64 ISOs need.
+        cmd: list[str | pathlib.Path] = ["xorriso", "-as", "mkisofs", "-r"]
+        if self.arch == "amd64":
+            cmd.extend(["-iso-level", "3"])
+        if volid:
+            cmd += ["-V", volid]
+        cmd += mkisofs_opts + [self.iso_root, "-o", dest]
+        with self.logger.logged("running xorriso"):
+            self.logger.run(cmd, cwd=self.workdir, check=True)

live-build/isobuilder/gpg_key.py

@@ -0,0 +1,58 @@
+import pathlib
+import subprocess
+
+key_conf = """\
+%no-protection
+Key-Type: eddsa
+Key-Curve: Ed25519
+Key-Usage: sign
+Name-Real: Ubuntu ISO One-Time Signing Key
+Name-Email: noone@nowhere.invalid
+Expire-Date: 0
+"""
+
+
+class EphemeralGPGKey:
+
+    def __init__(self, logger, gpghome):
+        self.logger = logger
+        self.gpghome = gpghome
+
+    def _run_gpg(self, cmd, **kwargs):
+        return self.logger.run(
+            ["gpg", "--homedir", self.gpghome] + cmd, check=True, **kwargs
+        )
+
+    def create(self):
+        with self.logger.logged("creating gpg key ...", done_msg="... done"):
+            self.gpghome.mkdir(mode=0o700)
+            self._run_gpg(
+                ["--gen-key", "--batch"],
+                input=key_conf,
+                text=True,
+            )
+
+    def sign(self, path: pathlib.Path):
+        with self.logger.logged(f"signing {path}"):
+            with path.open("rb") as inp:
+                with pathlib.Path(str(path) + ".gpg").open("wb") as outp:
+                    self._run_gpg(
+                        [
+                            "--no-options",
+                            "--batch",
+                            "--no-tty",
+                            "--armour",
+                            "--digest-algo",
+                            "SHA512",
+                            "--detach-sign",
+                        ],
+                        stdin=inp,
+                        stdout=outp,
+                    )
+
+    def export_public(self) -> str:
+        return self._run_gpg(
+            ["--export", "--armor"],
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout

live-build/isobuilder/pool_builder.py

@@ -0,0 +1,166 @@
+import pathlib
+import subprocess
+import tempfile
+
+from isobuilder.apt_state import AptStateManager, PackageInfo
+
+
+generate_template = """
+Dir::ArchiveDir "{root}";
+Dir::CacheDir "{scratch}/apt-ftparchive-db";
+
+TreeDefault::Contents " ";
+
+Tree "dists/{series}" {{
+    FileList "{scratch}/filelist_$(SECTION)";
+    Sections "{components}";
+    Architectures "{arches}";
+}}
+"""
+
+
+class PoolBuilder:
+
+    def __init__(
+        self, logger, series: str, apt_state: AptStateManager, rootdir: pathlib.Path
+    ):
+        self.logger = logger
+        self.series = series
+        self.apt_state = apt_state
+        self.rootdir = rootdir
+        self.arches: set[str] = set()
+        self._present_components: set[str] = set()
+
+    def add_packages(self, pkglist: list[PackageInfo]):
+        for pkg_info in pkglist:
+            if pkg_info.architecture != "all":
+                self.arches.add(pkg_info.architecture)
+            self.apt_state.download(self.rootdir, pkg_info)
+
+    def make_packages(self) -> None:
+        with self.logger.logged("making Packages files"):
+            with tempfile.TemporaryDirectory() as tmpdir:
+                scratchdir = pathlib.Path(tmpdir)
+                with self.logger.logged("scanning for packages"):
+                    for component in ["main", "restricted", "universe", "multiverse"]:
+                        if not self.rootdir.joinpath("pool", component).is_dir():
+                            continue
+                        self._present_components.add(component)
+                        for arch in self.arches:
+                            self.rootdir.joinpath(
+                                "dists", self.series, component, f"binary-{arch}"
+                            ).mkdir(parents=True)
+                        proc = self.logger.run(
+                            ["find", f"pool/{component}"],
+                            stdout=subprocess.PIPE,
+                            cwd=self.rootdir,
+                            encoding="utf-8",
+                            check=True,
+                        )
+                        scratchdir.joinpath(f"filelist_{component}").write_text(
+                            "\n".join(sorted(proc.stdout.splitlines()))
+                        )
+                with self.logger.logged("writing apt-ftparchive config"):
+                    scratchdir.joinpath("apt-ftparchive-db").mkdir()
+                    generate_path = scratchdir.joinpath("generate-binary")
+                    generate_path.write_text(
+                        generate_template.format(
+                            arches=" ".join(self.arches),
+                            series=self.series,
+                            root=self.rootdir.resolve(),
+                            scratch=scratchdir.resolve(),
+                            components=" ".join(self._present_components),
+                        )
+                    )
+                with self.logger.logged("running apt-ftparchive generate"):
+                    self.logger.run(
+                        [
+                            "apt-ftparchive",
+                            "--no-contents",
+                            "--no-md5",
+                            "--no-sha1",
+                            "--no-sha512",
+                            "generate",
+                            generate_path,
+                        ],
+                        check=True,
+                    )
+
+    def make_release(self) -> pathlib.Path:
+        # Build the Release file by merging metadata from the mirror with
+        # checksums for our pool. We can't just use apt-ftparchive's Release
+        # output directly because:
+        # 1. apt-ftparchive doesn't know about Origin, Label, Suite, Version,
+        #    Codename, etc. - these come from the mirror and maintain package
+        #    provenance
+        # 2. We keep the mirror's Date (when packages were released) rather than
+        #    apt-ftparchive's Date (when we ran the command)
+        # 3. We need to override Architectures/Components to match our pool
+        #
+        # There may be a cleaner way (apt-get indextargets?) but this works.
+        with self.logger.logged("making Release file"):
+            in_release = self.apt_state.in_release_path()
+            cp_mirror_release = self.logger.run(
+                ["gpg", "--verify", "--output", "-", in_release],
+                stdout=subprocess.PIPE,
+                encoding="utf-8",
+                check=False,
+            )
+            if cp_mirror_release.returncode not in (0, 2):
+                # gpg returns code 2 when the public key the InRelease is
+                # signed with is not available, which is most of the time.
+                raise Exception("gpg failed")
+            mirror_release_lines = cp_mirror_release.stdout.splitlines()
+            release_dir = self.rootdir.joinpath("dists", self.series)
+            af_release_lines = self.logger.run(
+                [
+                    "apt-ftparchive",
+                    "--no-contents",
+                    "--no-md5",
+                    "--no-sha1",
+                    "--no-sha512",
+                    "release",
+                    ".",
+                ],
+                stdout=subprocess.PIPE,
+                encoding="utf-8",
+                cwd=release_dir,
+                check=True,
+            ).stdout.splitlines()
+            # Build the final Release file by merging mirror metadata with pool
+            # checksums.
+            # Strategy:
+            # 1. Take metadata fields (Suite, Origin, etc.) from the mirror's InRelease
+            # 2. Override Architectures and Components to match what's actually in our
+            #    pool
+            # 3. Skip the mirror's checksum sections (MD5Sum, SHA256, etc.) because they
+            #    don't apply to our pool
+            # 4. Skip Acquire-By-Hash since we don't use it
+            # 5. Append checksums from apt-ftparchive (but not the Date field)
+            release_lines = []
+            skipping = False
+            for line in mirror_release_lines:
+                if line.startswith("Architectures:"):
+                    line = "Architectures: " + " ".join(sorted(self.arches))
+                elif line.startswith("Components:"):
+                    line = "Components: " + " ".join(sorted(self._present_components))
+                elif line.startswith("MD5") or line.startswith("SHA"):
+                    # Start of a checksum section - skip this and indented lines below
+                    # it
+                    skipping = True
+                elif not line.startswith(" "):
+                    # Non-indented line means we've left the checksum section if we were
+                    # in one.
+                    skipping = False
+                if line.startswith("Acquire-By-Hash"):
+                    continue
+                if not skipping:
+                    release_lines.append(line)
+            # Append checksums from apt-ftparchive, but skip its Date field
+            # (we want to keep the Date from the mirror release)
+            for line in af_release_lines:
+                if not line.startswith("Date"):
+                    release_lines.append(line)
+            release_path = release_dir.joinpath("Release")
+            release_path.write_text("\n".join(release_lines))
+            return release_path

live-build/lb_binary_layered

@@ -184,6 +184,20 @@ build_layered_squashfs () {
 		fi
 
 		create_squashfs "${overlay_dir}" ${squashfs_f}
+		# Create a "for-iso" variant of the squashfs for ISO builds. For
+		# the root layer (the base system) when building with a pool, we
+		# need to include cdrom.sources so casper can access the ISO's
+		# package repository. This requires regenerating the squashfs with
+		# that file included, then removing it (so it doesn't pollute the
+		# regular squashfs). Non-root layers (desktop environment, etc.)
+		# and builds without pools can just hardlink to the regular squashfs.
+		if [ -n "${POOL_SEED_NAME}" ] && $(is_root_layer $pass); then
+			isobuild generate-sources --mountpoint=/cdrom > ${overlay_dir}/etc/apt/sources.list.d/cdrom.sources
+			create_squashfs "${overlay_dir}" ${PWD}/for-iso.${pass}.squashfs
+			rm ${overlay_dir}/etc/apt/sources.list.d/cdrom.sources
+		else
+			ln ${squashfs_f} ${PWD}/for-iso.${pass}.squashfs
+		fi
 
 		if [ -f config/$pass.catalog-in.yaml ]; then
 			echo "Expanding catalog entry template for $pass"