From 7ad0444511ae1a4e0243b7353a2aa2f4bc45129d Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 10:47:54 +1300 Subject: [PATCH 01/11] do not include kernel in base install for ubuntu-server:live this actually makes things closer to the non-live server build and so makes things a bit simpler --- live-build/auto/config | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index fe82ccad..463fc83d 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -150,20 +150,15 @@ case $IMAGEFORMAT in ;; plain) - INITRAMFS_TYPE=none case $PROJECT:${SUBPROJECT:-} in ubuntu-server:live) - # Stop lb installing casper into filesystem.squashfs - # by skipping lb_chroot_live-packages. - skip_lb_stage chroot_live-packages - INITRAMFS_TYPE=auto touch config/universe-enabled ;; *) PREINSTALLED=true ;; esac - OPTS="${OPTS:+$OPTS }--initramfs $INITRAMFS_TYPE --chroot-filesystem $IMAGEFORMAT" + OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT" ;; ubuntu-image) @@ -536,9 +531,6 @@ case $PROJECT in add_task install standard add_task install server LIVE_TASK='cloud-image' - case $ARCH in - amd64) add_package live linux-signed-generic ;; - esac ;; esac COMPONENTS='main' @@ -821,8 +813,6 @@ case $ARCH in esac case $PROJECT:${SUBPROJECT:-} in - ubuntu-server:live) - ;; ubuntu-server:*|ubuntu-base:*|ubuntu-touch:*|ubuntu-touch-custom:*) OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none" KERNEL_FLAVOURS=none From d3eadc704c6533d34fe2b5624c3fc5e9ae8a410b Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 11:10:44 +1300 Subject: [PATCH 02/11] do not unmount the installer overlay in 032-installer-squashfs.binary so that a new hook can create yet further overlays on top of it --- .../hooks/032-installer-squashfs.binary | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary index bab6ab76..63edba4f 100755 --- a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary +++ b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary @@ -24,13 +24,14 @@ fi . config/functions . config/common -SQUASH_ROOT=binary/boot/squashfs.dir +FILESYSTEM_ROOT=binary/boot/squashfs.dir +INSTALLER_ROOT=binary/boot/installer.squashfs.dir OVERLAY_ROOT=binary/overlay -mkdir -p "$OVERLAY_ROOT" +mkdir -p "$INSTALLER_ROOT" "$OVERLAY_ROOT" # Create an installer squashfs layer -mount_overlay "$SQUASH_ROOT/" "$OVERLAY_ROOT/" "$SQUASH_ROOT/" +mount_overlay "$FILESYSTEM_ROOT/" "$OVERLAY_ROOT/" "$INSTALLER_ROOT/" setup_mountpoint binary/boot/squashfs.dir @@ -40,8 +41,8 @@ setup_mountpoint binary/boot/squashfs.dir # It would be better to have this in ../includes.binary/overlay but # you can't have backslashes in filenames in bzr branches! DEVICE_UNIT_NAME='dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device' -mkdir -p "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d" -cat > "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" < "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" < Date: Thu, 13 Dec 2018 11:18:55 +1300 Subject: [PATCH 03/11] add hook to create kernel/initrd/modules for ISO --- .../hooks/033-kernel-bits.binary | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 live-build/ubuntu-server/hooks/033-kernel-bits.binary diff --git a/live-build/ubuntu-server/hooks/033-kernel-bits.binary b/live-build/ubuntu-server/hooks/033-kernel-bits.binary new file mode 100644 index 00000000..9a193b52 --- /dev/null +++ b/live-build/ubuntu-server/hooks/033-kernel-bits.binary @@ -0,0 +1,66 @@ +#!/bin/bash -eux +# vi: ts=4 noexpandtab +# +# Generate a squashfs root and manifest + +echo "033-kernel-bits.binary" + +case ${IMAGE_TARGETS-} in + ""|*squashfs*) + ;; + *) + echo "Skipping squashfs build" + exit 0 + ;; +esac + +if [ -n "${SUBARCH-}" ]; then + echo "Skipping rootfs build for subarch flavor build" + exit 0 +fi + +. config/functions +. config/common + +INSTALLER_ROOT=binary/boot/installer.squashfs.dir + +KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir +KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits + +#variants='ga hwe' +variants='ga' + +for variant in $variants; do + if [ "$variant" = "ga" ]; then + kernel_metapkg=linux-generic + flavor=generic + elif [ "$variant" = "hwe" ]; then + kernel_metapkg=linux-generic-hwe-$(lsb_release -sc) + flavor=generic-hwe + else + echo "bogus variant: $variant" + exit 1 + fi + + # Make preparations + mkdir -p $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY + mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/" + setup_mountpoint $KERNEL_BITS_ROOT + + # Install the kernel! + env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg} + + # Fish out generated kernel image and initrd + mv "$KERNEL_BITS_ROOT"/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-$flavor + mv "$KERNEL_BITS_ROOT"/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-$flavor + + # Create squashfs containing all the modules + modules_squashfs_path="${PWD}/livecd.${PROJECT}.modules.squashfs-$flavor" + (cd "$KERNEL_BITS_ROOT/lib/modules" && + mksquashfs . $modules_squashfs_path -no-progress -xattrs -comp xz) + + # And clean up + teardown_mountpoint $KERNEL_BITS_ROOT + umount $KERNEL_BITS_ROOT + rm -rf $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY +done From bcbf9ea36e37869b3517d96adbd7ae327074881b Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 11:22:34 +1300 Subject: [PATCH 04/11] add initramfs hook to record kernel metapackage & mount modules --- .../hooks/033-kernel-bits.binary | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/live-build/ubuntu-server/hooks/033-kernel-bits.binary b/live-build/ubuntu-server/hooks/033-kernel-bits.binary index 9a193b52..af67924d 100644 --- a/live-build/ubuntu-server/hooks/033-kernel-bits.binary +++ b/live-build/ubuntu-server/hooks/033-kernel-bits.binary @@ -47,6 +47,27 @@ for variant in $variants; do mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/" setup_mountpoint $KERNEL_BITS_ROOT + # Configure initramfs creation + mkdir -p "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/ + if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then + echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf + fi + echo "CASPER_GENERATE_UUID=1" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/casper.conf + + # Add a hook to record which kernel was booted and mount the + # modules.squashfs created below. + cat < "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server +#!/bin/sh +case \$1 in +prereqs) exit 0;; +esac + +echo ${kernel_metapkg} > /run/kernel-meta-package +mkdir -p \$rootmnt/lib/modules +mount \$rootmnt/cdrom/casper/extras/modules.squashfs-$flavor \$rootmnt/lib/modules +EOF + chmod +x "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server + # Install the kernel! env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg} From bfeebc90abc7f3a01c6441d26124b64d4ceb6d8b Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 11:23:15 +1300 Subject: [PATCH 05/11] remove cloud-initramfs-copymods before installing kernel --- live-build/ubuntu-server/hooks/033-kernel-bits.binary | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/live-build/ubuntu-server/hooks/033-kernel-bits.binary b/live-build/ubuntu-server/hooks/033-kernel-bits.binary index af67924d..5ebfd1d9 100644 --- a/live-build/ubuntu-server/hooks/033-kernel-bits.binary +++ b/live-build/ubuntu-server/hooks/033-kernel-bits.binary @@ -47,6 +47,10 @@ for variant in $variants; do mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/" setup_mountpoint $KERNEL_BITS_ROOT + # Our initramfs hook implements a kind of extreme version of + # cloud-initramfs-copymods, so remove that and prevent duelling hooks + env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y remove cloud-initramfs-copymods ubuntu-server || true + # Configure initramfs creation mkdir -p "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/ if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then From f3a458a85f1c35949ee9a530fc871b12f05f50e5 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 15:28:02 +1300 Subject: [PATCH 06/11] enable hwe variant --- live-build/ubuntu-server/hooks/033-kernel-bits.binary | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/live-build/ubuntu-server/hooks/033-kernel-bits.binary b/live-build/ubuntu-server/hooks/033-kernel-bits.binary index 5ebfd1d9..ea72d19e 100644 --- a/live-build/ubuntu-server/hooks/033-kernel-bits.binary +++ b/live-build/ubuntu-server/hooks/033-kernel-bits.binary @@ -27,8 +27,7 @@ INSTALLER_ROOT=binary/boot/installer.squashfs.dir KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits -#variants='ga hwe' -variants='ga' +variants='ga hwe' for variant in $variants; do if [ "$variant" = "ga" ]; then From de447eb3e5a202d91618d29b776264ae47409ea9 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 11:51:11 +1300 Subject: [PATCH 07/11] changelog --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 25c111a3..1f789b7c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,13 @@ livecd-rootfs (2.525.13) UNRELEASED; urgency=medium [ Balint Reczey ] * Update Vcs-* fields in debian/control to point to git. + [ Michael Hudson-Doyle ] + * Changes to kernel handling for live-server: do not include kernel/initrd + in filesystem.squashfs but rather install it in a throwaway layer on top + of installer.squashfs and fish kernel, initrd and modules out of that with + an initrd hook that records kernel metapackage name in /run and mounts + /lib/modules from a squashfs on the ISO. + -- Michael Hudson-Doyle Thu, 13 Dec 2018 15:22:15 +1300 livecd-rootfs (2.525.12) bionic; urgency=medium From 7632df8e50655659677887debfc5edad6dc98b81 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 21:55:45 +1300 Subject: [PATCH 08/11] linux-generic-hwe-18.04 not linux-generic-hwe-bionic! --- live-build/ubuntu-server/hooks/033-kernel-bits.binary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/ubuntu-server/hooks/033-kernel-bits.binary b/live-build/ubuntu-server/hooks/033-kernel-bits.binary index ea72d19e..0d27344a 100644 --- a/live-build/ubuntu-server/hooks/033-kernel-bits.binary +++ b/live-build/ubuntu-server/hooks/033-kernel-bits.binary @@ -34,7 +34,7 @@ for variant in $variants; do kernel_metapkg=linux-generic flavor=generic elif [ "$variant" = "hwe" ]; then - kernel_metapkg=linux-generic-hwe-$(lsb_release -sc) + kernel_metapkg=linux-generic-hwe-$(lsb_release -sr) flavor=generic-hwe else echo "bogus variant: $variant" From a473683ac7d8ad76f07c16bf53ad11e6faecd087 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 13 Dec 2018 12:23:10 +1300 Subject: [PATCH 09/11] Do no install openssh-server in the base filsystem for the live server installer. --- debian/changelog | 2 ++ live-build/auto/config | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1f789b7c..c2d5c8b7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ livecd-rootfs (2.525.13) UNRELEASED; urgency=medium of installer.squashfs and fish kernel, initrd and modules out of that with an initrd hook that records kernel metapackage name in /run and mounts /lib/modules from a squashfs on the ISO. + * Do no install openssh-server in the base filsystem for the live server + installer. -- Michael Hudson-Doyle Thu, 13 Dec 2018 15:22:15 +1300 diff --git a/live-build/auto/config b/live-build/auto/config index 463fc83d..1026121c 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -530,7 +530,7 @@ case $PROJECT in live) add_task install standard add_task install server - LIVE_TASK='cloud-image' + add_package install cloud-init ;; esac COMPONENTS='main' From f129e5797a54185f07c9c065105fd83f94e4a89a Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Mon, 17 Dec 2018 13:37:36 +1300 Subject: [PATCH 10/11] A few simple tweaks to reduce size of live servers installer.squashfs - Do not run apt-get update (which can bring in package lists if we are unlucky wrt publisher schedules). - Run apt-get clean to clear out downloaded debs of curtin/casper and dependencies. - Do not install user-setup. - Use the core snap from the base filesystem if present. --- debian/changelog | 7 +++++++ .../hooks/032-installer-squashfs.binary | 17 ++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index c2d5c8b7..360155ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,13 @@ livecd-rootfs (2.525.13) UNRELEASED; urgency=medium /lib/modules from a squashfs on the ISO. * Do no install openssh-server in the base filsystem for the live server installer. + * A few simple tweaks to reduce size of live servers installer.squashfs: + - Do not run apt-get update (which can bring in package lists if we are + unlucky wrt publisher schedules). + - Run apt-get clean to clear out downloaded debs of curtin/casper and + dependencies. + - Do not install user-setup. + - Use the core snap from the base filesystem if present. -- Michael Hudson-Doyle Thu, 13 Dec 2018 15:22:15 +1300 diff --git a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary index 63edba4f..d08b4cd9 100755 --- a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary +++ b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary @@ -51,8 +51,9 @@ EOF # Install any requirements for the installer, for things we don't want # to see on the installed system -chroot $INSTALLER_ROOT apt-get update -chroot $INSTALLER_ROOT apt-get -y install user-setup curtin lupin-casper +chroot $INSTALLER_ROOT apt-get -y install curtin lupin-casper +chroot $INSTALLER_ROOT apt-get clean + # For bug #1743643 "Install to dirty disk with swap fails" remove the # "helpful" casper script that mounts any swap partitions it finds. @@ -65,15 +66,17 @@ touch $INSTALLER_ROOT/etc/cloud/cloud-init.disabled chroot $INSTALLER_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions chroot $INSTALLER_ROOT sh -c ' set -x; -cd /var/lib/snapd/seed; -sudo SNAPPY_STORE_NO_CDN=1 snap download core; -sudo SNAPPY_STORE_NO_CDN=1 snap download subiquity; +mkdir -p /var/lib/snapd/seed/snaps/ +cd /var/lib/snapd/seed/snaps/; +if [ ! -e core_*.snap ]; then + SNAPPY_STORE_NO_CDN=1 snap download core +fi +SNAPPY_STORE_NO_CDN=1 snap download subiquity; -CORE_SNAP=$(ls -1 core*.snap); +CORE_SNAP=$(ls -1 core_*.snap); SUBIQUITY_SNAP=$(ls -1 subiquity*.snap); mv *.assert /var/lib/snapd/seed/assertions/; -mv *.snap /var/lib/snapd/seed/snaps/; cat < /var/lib/snapd/seed/seed.yaml snaps: From e35dde7f68bda55438b83f1235c6c12631ff2d5b Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Tue, 18 Dec 2018 15:18:00 +1300 Subject: [PATCH 11/11] Do not include curtin in the live-server installer.squashfs as the version of subiquity that includes it in the snap has now been released to stable. --- debian/changelog | 3 +++ .../ubuntu-server/hooks/032-installer-squashfs.binary | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index 360155ed..92504d69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -18,6 +18,9 @@ livecd-rootfs (2.525.13) UNRELEASED; urgency=medium dependencies. - Do not install user-setup. - Use the core snap from the base filesystem if present. + * Do not include curtin in the live-server installer.squashfs as the + version of subiquity that includes it in the snap has now been released to + stable. -- Michael Hudson-Doyle Thu, 13 Dec 2018 15:22:15 +1300 diff --git a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary index d08b4cd9..518fc724 100755 --- a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary +++ b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary @@ -49,12 +49,10 @@ EOF # Prepare installer layer. -# Install any requirements for the installer, for things we don't want -# to see on the installed system -chroot $INSTALLER_ROOT apt-get -y install curtin lupin-casper +# Install casper for live session magic. +chroot $INSTALLER_ROOT apt-get -y install lupin-casper chroot $INSTALLER_ROOT apt-get clean - # For bug #1743643 "Install to dirty disk with swap fails" remove the # "helpful" casper script that mounts any swap partitions it finds. rm -f $INSTALLER_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap