[mwhudson, r=adconrad] Merge mwhudson/live-server-bionic for live-server HWE

debian/changelog

@@ -1,3 +1,26 @@
+livecd-rootfs (2.525.18) UNRELEASED; urgency=medium
+
+  [ Michael Hudson-Doyle ]
+  * Changes to kernel handling for live-server: do not include kernel/initrd
+    in filesystem.squashfs but rather install it in a throwaway layer on top
+    of installer.squashfs and fish kernel, initrd and modules out of that with
+    an initrd hook that records kernel metapackage name in /run and mounts
+    /lib/modules from a squashfs on the ISO.
+  * Do no install openssh-server in the base filsystem for the live server
+    installer.
+  * A few simple tweaks to reduce size of live servers installer.squashfs:
+    - Do not run apt-get update (which can bring in package lists if we are
+      unlucky wrt publisher schedules).
+    - Run apt-get clean to clear out downloaded debs of curtin/casper and
+      dependencies.
+    - Do not install user-setup.
+    - Use the core snap from the base filesystem if present.
+  * Do not include curtin in the live-server installer.squashfs as the
+    version of subiquity that includes it in the snap has now been released to
+    stable.
+
+ -- Adam Conrad <adconrad@ubuntu.com>  Tue, 12 Feb 2019 11:35:48 -0700
+
 livecd-rootfs (2.525.17) bionic; urgency=medium
 
   * Add a buildd subproject.

live-build/auto/config

@@ -162,20 +162,15 @@ case $IMAGEFORMAT in
 		;;
 
 	plain)
-		INITRAMFS_TYPE=none
 		case $PROJECT:${SUBPROJECT:-} in
 			ubuntu-server:live)
-				# Stop lb installing casper into filesystem.squashfs
-				#  by skipping lb_chroot_live-packages.
-				skip_lb_stage chroot_live-packages
-				INITRAMFS_TYPE=auto
 	                        touch config/universe-enabled
 				;;
 			*)
 				PREINSTALLED=true
 				;;
 		esac
-		OPTS="${OPTS:+$OPTS }--initramfs $INITRAMFS_TYPE --chroot-filesystem $IMAGEFORMAT"
+		OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT"
 		;;
 
 	ubuntu-image)
@@ -557,10 +552,7 @@ case $PROJECT in
 			live)
 				add_task install standard
 				add_task install server
-				LIVE_TASK='cloud-image'
+				add_package install cloud-init
-				case $ARCH in
-					amd64)	add_package live linux-signed-generic ;;
-				esac
 				;;
 		esac
 		COMPONENTS='main'
@@ -874,8 +866,6 @@ case $ARCH in
 esac
 
 case $PROJECT:${SUBPROJECT:-} in
-	ubuntu-server:live)
-		;;
 	ubuntu-server:*|ubuntu-base:*|ubuntu-touch:*|ubuntu-touch-custom:*)
 		OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none"
 		KERNEL_FLAVOURS=none

live-build/ubuntu-server/hooks/032-installer-squashfs.binary

@@ -24,13 +24,14 @@ fi
 . config/functions
 . config/common
 
-SQUASH_ROOT=binary/boot/squashfs.dir
+FILESYSTEM_ROOT=binary/boot/squashfs.dir
+INSTALLER_ROOT=binary/boot/installer.squashfs.dir
 OVERLAY_ROOT=binary/overlay
 
-mkdir -p "$OVERLAY_ROOT"
+mkdir -p "$INSTALLER_ROOT" "$OVERLAY_ROOT"
 
 # Create an installer squashfs layer
-mount_overlay "$SQUASH_ROOT/" "$OVERLAY_ROOT/" "$SQUASH_ROOT/"
+mount_overlay "$FILESYSTEM_ROOT/" "$OVERLAY_ROOT/" "$INSTALLER_ROOT/"
 
 setup_mountpoint binary/boot/squashfs.dir
 
@@ -40,42 +41,40 @@ setup_mountpoint binary/boot/squashfs.dir
 # It would be better to have this in ../includes.binary/overlay but
 # you can't have backslashes in filenames in bzr branches!
 DEVICE_UNIT_NAME='dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device'
-mkdir -p "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d"
+mkdir -p "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d"
-cat > "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" <<EOF
+cat > "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" <<EOF
 [Unit]
 JobRunningTimeoutSec=0s
 EOF
 
 # Prepare installer layer.
 
-# Install any requirements for the installer, for things we don't want
+# Install casper for live session magic.
-# to see on the installed system
+chroot $INSTALLER_ROOT apt-get -y install lupin-casper
-chroot $SQUASH_ROOT apt-get update
+chroot $INSTALLER_ROOT apt-get clean
-chroot $SQUASH_ROOT apt-get -y install user-setup curtin lupin-casper
 
 # For bug #1743643 "Install to dirty disk with swap fails" remove the
 # "helpful" casper script that mounts any swap partitions it finds.
-rm -f $SQUASH_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
+rm -f $INSTALLER_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
-
-# Installing casper means we need a new initramfs
-UPDATE_INITRAMFS_OPTIONS=CASPER_GENERATE_UUID=1 recreate_initramfs $SQUASH_ROOT
 
 # Don't let cloud-init run in the live session.
-touch $SQUASH_ROOT/etc/cloud/cloud-init.disabled
+touch $INSTALLER_ROOT/etc/cloud/cloud-init.disabled
 
 # Do the snap seeding dance.
-chroot $SQUASH_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions
+chroot $INSTALLER_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions
-chroot $SQUASH_ROOT sh -c '
+chroot $INSTALLER_ROOT sh -c '
 set -x;
-cd /var/lib/snapd/seed;
+mkdir -p /var/lib/snapd/seed/snaps/
-sudo SNAPPY_STORE_NO_CDN=1 snap download core;
+cd /var/lib/snapd/seed/snaps/;
-sudo SNAPPY_STORE_NO_CDN=1 snap download subiquity;
+if [ ! -e core_*.snap ]; then
+	SNAPPY_STORE_NO_CDN=1 snap download core
+fi
+SNAPPY_STORE_NO_CDN=1 snap download subiquity;
 
-CORE_SNAP=$(ls -1 core*.snap);
+CORE_SNAP=$(ls -1 core_*.snap);
 SUBIQUITY_SNAP=$(ls -1 subiquity*.snap);
 
 mv *.assert /var/lib/snapd/seed/assertions/;
-mv *.snap /var/lib/snapd/seed/snaps/;
 
 cat <<EOF > /var/lib/snapd/seed/seed.yaml
 snaps:
@@ -89,10 +88,7 @@ snaps:
 EOF
 '
 
-teardown_mountpoint "$SQUASH_ROOT"
+teardown_mountpoint "$INSTALLER_ROOT"
-
-# Then unmount the overlay
-umount "$SQUASH_ROOT"
 
 squashfs_f="${PWD}/livecd.${PROJECT}.installer.squashfs"
 

live-build/ubuntu-server/hooks/033-kernel-bits.binary

@@ -0,0 +1,90 @@
+#!/bin/bash -eux
+# vi: ts=4 noexpandtab
+#
+# Generate a squashfs root and manifest
+
+echo "033-kernel-bits.binary"
+
+case ${IMAGE_TARGETS-} in
+	""|*squashfs*)
+		;;
+	*)
+		echo "Skipping squashfs build"
+		exit 0
+		;;
+esac
+
+if [ -n "${SUBARCH-}" ]; then
+	echo "Skipping rootfs build for subarch flavor build"
+	exit 0
+fi
+
+. config/functions
+. config/common
+
+INSTALLER_ROOT=binary/boot/installer.squashfs.dir
+
+KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir
+KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits
+
+variants='ga hwe'
+
+for variant in $variants; do
+	if [ "$variant" = "ga" ]; then
+		kernel_metapkg=linux-generic
+		flavor=generic
+	elif [ "$variant" = "hwe" ]; then
+		kernel_metapkg=linux-generic-hwe-$(lsb_release -sr)
+		flavor=generic-hwe
+	else
+		echo "bogus variant: $variant"
+		exit 1
+	fi
+
+	# Make preparations
+	mkdir -p $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
+	mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/"
+	setup_mountpoint $KERNEL_BITS_ROOT
+
+	# Our initramfs hook implements a kind of extreme version of
+	# cloud-initramfs-copymods, so remove that and prevent duelling hooks
+	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y remove cloud-initramfs-copymods ubuntu-server || true
+
+	# Configure initramfs creation
+	mkdir -p "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/
+	if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then
+		echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf
+	fi
+	echo "CASPER_GENERATE_UUID=1" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/casper.conf
+
+	# Add a hook to record which kernel was booted and mount the
+	# modules.squashfs created below.
+	cat <<EOF > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
+#!/bin/sh
+case \$1 in
+prereqs) exit 0;;
+esac
+
+echo ${kernel_metapkg} > /run/kernel-meta-package
+mkdir -p \$rootmnt/lib/modules
+mount \$rootmnt/cdrom/casper/extras/modules.squashfs-$flavor \$rootmnt/lib/modules
+EOF
+	chmod +x "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
+
+	# Install the kernel!
+	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg}
+
+	# Fish out generated kernel image and initrd
+	mv "$KERNEL_BITS_ROOT"/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-$flavor
+	mv "$KERNEL_BITS_ROOT"/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-$flavor
+
+	# Create squashfs containing all the modules
+	modules_squashfs_path="${PWD}/livecd.${PROJECT}.modules.squashfs-$flavor"
+	(cd "$KERNEL_BITS_ROOT/lib/modules" &&
+		mksquashfs . $modules_squashfs_path -no-progress -xattrs -comp xz)
+
+	# And clean up
+	teardown_mountpoint $KERNEL_BITS_ROOT
+	umount $KERNEL_BITS_ROOT
+	rm -rf $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
+done