Merge branch 'move-live-server-to-layers-3' into ubuntu/master

debian/changelog

@@ -1,8 +1,9 @@
 livecd-rootfs (2.729) UNRELEASED; urgency=medium
 
   [ Michael Hudson-Doyle ]
-  * Fix a few things found while porting the liver server ISO builds to
+  * Fix a few things found while porting the live server ISO builds to
     layers.
+  * Port live server ISO builds to layers.
 
   [ Chloé 'kajiya' Smith ]
   * Image hangs during boot after deploying groovy-server-cloudimg-amd64.ova,

live-build/auto/config

@@ -764,12 +764,47 @@ case $PROJECT in
 		;;
 
 	ubuntu-server)
-		add_task install minimal
 		case ${SUBPROJECT:-} in
 			live)
-				add_task install standard
-				add_task install server
-				add_package install cloud-init
+				PASSES_TO_LAYERS=true
+				add_task ubuntu-server minimal standard server
+				# add_task really should do this itself but for now...
+				snap_from_seed server config/package-lists/livecd-rootfs.snaplist.chroot_ubuntu-server.full
+				add_package ubuntu-server cloud-init
+
+				add_package ubuntu-server.installer linux-firmware lupin-casper openssh-server
+				add_snap ubuntu-server.installer subiquity/classic
+				if [ $ARCH = s390x ]; then
+					add_package ubuntu-server.installer s390-tools-zkey
+				fi
+
+				# Live server ISOs for LTS point releases past .2 offer both
+				# the GA and HWE kernels (in separate layers) so this code is
+				# written generically to support both even though a lot of the
+				# time only one kernel is offered.
+
+				# variants='ga hwe'
+				variants='ga'
+
+				for variant in $variants; do
+					if [ "$variant" = "ga" ]; then
+						kernel_metapkg=linux-generic
+						flavor=generic
+					elif [ "$variant" = "hwe" ]; then
+						kernel_metapkg=linux-generic-hwe-$(lsb_release -sr)
+						flavor=generic-hwe
+					else
+						echo "bogus variant: $variant"
+						exit 1
+					fi
+
+					add_package ubuntu-server.installer.$flavor $kernel_metapkg
+					LIVE_PASSES="${LIVE_PASSES:+$LIVE_PASSES }ubuntu-server.installer.$flavor"
+				done
+				;;
+			*)
+				echo "unrecognized subproject for server: '$SUBPROJECT'"
+				exit 1
 				;;
 		esac
 		COMPONENTS='main'

live-build/lb_chroot_layered

@@ -130,6 +130,25 @@ reduce_pass_size () {
 		if ! diff --brief --no-dereference "$pass_dir/$f" "$parent_pass_dir/$f" >/dev/null; then
 			continue
 		fi
+		p=$(dirname $f)
+		while [ "$p" != . ]; do
+			# As explained in the overlayfs documentation
+			# https://www.kernel.org/doc/html/latest/filesystems/overlayfs.html#whiteouts-and-opaque-directories
+			# an xattr of trusted.overlay.opaque indicates an 'opaque' directory
+			# that was deleted from the overlay. Removing files from within the
+			# directory, even if identical with one in the lower layer, will result
+			# in it going missing from the combined filesystem.
+			#
+			# It would be possible to be more clever, e.g. if the two directories
+			# are still similar, we could the delete the attribute and convert any
+			# still-needed deletions to whiteouts but it doesn't seem worth it in
+			# the cases seen so far.
+			if [ -n "$(getfattr -n trusted.overlay.opaque -h --only-values -- "$pass_dir/$p" 2>/dev/null)" ]; then
+				Echo_message "reduce_pass_size: '%s' would be removed from overlay but for trusted.overlay.opaque on '%s'" "$f" "$p"
+				continue 2
+			fi
+			p=$(dirname $p)
+		done
 		# Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta.
 		Echo_message "reduce_pass_size: '%s' is strictly identical between %s and %s. Removing." "$f" "$pass" "$parent"
 		rm "$pass_dir/$f"

live-build/ubuntu-server/hooks/02-installer-bits.chroot

@@ -0,0 +1,17 @@
+#!/bin/bash -ex
+# vi: ts=4 noexpandtab
+
+if [ "${PASS}" != "ubuntu-server.installer" ]; then
+    exit 0
+fi
+
+# Make sure NoCloud is last
+values=$(echo get cloud-init/datasources | debconf-communicate | sed 's/^0 //;s/NoCloud, //;s/None/NoCloud, None/')
+printf "%s\t%s\t%s\t%s\n" \
+   cloud-init cloud-init/datasources multiselect  "$values" | debconf-set-selections
+dpkg-reconfigure --frontend=noninteractive cloud-init
+
+if [ `dpkg --print-architecture` = s390x ]; then
+    # because z/VM x3270 is just ttyS0
+    cp -r /usr/lib/systemd/system/serial-getty@sclp_line0.service.d /usr/lib/systemd/system/serial-getty@ttyS0.service.d
+fi

live-build/ubuntu-server/hooks/03-kernel-metapkg.chroot_early

@@ -0,0 +1,37 @@
+#!/bin/bash -ex
+# vi: ts=4 noexpandtab
+
+case $PASS in
+    ubuntu-server.installer.*)
+        flavor=${PASS##*.}
+        if [ "$flavor" = "generic" ]; then
+            kernel_metapkg=linux-generic
+        elif [ "$flavor" = "generic-hwe" ]; then
+            kernel_metapkg=linux-generic-hwe-$(lsb_release -sr)
+        else
+            echo "bogus flavor: $flavor"
+            exit 1
+        fi
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+cat <<EOF > /etc/initramfs-tools/scripts/init-bottom/live-server
+#!/bin/sh
+case \$1 in
+prereqs) exit 0;;
+esac
+
+echo ${kernel_metapkg} > /run/kernel-meta-package
+EOF
+chmod +x /etc/initramfs-tools/scripts/init-bottom/live-server
+
+cat <<EOF > /etc/initramfs-tools/conf.d/casperize.conf
+export CASPER_GENERATE_UUID=1
+EOF
+
+cat <<EOF > /etc/initramfs-tools/conf.d/default-layer.conf
+LAYERFS_PATH=${PASS}.squashfs
+EOF

live-build/ubuntu-server/hooks/030-root-squashfs.binary

@@ -1,33 +0,0 @@
-#!/bin/bash -ex
-# vi: ts=4 noexpandtab
-#
-# Generate a squashfs root and manifest
-
-set -x
-echo "030-root-squashfs.binary"
-
-case $IMAGE_TARGETS in
-	""|*squashfs*)
-		;;
-	*)
-		echo "Skipping squashfs build"
-		exit 0
-		;;
-esac
-
-if [ -n "$SUBARCH" ]; then
-	echo "Skipping rootfs build for subarch flavor build"
-	exit 0
-fi
-
-. config/functions
-
-mkdir binary/boot/squashfs.dir
-cp -a chroot/* binary/boot/squashfs.dir
-
-squashfs_f="${PWD}/livecd.${PROJECT}.squashfs"
-squashfs_f_manifest="${squashfs_f}.manifest"
-
-create_manifest "binary/boot/squashfs.dir" "${squashfs_f_manifest}"
-
-create_squashfs "binary/boot/squashfs.dir/" ${squashfs_f}

live-build/ubuntu-server/hooks/032-installer-squashfs.binary

@@ -1,97 +0,0 @@
-#!/bin/bash -ex
-# vi: ts=4 noexpandtab
-#
-# Generate a squashfs root and manifest
-
-set -x
-
-echo "032-installer-squashfs.binary"
-
-case $IMAGE_TARGETS in
-	""|*squashfs*)
-		;;
-	*)
-		echo "Skipping squashfs build"
-		exit 0
-		;;
-esac
-
-if [ -n "$SUBARCH" ]; then
-	echo "Skipping rootfs build for subarch flavor build"
-	exit 0
-fi
-
-. config/binary
-. config/functions
-. config/common
-
-FILESYSTEM_ROOT=binary/boot/squashfs.dir
-INSTALLER_ROOT=binary/boot/installer.squashfs.dir
-OVERLAY_ROOT=binary/overlay
-
-mkdir -p "$INSTALLER_ROOT" "$OVERLAY_ROOT"
-
-# Create an installer squashfs layer
-mount_overlay "$FILESYSTEM_ROOT/" "$OVERLAY_ROOT/" "$INSTALLER_ROOT/"
-
-setup_mountpoint "$INSTALLER_ROOT"
-
-# Override JobRunningTimeoutSec to 0s on the .device unit that
-# subiquity_config.mount depends on to avoid a 5s delay on switching
-# to a new VT when there is no device there (LP: #1750117).
-# It would be better to have this in ../includes.binary/overlay but
-# you can't have backslashes in filenames in bzr branches!
-ANSWERS_DEVICE_UNIT='dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device'
-mkdir -p "$INSTALLER_ROOT/etc/systemd/system/$ANSWERS_DEVICE_UNIT.d"
-cat > "$INSTALLER_ROOT/etc/systemd/system/$ANSWERS_DEVICE_UNIT.d/override.conf" <<EOF
-[Unit]
-JobRunningTimeoutSec=0s
-Wants=subiquity_config.mount
-EOF
-
-# Prepare installer layer.
-
-# Install:
-#   1. linux-firmware for kernel to upload into hardware.
-#   2. casper for live session magic.
-#   3. openssh-server to enable the "ssh into live session" feature
-chroot $INSTALLER_ROOT apt-get -y install linux-firmware lupin-casper openssh-server
-
-# Make sure NoCloud is last
-values=$(echo get cloud-init/datasources | chroot $INSTALLER_ROOT debconf-communicate | sed 's/^0 //;s/NoCloud, //;s/None/NoCloud, None/')
-printf "%s\t%s\t%s\t%s\n" \
-   cloud-init cloud-init/datasources multiselect  "$values" |
-      chroot $INSTALLER_ROOT debconf-set-selections
-chroot $INSTALLER_ROOT dpkg-reconfigure --frontend=noninteractive cloud-init
-
-if [ `dpkg --print-architecture` = s390x ]; then
-    chroot $INSTALLER_ROOT apt-get -y install s390-tools-zkey
-    # because z/VM x3270 is just ttyS0
-    cp -r $INSTALLER_ROOT/usr/lib/systemd/system/serial-getty@sclp_line0.service.d $INSTALLER_ROOT/usr/lib/systemd/system/serial-getty@ttyS0.service.d
-fi
-chroot $INSTALLER_ROOT apt-get clean
-
-# For bug #1743643 "Install to dirty disk with swap fails" remove the
-# "helpful" casper script that mounts any swap partitions it finds.
-rm -f $INSTALLER_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
-
-# Preseed subiquity into installer layer
-snap_prepare $INSTALLER_ROOT
-snap_preseed $INSTALLER_ROOT subiquity/classic
-# Drop lxd from the installer layer preseed
-sed -i -e'N;/name: lxd/,+2d' $INSTALLER_ROOT/var/lib/snapd/seed/seed.yaml
-
-teardown_mountpoint "$INSTALLER_ROOT"
-
-# Drop any snaps that got copied up from base layer, due to
-# snap-preseed tool doing --reset & speedup
-for snap in $(cd $OVERLAY_ROOT/; find var/lib/snapd/ -name '*.snap'); do
-    if [ -f $FILESYSTEM_ROOT/$snap ]; then
-        echo "removing $snap from overlay as already present in base layer"
-        rm $OVERLAY_ROOT/$snap
-    fi
-done
-
-squashfs_f="${PWD}/livecd.${PROJECT}.installer.squashfs"
-
-create_squashfs "$OVERLAY_ROOT/" ${squashfs_f}

live-build/ubuntu-server/hooks/033-kernel-bits.binary

@@ -1,94 +0,0 @@
-#!/bin/bash -eux
-# vi: ts=4 noexpandtab
-#
-# Generate a squashfs root and manifest
-
-echo "033-kernel-bits.binary"
-
-case ${IMAGE_TARGETS-} in
-	""|*squashfs*)
-		;;
-	*)
-		echo "Skipping squashfs build"
-		exit 0
-		;;
-esac
-
-if [ -n "${SUBARCH-}" ]; then
-	echo "Skipping rootfs build for subarch flavor build"
-	exit 0
-fi
-
-. config/functions
-. config/common
-
-INSTALLER_ROOT=binary/boot/installer.squashfs.dir
-
-KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir
-KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits
-
-#variants='ga hwe'
-variants='ga'
-
-for variant in $variants; do
-	if [ "$variant" = "ga" ]; then
-		kernel_metapkg=linux-generic
-		flavor=generic
-	elif [ "$variant" = "hwe" ]; then
-		kernel_metapkg=linux-generic-hwe-$(lsb_release -sr)
-		flavor=generic-hwe
-	else
-		echo "bogus variant: $variant"
-		exit 1
-	fi
-
-	# Make preparations
-	mkdir -p $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
-	mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/"
-	setup_mountpoint $KERNEL_BITS_ROOT
-
-	# Our initramfs hook implements a kind of extreme version of
-	# cloud-initramfs-copymods, so remove that and prevent duelling hooks
-	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y remove cloud-initramfs-copymods ubuntu-server || true
-
-	# Configure initramfs creation
-	mkdir -p "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/
-	if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then
-		echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf
-	fi
-	echo "export CASPER_GENERATE_UUID=1" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/casper.conf
-
-	# Add a hook to record which kernel was booted and mount the
-	# modules.squashfs created below.
-	cat <<EOF > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
-#!/bin/sh
-case \$1 in
-prereqs) exit 0;;
-esac
-
-echo ${kernel_metapkg} > /run/kernel-meta-package
-mkdir -p \$rootmnt/usr/lib/modules
-mount \$rootmnt/cdrom/casper/extras/modules.squashfs-$flavor \$rootmnt/usr/lib/modules
-mkdir -p /run/systemd/system/usr-lib-modules.mount.d
-echo '[Mount]' >> /run/systemd/system/usr-lib-modules.mount.d/lazy.conf
-echo 'LazyUnmount=yes' >> /run/systemd/system/usr-lib-modules.mount.d/lazy.conf
-EOF
-	chmod +x "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
-
-	# Install the kernel!
-	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg}
-
-	# Fish out generated kernel image and initrd
-	mv "$KERNEL_BITS_ROOT"/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-$flavor
-	mv "$KERNEL_BITS_ROOT"/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-$flavor
-
-	# Create squashfs containing all the modules
-	modules_squashfs_path="${PWD}/livecd.${PROJECT}.modules.squashfs-$flavor"
-	(cd "$KERNEL_BITS_ROOT/lib/modules" &&
-		mksquashfs . $modules_squashfs_path -no-progress -xattrs -comp xz)
-
-	# And clean up
-	teardown_mountpoint $KERNEL_BITS_ROOT
-	umount $KERNEL_BITS_ROOT
-	rm -rf $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
-done

live-build/ubuntu-server/hooks/04-kernel-bits.binary

@@ -0,0 +1,16 @@
+#!/bin/bash -eux
+# vi: ts=4 noexpandtab
+
+case $PASS in
+    ubuntu-server.installer.*)
+        flavor=${PASS##*.}
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+# Fish out generated kernel image and initrd
+mv chroot/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-$flavor
+mv chroot/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-$flavor
+chmod a+r ${PWD}/livecd.${PROJECT}.initrd-$flavor ${PWD}/livecd.${PROJECT}.kernel-$flavor

live-build/ubuntu-server/includes.chroot.ubuntu-server.installer/etc/systemd/system/dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device.d/override.conf

@@ -0,0 +1,3 @@
+[Unit]
+JobRunningTimeoutSec=0s
+Wants=subiquity_config.mount

live-build/ubuntu-server/includes.chroot.ubuntu-server.installer/usr/lib/systemd/system/media-filesystem.mount

@@ -1,5 +1,5 @@
 [Mount]
-What=/cdrom/casper/filesystem.squashfs
+What=/cdrom/casper/ubuntu-server.squashfs
 Where=/media/filesystem
 Type=squashfs
 Options=ro