Merge branch 'layer-fixes' into ubuntu/master

debian/changelog

@@ -1,3 +1,10 @@
+livecd-rootfs (2.729) UNRELEASED; urgency=medium
+
+  * Fix a few things found while porting the liver server ISO builds to
+    layers.
+
+ -- Michael Hudson-Doyle <michael.hudson@ubuntu.com>  Fri, 25 Jun 2021 10:35:47 +1200
+
 livecd-rootfs (2.728) impish; urgency=medium
 
   [ Dan Bungert ]

live-build/auto/config

@@ -1012,8 +1012,8 @@ PASSES=$(_sanitize_passes "$PASSES")
 LIVE_PASSES=${LIVE_PASSES:-$(_get_live_passes "$PASSES")}
 
 if [ -n "$PASSES" ] && [ -z "$LIVE_PASSES" ]; then
-	Echo_warning "Multi-layered mode is enabled, but we didn't find any live pass." \
-	             "Either set \$LIVE_PASSES or add a pass ending with '.live'."
+	echo "W: Multi-layered mode is enabled, but we didn't find any live pass." \
+	     "Either set \$LIVE_PASSES or add a pass ending with '.live'."
 fi
 
 echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot
@@ -1023,6 +1023,9 @@ echo "IMAGEFORMAT=\"$IMAGEFORMAT\"" >> config/chroot
 if [ -n "$PASSES" ]; then
 	echo "PASSES=\"$PASSES\"" >> config/common
 fi
+if [ -n "$NO_SQUASHFS_PASSES" ]; then
+    echo "NO_SQUASHFS_PASSES=\"$NO_SQUASHFS_PASSES\"" >> config/common
+fi
 if [ -n "$LIVE_PASSES" ]; then
 	echo "LIVE_PASSES=\"$LIVE_PASSES\"" >> config/common
 fi

live-build/lb_binary_layered

@@ -66,7 +66,7 @@ build_layered_squashfs () {
 		return
 	fi
 
-	rm -f .build/binary_chroot
+	rm -f .build/binary_chroot .build/binary_hooks
 
 	mkdir -p "$overlay_dir/"
 	lowerdirs=$(get_lowerdirs_for_pass $pass)
@@ -92,39 +92,54 @@ build_layered_squashfs () {
 		rm -f chroot/boot/initrd.img-* chroot/boot/vmlinu{x,z}-*
 	fi
 
-	# Full manifest until that PASS
-	squashfs_f_manifest="${base}.manifest"
-	create_manifest "chroot" "${squashfs_f_manifest}.full"
-
-	# Delta manifest
-	diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
-
-	squashfs_f_size="${base}.size"
-	du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}"
-
-	# We take first live pass for "global" ISO properties (used by installers and checkers):
-	# Prepare initrd + kernel
-	# Main manifest and size files
-	prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
-	if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then
-		totalsize=$(cat ${squashfs_f_size})
-		curpass="$pass"
-		while :; do
-			curpass=$(get_parent_pass $curpass)
-			# We climbed up the tree to the root layer, we are done
-			[ -z "$curpass" ] && break
-
-			totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size"))
-		done
-		echo ${totalsize} > "${prefix}.size"
-
-		cp "${squashfs_f_manifest}.full" "${prefix}.manifest"
+        make_squashfs=yes
+        ppass="${pass}"
+        while :; do
+            for nsp in ${NO_SQUASHFS_PASSES}; do
+                if [ "${ppass}" = "${nsp}" ]; then
+                    make_squashfs=no
+                    break 2
+                fi
+            done
+            ppass="$(get_parent_pass "${ppass}")"
+            if [ -z "${ppass}" ]; then
+                break
+            fi
+        done
+
+        if [ $make_squashfs = yes ]; then
+		# Full manifest until that PASS
+		squashfs_f_manifest="${base}.manifest"
+		create_manifest "chroot" "${squashfs_f_manifest}.full"
+
+		# Delta manifest
+		diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
+
+		squashfs_f_size="${base}.size"
+		du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}"
+
+		# We take first live pass for "global" ISO properties (used by installers and checkers):
+		# Prepare initrd + kernel
+		# Main manifest and size files
+		prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
+		if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then
+			totalsize=$(cat ${squashfs_f_size})
+			curpass="$pass"
+			while :; do
+				curpass=$(get_parent_pass $curpass)
+				# We climbed up the tree to the root layer, we are done
+				[ -z "$curpass" ] && break
+
+				totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size"))
+			done
+			echo ${totalsize} > "${prefix}.size"
+
+			cp "${squashfs_f_manifest}.full" "${prefix}.manifest"
+		fi
+
+		create_squashfs "overlay.${pass}/" ${squashfs_f}
 	fi
 
-	(cd "overlay.${pass}/" &&
-		mksquashfs . ${squashfs_f} \
-		-no-progress -xattrs -comp xz )
-
 	if [ -n "$lowerdirs" ]; then
 		umount chroot
 		rmdir chroot

live-build/lb_chroot_layered

@@ -131,7 +131,7 @@ reduce_pass_size () {
 			continue
 		fi
 		# Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta.
-		Echo_message "reduce_pass_size: '$f' is strictly identical between $parent and $pass. Removing."
+		Echo_message "reduce_pass_size: '%s' is strictly identical between %s and %s. Removing." "$f" "$pass" "$parent"
 		rm "$pass_dir/$f"
 	done
 
@@ -143,7 +143,7 @@ create_chroot_pass () {
 	local pass=$1
 	shift 1 # restore ${*}
 
-	Echo_message "lb_chroot_layered: treating pass $pass"
+	Echo_message "lb_chroot_layered: treating pass %s" "$pass"
 
 	# We have already treated that pass just return.
 	local overlay_dir="overlay.${pass}"
@@ -154,7 +154,7 @@ create_chroot_pass () {
 	# Only get some function executed on root passes
 	# Copy bootstrap on root layers
 	if $(is_root_layer $pass); then
-		rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks
+		rm -f .build/chroot_linux-image .build/chroot_preseed
 		cp -a chroot.bootstrap/ "$overlay_dir/"
 	fi
 	# Others have to be executed on every pass
@@ -196,7 +196,6 @@ create_chroot_pass () {
 	# Only configure universe on root passes
 	if $(is_root_layer $pass); then
 		lb chroot_archives chroot install ${*}
-		configure_universe
 	fi
 
 	# Customizing chroot
@@ -247,13 +246,24 @@ create_chroot_pass () {
 
 	lb chroot_hooks ${*}
 
-	# Run chroot_hacks only on root layers.
-	# chroot_hacks changes the mode of boot/initrd*. The side effect in
-	# layered mode is to create an initrd on each layer with a significant
-	# impact on image size (+30MB per layer). This is an issue even with
-	# update-initramfs disabled.
+	# Run chroot_hacks only on live layers.  chroot_hacks is what repacks
+	# the initrd in a full casperized mode. We don't want to do that for
+	# each and every layer, just for the live layers (which
+	# lb_binary_layered will call lb binary_linux-image on).
 	if $(is_live_layer $pass); then
+		rm -f .build/chroot_hacks
 		lb chroot_hacks ${*}
+	else
+		# chroot_hacks also removes some cruft, which is appropriate for
+		# any layer so we copy and paste that bit here.
+		rm -f chroot/boot/initrd*bak*
+		rm -f chroot/etc/apt/trusted.gpg~
+		rm -f chroot/etc/group- chroot/etc/passwd-
+		rm -f chroot/etc/gshadow- chroot/etc/shadow-
+		rm -f chroot/var/cache/debconf/*-old
+		rm -f chroot/var/lib/dpkg/*-old
+		rm -f chroot/var/log/apt/term.log
+		rm -f chroot/etc/adjtime
 	fi
 
 	lb chroot_interactive ${*}
@@ -267,6 +277,7 @@ create_chroot_pass () {
 	# Deconfiguring chroot
 	if $(is_root_layer $pass); then
 		lb chroot_archives chroot remove ${*}
+		configure_universe
 	fi
 	lb chroot_apt remove ${*}
 	lb chroot_hostname remove ${*}