Deduplicate the layers just before creating the squashfs, to enable deduplication of the result of binary hooks as well.

debian/changelog

@@ -1,3 +1,10 @@
+livecd-rootfs (23.10.21) UNRELEASED; urgency=medium
+
+  * Deduplicate the layers just before creating the squashfs, to enable
+    deduplication of the result of binary hooks as well.
+
+ -- Michael Hudson-Doyle <michael.hudson@ubuntu.com>  Wed, 23 Aug 2023 11:15:26 +1200
+
 livecd-rootfs (23.10.20) mantic; urgency=medium
 
   * Update canary model assertions to include all the regular seeded snaps,

live-build/lb_binary_layered

@@ -142,7 +142,23 @@ build_layered_squashfs () {
 			cp "${squashfs_f_manifest}.full" "${prefix}.manifest"
 		fi
 
-		create_squashfs "overlay.${pass}/" ${squashfs_f}
+		if [ -n "$lowerdirs" ]; then
+			# Although the current chroot was created as an overlay over
+			# the previous layer, many operations can result in redundant
+			# files in the upperdir. Rather than trying to minimize the
+			# overlay by hand, we rsync the chroot into a fresh overlay,
+			# rely on rsyncs ability to avoid redundant file operations,
+			# and take _that_ overlay's upperdir as the content of the
+			# layer.
+			mkdir -p chroot-2 "$overlay_dir-2"
+			mount_overlay "$lowerdirs" "$overlay_dir-2" chroot-2
+			rsync -caXHAS --no-t --del chroot/ chroot-2/
+			umount chroot-2
+			rmdir chroot-2
+			overlay_dir="$overlay_dir-2"
+		fi
+
+		create_squashfs "${overlay_dir}" ${squashfs_f}
 
 		if [ -f config/$pass.catalog-in.yaml ]; then
 			echo "Expanding catalog entry template for $pass"

live-build/lb_chroot_layered

@@ -131,8 +131,8 @@ create_chroot_pass () {
 	mkdir -p "$overlay_dir/"
 	lowerdirs=$(get_lowerdirs_for_pass $pass)
 	if [ -n "$lowerdirs" ]; then
-		mkdir -p chroot "${overlay_dir}-initial"
+		mkdir -p chroot/
-		mount_overlay "$lowerdirs" "${overlay_dir}-initial" chroot/
+		mount_overlay "$lowerdirs" "$overlay_dir" chroot/
 	else
 		ln -s "$overlay_dir/" chroot
 	fi
@@ -268,19 +268,8 @@ create_chroot_pass () {
 	lb chroot_devpts remove ${*}
 
 	if [ -n "$lowerdirs" ]; then
-		# Although the current chroot was created as an overlay over
+		umount chroot
-		# the previous layer, many operations can result in redundant
+		rmdir chroot
-		# files in the upperdir. Rather than trying to minimize the
-		# overlay by hand, we rsync the chroot into a fresh overlay,
-		# rely on rsyncs ability to avoid redundant file operations,
-		# and take _that_ overlay's upperdir as the content of the
-		# layer.
-		mkdir chroot-2
-		mount_overlay "$lowerdirs" "$overlay_dir" chroot-2/
-		rsync -aXHAS --del chroot/ chroot-2/
-		umount chroot chroot-2
-		rmdir chroot chroot-2
-		rm -rf ${overlay_dir}-initial
 	else
 		rm chroot
 	fi