Imported 23.10.21

No reason for CPC update specified.

debian/changelog

@@ -1,3 +1,18 @@
+livecd-rootfs (23.10.21) mantic; urgency=medium
+
+  * Deduplicate the layers just before creating the squashfs, to enable
+    deduplication of the result of binary hooks as well.
+
+  [ Steve Langasek ]
+  * Deduplicate snaps between squashfs layers on classic:
+    - Consolidate canary layers to merge 'classic' back into 'standard'
+    - Reset /var/lib/snapd in the upper layers before calling snap
+      prepare-image
+    - No need to use rsync in hooks now to avoid file duplication since layer
+      handling as a whole now uses rsync.
+
+ -- Steve Langasek <steve.langasek@ubuntu.com>  Wed, 23 Aug 2023 10:31:42 -0700
+
 livecd-rootfs (23.10.20) mantic; urgency=medium
 
   * Update canary model assertions to include all the regular seeded snaps,

live-build/auto/config

@@ -723,15 +723,9 @@ case $PROJECT in
 				;;
 			canary)
 				PASSES_TO_LAYERS="true"
-				# the standard layer, contains all base common packages for later layers (we're splitting out the snaps)
+				# the standard layer, contains all base common packages for later layers
 				add_task standard minimal standard ubuntu-desktop ubuntu-desktop-default-languages
 				add_package standard cloud-init
-				# the classic layer, basically only contains snaps from the standard and minimal tasks
-				add_pass standard.classic
-				if [ -e "config/package-lists/livecd-rootfs.snaplist.chroot_standard.full" ]; then
-					cat config/package-lists/livecd-rootfs.snaplist.chroot_standard.full >>config/package-lists/livecd-rootfs.snaplist.chroot_standard.classic.full
-					rm config/package-lists/livecd-rootfs.snaplist.chroot_standard.full
-				fi
 				# the live layer, contains all packages for the live session installer
 				# TODO: we should probably add the kernel per KERNEL_FLAVOURS
 				add_package standard.live linux-generic casper lvm2 mdadm cryptsetup dctrl-tools
@@ -742,7 +736,7 @@ case $PROJECT in
 				# language support
 				seeded_langs="$(get_seeded_languages desktop)"
 				echo "$seeded_langs" | tr ' ' ',' > config/seeded-languages
-				derive_language_layers standard.classic desktop desktop-default-languages "$seeded_langs"
+				derive_language_layers standard desktop desktop-default-languages "$seeded_langs"
 				derive_language_layers standard.enhanced-secureboot desktop desktop-default-languages "$seeded_langs"
 
 				# now let's create the neccessary catalog files
@@ -755,12 +749,9 @@ case $PROJECT in
 					variant: desktop
 					locale_support: langpack
 					default: yes
-				EOF
-				cat <<-EOF > config/standard.classic.catalog-in.yaml
-					id: ubuntu-desktop
 					variations:
-					  classic:
+					  standard:
-					    path: standard.classic.squashfs
+					    path: standard.squashfs
 				EOF
 				cat <<-EOF > config/standard.enhanced-secureboot.catalog-in.yaml
 					id: ubuntu-desktop

live-build/functions

@@ -1230,3 +1230,14 @@ EOF
         undivert_grub "${mountpoint}"
     fi
 }
+
+# find all files under /var/lib/snapd in the target directory that aren't
+# shipped by the snapd package itself, and remove them
+reset_snapd_state() {
+    rootdir="$1"
+
+    rm -rf "$rootdir/var/lib/snapd"
+    setup_mountpoint "$rootdir"
+    chroot "$rootdir" apt-get install --reinstall -y snapd
+    teardown_mountpoint "$rootdir"
+}

live-build/lb_binary_layered

@@ -142,7 +142,39 @@ build_layered_squashfs () {
 			cp "${squashfs_f_manifest}.full" "${prefix}.manifest"
 		fi
 
-		create_squashfs "overlay.${pass}/" ${squashfs_f}
+		if [ -n "$lowerdirs" ]; then
+			# Although the current chroot was created as an overlay over
+			# the previous layer, many operations can result in redundant
+			# files in the upperdir. Rather than trying to minimize the
+			# overlay by hand, we rsync the chroot into a fresh overlay,
+			# rely on rsyncs ability to avoid redundant file operations,
+			# and take _that_ overlay's upperdir as the content of the
+			# layer.
+			mkdir -p chroot-2 "$overlay_dir-2"
+			mount_overlay "$lowerdirs" "$overlay_dir-2" chroot-2
+			# rsync takes many, many options. The subset
+			# we pass here is quite important:
+			#  -a is standard to operate in the basic way required here.
+			#  -X to preserve xattrs
+			#  -H to preserve hardlinks
+			#  -A to preserve ACLs
+			#  -S to preserve sparseness
+			#  --checksum to skip copies based on the content of the file
+			#    (rather than the default which is to skip copies based
+			#    on size + mtime)
+			#  --no-times to not copy mtimes from source to dest (we
+			#    don't care about mtime in the image and want to
+			#    deduplicate files that have indentical contents but
+			#    different mtimes)
+			#  --del because we want to remove files that have been
+			#    deleted in this layer.
+			rsync -aXHAS --checksum --no-times --del chroot/ chroot-2/
+			umount chroot-2
+			rmdir chroot-2
+			overlay_dir="$overlay_dir-2"
+		fi
+
+		create_squashfs "${overlay_dir}" ${squashfs_f}
 
 		if [ -f config/$pass.catalog-in.yaml ]; then
 			echo "Expanding catalog entry template for $pass"

live-build/lb_chroot_layered

@@ -131,8 +131,8 @@ create_chroot_pass () {
 	mkdir -p "$overlay_dir/"
 	lowerdirs=$(get_lowerdirs_for_pass $pass)
 	if [ -n "$lowerdirs" ]; then
-		mkdir -p chroot "${overlay_dir}-initial"
+		mkdir -p chroot/
-		mount_overlay "$lowerdirs" "${overlay_dir}-initial" chroot/
+		mount_overlay "$lowerdirs" "$overlay_dir" chroot/
 	else
 		ln -s "$overlay_dir/" chroot
 	fi
@@ -268,19 +268,8 @@ create_chroot_pass () {
 	lb chroot_devpts remove ${*}
 
 	if [ -n "$lowerdirs" ]; then
-		# Although the current chroot was created as an overlay over
+		umount chroot
-		# the previous layer, many operations can result in redundant
+		rmdir chroot
-		# files in the upperdir. Rather than trying to minimize the
-		# overlay by hand, we rsync the chroot into a fresh overlay,
-		# rely on rsyncs ability to avoid redundant file operations,
-		# and take _that_ overlay's upperdir as the content of the
-		# layer.
-		mkdir chroot-2
-		mount_overlay "$lowerdirs" "$overlay_dir" chroot-2/
-		rsync -aXHAS --del chroot/ chroot-2/
-		umount chroot chroot-2
-		rmdir chroot chroot-2
-		rm -rf ${overlay_dir}-initial
 	else
 		rm chroot
 	fi

live-build/ubuntu/hooks/020-canary-enhanced-sb.binary

@@ -20,6 +20,7 @@ case ${SUBPROJECT:-} in
 esac
 
 . config/binary
+. config/functions
 
 # env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-classic-2310-amd64 > config/classic-model.model
 cat <<EOF > config/classic-model.model
@@ -134,8 +135,11 @@ channel=""
 if [ -n "${CHANNEL:-}" ]; then
     channel="--channel $CHANNEL"
 fi
+
+reset_snapd_state chroot
+
 env SNAPPY_STORE_NO_CDN=1 snap prepare-image \
     --classic config/classic-model.model $channel chroot
 mv chroot/system-seed/systems/* chroot/system-seed/systems/enhanced-secureboot-desktop
-rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed
+rm -rf chroot/var/lib/snapd/seed
-rm -rf chroot/system-seed/
+mv chroot/system-seed chroot/var/lib/snapd/seed

live-build/ubuntu/hooks/020-canary-live.binary

@@ -20,6 +20,7 @@ case ${SUBPROJECT:-} in
 esac
 
 . config/binary
+. config/functions
 
 # env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-installer-classic-2310-amd64 > config/classic-model-installer.model
 cat <<EOF > config/classic-model-installer.model
@@ -130,11 +131,14 @@ channel=""
 if [ -n "${CHANNEL:-}" ]; then
     channel="--channel $CHANNEL"
 fi
+
+reset_snapd_state chroot
+
 env SNAPPY_STORE_NO_CDN=1 snap prepare-image \
     --classic config/classic-model-installer.model $channel chroot
 mv chroot/system-seed/systems/* chroot/system-seed/systems/classic-installer
-rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed
+rm -rf chroot/var/lib/snapd/seed
-rm -rf chroot/system-seed/
+mv chroot/system-seed chroot/var/lib/snapd/seed
 
 cat <<EOF  > chroot/var/lib/snapd/modeenv
 mode=run