diff --git a/debian/changelog b/debian/changelog index 025befe2..4d0db9e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +livecd-rootfs (23.10.21) mantic; urgency=medium + + * Deduplicate the layers just before creating the squashfs, to enable + deduplication of the result of binary hooks as well. + + [ Steve Langasek ] + * Deduplicate snaps between squashfs layers on classic: + - Consolidate canary layers to merge 'classic' back into 'standard' + - Reset /var/lib/snapd in the upper layers before calling snap + prepare-image + - No need to use rsync in hooks now to avoid file duplication since layer + handling as a whole now uses rsync. + + -- Steve Langasek Wed, 23 Aug 2023 10:31:42 -0700 + livecd-rootfs (23.10.20) mantic; urgency=medium * Update canary model assertions to include all the regular seeded snaps, diff --git a/live-build/auto/config b/live-build/auto/config index 8cc08676..c2e969fc 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -723,15 +723,9 @@ case $PROJECT in ;; canary) PASSES_TO_LAYERS="true" - # the standard layer, contains all base common packages for later layers (we're splitting out the snaps) + # the standard layer, contains all base common packages for later layers add_task standard minimal standard ubuntu-desktop ubuntu-desktop-default-languages add_package standard cloud-init - # the classic layer, basically only contains snaps from the standard and minimal tasks - add_pass standard.classic - if [ -e "config/package-lists/livecd-rootfs.snaplist.chroot_standard.full" ]; then - cat config/package-lists/livecd-rootfs.snaplist.chroot_standard.full >>config/package-lists/livecd-rootfs.snaplist.chroot_standard.classic.full - rm config/package-lists/livecd-rootfs.snaplist.chroot_standard.full - fi # the live layer, contains all packages for the live session installer # TODO: we should probably add the kernel per KERNEL_FLAVOURS add_package standard.live linux-generic casper lvm2 mdadm cryptsetup dctrl-tools @@ -742,7 +736,7 @@ case $PROJECT in # language support seeded_langs="$(get_seeded_languages desktop)" echo "$seeded_langs" | tr ' ' ',' > config/seeded-languages - derive_language_layers standard.classic desktop desktop-default-languages "$seeded_langs" + derive_language_layers standard desktop desktop-default-languages "$seeded_langs" derive_language_layers standard.enhanced-secureboot desktop desktop-default-languages "$seeded_langs" # now let's create the neccessary catalog files @@ -755,12 +749,9 @@ case $PROJECT in variant: desktop locale_support: langpack default: yes - EOF - cat <<-EOF > config/standard.classic.catalog-in.yaml - id: ubuntu-desktop variations: - classic: - path: standard.classic.squashfs + standard: + path: standard.squashfs EOF cat <<-EOF > config/standard.enhanced-secureboot.catalog-in.yaml id: ubuntu-desktop diff --git a/live-build/functions b/live-build/functions index dea971dc..01ea76f8 100644 --- a/live-build/functions +++ b/live-build/functions @@ -1230,3 +1230,14 @@ EOF undivert_grub "${mountpoint}" fi } + +# find all files under /var/lib/snapd in the target directory that aren't +# shipped by the snapd package itself, and remove them +reset_snapd_state() { + rootdir="$1" + + rm -rf "$rootdir/var/lib/snapd" + setup_mountpoint "$rootdir" + chroot "$rootdir" apt-get install --reinstall -y snapd + teardown_mountpoint "$rootdir" +} diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index 146978d3..6c00f96b 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -142,7 +142,39 @@ build_layered_squashfs () { cp "${squashfs_f_manifest}.full" "${prefix}.manifest" fi - create_squashfs "overlay.${pass}/" ${squashfs_f} + if [ -n "$lowerdirs" ]; then + # Although the current chroot was created as an overlay over + # the previous layer, many operations can result in redundant + # files in the upperdir. Rather than trying to minimize the + # overlay by hand, we rsync the chroot into a fresh overlay, + # rely on rsyncs ability to avoid redundant file operations, + # and take _that_ overlay's upperdir as the content of the + # layer. + mkdir -p chroot-2 "$overlay_dir-2" + mount_overlay "$lowerdirs" "$overlay_dir-2" chroot-2 + # rsync takes many, many options. The subset + # we pass here is quite important: + # -a is standard to operate in the basic way required here. + # -X to preserve xattrs + # -H to preserve hardlinks + # -A to preserve ACLs + # -S to preserve sparseness + # --checksum to skip copies based on the content of the file + # (rather than the default which is to skip copies based + # on size + mtime) + # --no-times to not copy mtimes from source to dest (we + # don't care about mtime in the image and want to + # deduplicate files that have indentical contents but + # different mtimes) + # --del because we want to remove files that have been + # deleted in this layer. + rsync -aXHAS --checksum --no-times --del chroot/ chroot-2/ + umount chroot-2 + rmdir chroot-2 + overlay_dir="$overlay_dir-2" + fi + + create_squashfs "${overlay_dir}" ${squashfs_f} if [ -f config/$pass.catalog-in.yaml ]; then echo "Expanding catalog entry template for $pass" diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index d3173755..eb55131f 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -131,8 +131,8 @@ create_chroot_pass () { mkdir -p "$overlay_dir/" lowerdirs=$(get_lowerdirs_for_pass $pass) if [ -n "$lowerdirs" ]; then - mkdir -p chroot "${overlay_dir}-initial" - mount_overlay "$lowerdirs" "${overlay_dir}-initial" chroot/ + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ else ln -s "$overlay_dir/" chroot fi @@ -268,19 +268,8 @@ create_chroot_pass () { lb chroot_devpts remove ${*} if [ -n "$lowerdirs" ]; then - # Although the current chroot was created as an overlay over - # the previous layer, many operations can result in redundant - # files in the upperdir. Rather than trying to minimize the - # overlay by hand, we rsync the chroot into a fresh overlay, - # rely on rsyncs ability to avoid redundant file operations, - # and take _that_ overlay's upperdir as the content of the - # layer. - mkdir chroot-2 - mount_overlay "$lowerdirs" "$overlay_dir" chroot-2/ - rsync -aXHAS --del chroot/ chroot-2/ - umount chroot chroot-2 - rmdir chroot chroot-2 - rm -rf ${overlay_dir}-initial + umount chroot + rmdir chroot else rm chroot fi diff --git a/live-build/ubuntu/hooks/020-canary-enhanced-sb.binary b/live-build/ubuntu/hooks/020-canary-enhanced-sb.binary index e1b41ad8..8e1f6239 100644 --- a/live-build/ubuntu/hooks/020-canary-enhanced-sb.binary +++ b/live-build/ubuntu/hooks/020-canary-enhanced-sb.binary @@ -20,6 +20,7 @@ case ${SUBPROJECT:-} in esac . config/binary +. config/functions # env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-classic-2310-amd64 > config/classic-model.model cat < config/classic-model.model @@ -134,8 +135,11 @@ channel="" if [ -n "${CHANNEL:-}" ]; then channel="--channel $CHANNEL" fi + +reset_snapd_state chroot + env SNAPPY_STORE_NO_CDN=1 snap prepare-image \ --classic config/classic-model.model $channel chroot mv chroot/system-seed/systems/* chroot/system-seed/systems/enhanced-secureboot-desktop -rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed -rm -rf chroot/system-seed/ +rm -rf chroot/var/lib/snapd/seed +mv chroot/system-seed chroot/var/lib/snapd/seed diff --git a/live-build/ubuntu/hooks/020-canary-live.binary b/live-build/ubuntu/hooks/020-canary-live.binary index a382587f..955e738d 100755 --- a/live-build/ubuntu/hooks/020-canary-live.binary +++ b/live-build/ubuntu/hooks/020-canary-live.binary @@ -20,6 +20,7 @@ case ${SUBPROJECT:-} in esac . config/binary +. config/functions # env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-installer-classic-2310-amd64 > config/classic-model-installer.model cat < config/classic-model-installer.model @@ -130,11 +131,14 @@ channel="" if [ -n "${CHANNEL:-}" ]; then channel="--channel $CHANNEL" fi + +reset_snapd_state chroot + env SNAPPY_STORE_NO_CDN=1 snap prepare-image \ --classic config/classic-model-installer.model $channel chroot mv chroot/system-seed/systems/* chroot/system-seed/systems/classic-installer -rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed -rm -rf chroot/system-seed/ +rm -rf chroot/var/lib/snapd/seed +mv chroot/system-seed chroot/var/lib/snapd/seed cat < chroot/var/lib/snapd/modeenv mode=run