From 75165aeb349fc56fd13e36c681fc51dbb8752451 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:16:12 +1200 Subject: [PATCH 01/10] replace call to Echo_warning with plan echo as former is not defined at this point --- live-build/auto/config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 196160d2..6ddde4ea 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -1012,8 +1012,8 @@ PASSES=$(_sanitize_passes "$PASSES") LIVE_PASSES=${LIVE_PASSES:-$(_get_live_passes "$PASSES")} if [ -n "$PASSES" ] && [ -z "$LIVE_PASSES" ]; then - Echo_warning "Multi-layered mode is enabled, but we didn't find any live pass." \ - "Either set \$LIVE_PASSES or add a pass ending with '.live'." + echo "W: Multi-layered mode is enabled, but we didn't find any live pass." \ + "Either set \$LIVE_PASSES or add a pass ending with '.live'." fi echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot From ce458ba31e42f20b434f358c4ba3485008d88859 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:18:37 +1200 Subject: [PATCH 02/10] run binary hooks on each layer in lb_binary_layered --- live-build/lb_binary_layered | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index 2cf56dbb..f379ddc5 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -66,7 +66,7 @@ build_layered_squashfs () { return fi - rm -f .build/binary_chroot + rm -f .build/binary_chroot .build/binary_hooks mkdir -p "$overlay_dir/" lowerdirs=$(get_lowerdirs_for_pass $pass) From 3a3558203b330ac50a2f7259cb6322258a194a9c Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:20:12 +1200 Subject: [PATCH 03/10] run chroot_hacks on each layer in lb_chroot_layered the comment says this was not done to avoid overlay size inflation but aiui reduce_pass_size deals with that now --- live-build/lb_chroot_layered | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 261d447b..1ade8e1b 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -154,11 +154,11 @@ create_chroot_pass () { # Only get some function executed on root passes # Copy bootstrap on root layers if $(is_root_layer $pass); then - rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks + rm -f .build/chroot_linux-image .build/chroot_preseed cp -a chroot.bootstrap/ "$overlay_dir/" fi # Others have to be executed on every pass - rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive + rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive .build/chroot_hacks mkdir -p "$overlay_dir/" lowerdirs=$(get_lowerdirs_for_pass $pass) From 30cc2f1982cd8156fc17201bd3083bf1dd6d8c58 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:22:50 +1200 Subject: [PATCH 04/10] lb_chroot_layered: install archives on each layer also only call configure_universe after packages are installed. --- live-build/lb_chroot_layered | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 1ade8e1b..27c83a16 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -191,13 +191,7 @@ create_chroot_pass () { lb chroot_resolv install ${*} lb chroot_hostname install ${*} lb chroot_apt install ${*} - # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more - # diff content than desired. So only running this on root pass. - # Only configure universe on root passes - if $(is_root_layer $pass); then - lb chroot_archives chroot install ${*} - configure_universe - fi + lb chroot_archives chroot install ${*} # Customizing chroot lb chroot_linux-image ${*} @@ -247,14 +241,7 @@ create_chroot_pass () { lb chroot_hooks ${*} - # Run chroot_hacks only on root layers. - # chroot_hacks changes the mode of boot/initrd*. The side effect in - # layered mode is to create an initrd on each layer with a significant - # impact on image size (+30MB per layer). This is an issue even with - # update-initramfs disabled. - if $(is_live_layer $pass); then - lb chroot_hacks ${*} - fi + lb chroot_hacks ${*} lb chroot_interactive ${*} @@ -264,10 +251,8 @@ create_chroot_pass () { Chroot chroot "dpkg-query -W" > chroot.packages.${pass} - # Deconfiguring chroot - if $(is_root_layer $pass); then - lb chroot_archives chroot remove ${*} - fi + lb chroot_archives chroot remove ${*} + configure_universe lb chroot_apt remove ${*} lb chroot_hostname remove ${*} lb chroot_resolv remove ${*} From e4ca30e351c8f748d6c31400127146411a58aca7 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:24:30 +1200 Subject: [PATCH 05/10] lb_binary_layered: use create_squashfs function to pick up some default excludes --- live-build/lb_binary_layered | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index f379ddc5..bea1a0fc 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -121,9 +121,7 @@ build_layered_squashfs () { cp "${squashfs_f_manifest}.full" "${prefix}.manifest" fi - (cd "overlay.${pass}/" && - mksquashfs . ${squashfs_f} \ - -no-progress -xattrs -comp xz ) + create_squashfs "overlay.${pass}" ${squashfs_f} if [ -n "$lowerdirs" ]; then umount chroot From fb8af1fd8033f2ca4d7c5c4df8953a8d6e2cccac Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:25:40 +1200 Subject: [PATCH 06/10] add a way to prevent a layer (and descendants) from being made into a sqashfs for the live server build, i want to make a layer to install the kernel into but do not want the layer itself to be published. the implementation is a bit clunky but it works. --- live-build/auto/config | 3 ++ live-build/lb_binary_layered | 75 ++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 29 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 6ddde4ea..df5bc887 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -1023,6 +1023,9 @@ echo "IMAGEFORMAT=\"$IMAGEFORMAT\"" >> config/chroot if [ -n "$PASSES" ]; then echo "PASSES=\"$PASSES\"" >> config/common fi +if [ -n "$NO_SQUASHFS_PASSES" ]; then + echo "NO_SQUASHFS_PASSES=\"$NO_SQUASHFS_PASSES\"" >> config/common +fi if [ -n "$LIVE_PASSES" ]; then echo "LIVE_PASSES=\"$LIVE_PASSES\"" >> config/common fi diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index bea1a0fc..acd6f591 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -92,37 +92,54 @@ build_layered_squashfs () { rm -f chroot/boot/initrd.img-* chroot/boot/vmlinu{x,z}-* fi - # Full manifest until that PASS - squashfs_f_manifest="${base}.manifest" - create_manifest "chroot" "${squashfs_f_manifest}.full" - - # Delta manifest - diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest - - squashfs_f_size="${base}.size" - du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" - - # We take first live pass for "global" ISO properties (used by installers and checkers): - # Prepare initrd + kernel - # Main manifest and size files - prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" - if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then - totalsize=$(cat ${squashfs_f_size}) - curpass="$pass" - while :; do - curpass=$(get_parent_pass $curpass) - # We climbed up the tree to the root layer, we are done - [ -z "$curpass" ] && break - - totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) - done - echo ${totalsize} > "${prefix}.size" - - cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + make_squashfs=yes + ppass="${pass}" + while :; do + for nsp in ${NO_SQUASHFS_PASSES}; do + if [ "${ppass}" = "${nsp}" ]; then + make_squashfs=no + break 2 + fi + done + ppass="$(get_parent_pass "${ppass}")" + if [ -z "${ppass}" ]; then + break + fi + done + + if [ $make_squashfs = yes ]; then + # Full manifest until that PASS + squashfs_f_manifest="${base}.manifest" + create_manifest "chroot" "${squashfs_f_manifest}.full" + + # Delta manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + + squashfs_f_size="${base}.size" + du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" + + # We take first live pass for "global" ISO properties (used by installers and checkers): + # Prepare initrd + kernel + # Main manifest and size files + prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" + if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then + totalsize=$(cat ${squashfs_f_size}) + curpass="$pass" + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) + done + echo ${totalsize} > "${prefix}.size" + + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + fi + + create_squashfs "overlay.${pass}/" ${squashfs_f} fi - create_squashfs "overlay.${pass}" ${squashfs_f} - if [ -n "$lowerdirs" ]; then umount chroot rmdir chroot From 0501f73231d2d2bda5a47d98b43542a2a71b958c Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 25 Jun 2021 10:36:34 +1200 Subject: [PATCH 07/10] update changelog --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index e39049f3..731c9ae9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +livecd-rootfs (2.729) UNRELEASED; urgency=medium + + * Fix a few things found while porting the liver server ISO builds to + layers. + + -- Michael Hudson-Doyle Fri, 25 Jun 2021 10:35:47 +1200 + livecd-rootfs (2.728) impish; urgency=medium [ Dan Bungert ] From 653b9256465c04621edaf1fa312616c54da868fe Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Tue, 29 Jun 2021 22:30:11 +1200 Subject: [PATCH 08/10] attempt to fix obscure failure --- live-build/lb_chroot_layered | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 27c83a16..5aa768ac 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -131,7 +131,7 @@ reduce_pass_size () { continue fi # Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta. - Echo_message "reduce_pass_size: '$f' is strictly identical between $parent and $pass. Removing." + Echo_message "reduce_pass_size: '%s' is strictly identical between %s and %s. Removing." "$f" "$pass" "$parent" rm "$pass_dir/$f" done @@ -143,7 +143,7 @@ create_chroot_pass () { local pass=$1 shift 1 # restore ${*} - Echo_message "lb_chroot_layered: treating pass $pass" + Echo_message "lb_chroot_layered: treating pass %s" "$pass" # We have already treated that pass just return. local overlay_dir="overlay.${pass}" From c4402139a2b6793f98b704a16ceb0ce5db123119 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Wed, 30 Jun 2021 14:37:18 +1200 Subject: [PATCH 09/10] go back to only running lb_chroot_hacks on live layers otherwise each and every layer above a layer with a kernel gets its own initramfs, which is silly. Copy/paste the cruft cleaning bit of lb_chroot_hacks to be run on non-live layers. --- live-build/lb_chroot_layered | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 5aa768ac..3fe761cf 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -158,7 +158,7 @@ create_chroot_pass () { cp -a chroot.bootstrap/ "$overlay_dir/" fi # Others have to be executed on every pass - rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive .build/chroot_hacks + rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive mkdir -p "$overlay_dir/" lowerdirs=$(get_lowerdirs_for_pass $pass) @@ -241,7 +241,25 @@ create_chroot_pass () { lb chroot_hooks ${*} - lb chroot_hacks ${*} + # Run chroot_hacks only on live layers. chroot_hacks is what repacks + # the initrd in a full casperized mode. We don't want to do that for + # each and every layer, just for the live layers (which + # lb_binary_layered will call lb binary_linux-image on). + if $(is_live_layer $pass); then + rm -f .build/chroot_hacks + lb chroot_hacks ${*} + else + # chroot_hacks also removes some cruft, which is appropriate for + # any layer so we copy and paste that bit here. + rm -f chroot/boot/initrd*bak* + rm -f chroot/etc/apt/trusted.gpg~ + rm -f chroot/etc/group- chroot/etc/passwd- + rm -f chroot/etc/gshadow- chroot/etc/shadow- + rm -f chroot/var/cache/debconf/*-old + rm -f chroot/var/lib/dpkg/*-old + rm -f chroot/var/log/apt/term.log + rm -f chroot/etc/adjtime + fi lb chroot_interactive ${*} From 18bf308be07c6a3be3f32bc704d23396f89f683b Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Wed, 30 Jun 2021 17:10:51 +1200 Subject: [PATCH 10/10] go back to only running 'lb chroot_archives' on root layers but still run configure_universe after archive removal, not before --- live-build/lb_chroot_layered | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 3fe761cf..c08a43ef 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -191,7 +191,12 @@ create_chroot_pass () { lb chroot_resolv install ${*} lb chroot_hostname install ${*} lb chroot_apt install ${*} - lb chroot_archives chroot install ${*} + # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more + # diff content than desired. So only running this on root pass. + # Only configure universe on root passes + if $(is_root_layer $pass); then + lb chroot_archives chroot install ${*} + fi # Customizing chroot lb chroot_linux-image ${*} @@ -269,8 +274,11 @@ create_chroot_pass () { Chroot chroot "dpkg-query -W" > chroot.packages.${pass} - lb chroot_archives chroot remove ${*} - configure_universe + # Deconfiguring chroot + if $(is_root_layer $pass); then + lb chroot_archives chroot remove ${*} + configure_universe + fi lb chroot_apt remove ${*} lb chroot_hostname remove ${*} lb chroot_resolv remove ${*}