From 3e34faaa7046ff86cedc3ffd52ebc130c938adde Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 31 Oct 2018 16:37:32 +0100 Subject: [PATCH 01/44] Moved add_task and add_package to functions in order to be available in hooks --- live-build/auto/config | 40 +--------------------------------------- live-build/functions | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 65f5df56..2ec19bfe 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -37,45 +37,7 @@ cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-p mkdir -p config/package-lists -add_task () -{ - local pass="$1" - shift - local task - - # The removal of direct task installation support from live-build - # poses some problems. If the chroot has multiarch configured - for - # example, if we're building for amd64 - then dumpavail will show - # foreign-architecture packages which will have their own Task - # lines, but which we don't want to install. (Compare - # PackageContainerInterface::FromTask in apt, which restricts task - # expansion to the native architecture.) We therefore restrict our - # search to stanzas with Architecture: $ARCH or all. - # - # However, even this may not be accurate enough. At the moment I - # have no idea what happens if an Architecture: all package has - # different Task fields on different architectures. This is - # probably a lurking timebomb that we need to fix. In the meantime, - # the Architecture restriction at least saves us from abject - # failure. - - for task; do - # We need a ridiculous number of backslashes to protect - # parentheses from eval. - echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" - done -} - -add_package () -{ - local pass="$1" - shift - local pkg - - for pkg; do - echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" - done -} +. config/functions OPTS= COMPONENTS= diff --git a/live-build/functions b/live-build/functions index 2cbe3e02..fe6da6d9 100644 --- a/live-build/functions +++ b/live-build/functions @@ -501,3 +501,44 @@ snap_preseed() { fi _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL } + + +add_task () +{ + local pass="$1" + shift + local task + + # The removal of direct task installation support from live-build + # poses some problems. If the chroot has multiarch configured - for + # example, if we're building for amd64 - then dumpavail will show + # foreign-architecture packages which will have their own Task + # lines, but which we don't want to install. (Compare + # PackageContainerInterface::FromTask in apt, which restricts task + # expansion to the native architecture.) We therefore restrict our + # search to stanzas with Architecture: $ARCH or all. + # + # However, even this may not be accurate enough. At the moment I + # have no idea what happens if an Architecture: all package has + # different Task fields on different architectures. This is + # probably a lurking timebomb that we need to fix. In the meantime, + # the Architecture restriction at least saves us from abject + # failure. + + for task; do + # We need a ridiculous number of backslashes to protect + # parentheses from eval. + echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" + done +} + +add_package () +{ + local pass="$1" + shift + local pkg + + for pkg; do + echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" + done +} From 7b4acba31d2f51990505d1f8e90a9f63bd010d2c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 8 Nov 2018 10:00:54 +0100 Subject: [PATCH 02/44] * Added lb_chroot_layered to add support for multi layered squashfs images * auto/*: Added live-layered image format --- live-build/auto/build | 28 ++++++- live-build/auto/clean | 1 + live-build/auto/config | 119 +++++++++++++++++------------ live-build/lb_chroot_layered | 141 +++++++++++++++++++++++++++++++++++ 4 files changed, 242 insertions(+), 47 deletions(-) create mode 100755 live-build/lb_chroot_layered diff --git a/live-build/auto/build b/live-build/auto/build index 70287baa..08d85c1f 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -254,7 +254,11 @@ rm -f /usr/sbin/update-initramfs dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs EOF - lb chroot "$@" + if [ "${IMAGEFORMAT:-}" = live-layered ]; then + PATH="config/:$PATH" lb chroot_layered "$@" + else + lb chroot "$@" + fi if [ "${SUBPROJECT:-}" = minimized ]; then # force removal of initramfs-tools, which we assert is not @@ -474,6 +478,28 @@ EOF /usr/share/livecd-rootfs/minimize-manual chroot lb binary "$@" + + # Create layered squashfs system + if [ "${IMAGEFORMAT:-}" = live-layered ]; then + for _PASS in $PASSES + do + squashfs_f="${PWD}/livecd.${PROJECT}.${_PASS}.squashfs" + + (cd "chroot.${_PASS}/" && + mksquashfs . ${squashfs_f} \ + -no-progress -xattrs -comp xz ) + + squashfs_f_manifest="${squashfs_f}.manifest" + create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}" + done + # We are skipping lb_binary_rootfs + fi + + # Teardown mount points in layered mode + if [ -x config/teardown_chroot_layered ]; then + config/teardown_chroot_layered + fi + touch binary.success ) 2>&1 | tee binary.log diff --git a/live-build/auto/clean b/live-build/auto/clean index 9ddf752e..4794c311 100755 --- a/live-build/auto/clean +++ b/live-build/auto/clean @@ -8,3 +8,4 @@ rm -rf config rm -f binary.manifest binary.manifest-desktop binary.log rm -f livecd.* rm -rf userdata +rm -rf chroot.* diff --git a/live-build/auto/config b/live-build/auto/config index 2ec19bfe..27a319e1 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -33,6 +33,7 @@ fi mkdir -p config cp -af /usr/share/livecd-rootfs/live-build/functions config/functions +cp -af /usr/share/livecd-rootfs/live-build/lb_chroot_layered config/lb_chroot_layered cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse mkdir -p config/package-lists @@ -69,6 +70,9 @@ add_binary_hook () } if [ -z "${IMAGEFORMAT:-}" ]; then + if [ ${SUBPROJECT:-} = layered ]; then + IMAGEFORMAT=live-layered + fi case $PROJECT:${SUBPROJECT:-} in ubuntu-cpc:*) if [ "$SUBARCH" = "raspi3" ]; then @@ -189,6 +193,12 @@ case $IMAGEFORMAT in exit 0 ;; + live-layered) + # Stop lb creating filesystem.squashfs by skipping lb_binary_rootfs and lb_binary_manifest + skip_lb_stage binary_rootfs + skip_lb_stage binary_manifest + ;; + none) OPTS="${OPTS:+$OPTS }--chroot-filesystem $IMAGEFORMAT" ;; @@ -321,8 +331,19 @@ esac case $PROJECT in ubuntu|ubuntu-dvd) - add_task install minimal standard ubuntu-desktop - LIVE_TASK='ubuntu-live' + case ${SUBPROJECT:-} in + layered) + PASSES="install-minimal install live" + add_task install-minimal minimal standard + add_task install ubuntu-desktop + LIVE_TASK='ubuntu-live' + ;; + *) + add_task install minimal standard ubuntu-desktop + LIVE_TASK='ubuntu-live' + ;; + esac + case $ARCH in amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; esac @@ -686,54 +707,56 @@ case $SUBPROJECT in ;; esac -# we'll expand the base seed given here according to the STRUCTURE file, and -# then look in all of the seeds found to see which snaps are seeded -case $PROJECT:${SUBPROJECT:-} in - ubuntu:*|kubuntu*:*|lubuntu*:*|xubuntu*:*|ubuntu-mate*:*|ubuntustudio*:*|ubuntukylin*:*|ubuntu-budgie*:*) - BASE_SEED='desktop' - ;; - ubuntu-cpc:*) - BASE_SEED='server' - ;; - ubuntu-server:live) - BASE_SEED='server' - # subiquity is seeded but in a separate squashfs via hooks; set HOOK_SNAPS and ALL_SNAPS. - HOOK_SNAPS='subiquity' - ALL_SNAPS='' - ;; -esac +if [ "${IMAGEFORMAT:-}" != live-layered ]; then + # we'll expand the base seed given here according to the STRUCTURE file, and + # then look in all of the seeds found to see which snaps are seeded + case $PROJECT:${SUBPROJECT:-} in + ubuntu:*|kubuntu*:*|lubuntu*:*|xubuntu*:*|ubuntu-mate*:*|ubuntustudio*:*|ubuntukylin*:*|ubuntu-budgie*:*) + BASE_SEED='desktop' + ;; + ubuntu-cpc:*) + BASE_SEED='server' + ;; + ubuntu-server:live) + BASE_SEED='server' + # subiquity is seeded but in a separate squashfs via hooks; set HOOK_SNAPS and ALL_SNAPS. + HOOK_SNAPS='subiquity' + ALL_SNAPS='' + ;; + esac -if [ -n "${BASE_SEED}" ]; then - SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) - for seed in ${SEEDS_EXPANDED}; do - echo "snap: considering ${seed}" - file=config/germinate-output/${seed}.snaps - [ -e "${file}" ] || continue - # extract the first column (snap package name) from germinate's output - # translate the human-readable "foo (classic)" into a - # more machine readable "foo/classic" - seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") - for snap in ${seed_snaps}; do - echo "snap: found ${snap}" - ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" + if [ -n "${BASE_SEED}" ]; then + SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) + for seed in ${SEEDS_EXPANDED}; do + echo "snap: considering ${seed}" + file=config/germinate-output/${seed}.snaps + [ -e "${file}" ] || continue + # extract the first column (snap package name) from germinate's output + # translate the human-readable "foo (classic)" into a + # more machine readable "foo/classic" + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") + for snap in ${seed_snaps}; do + echo "snap: found ${snap}" + ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" + done done - done - if [ -n "${ALL_SNAPS}" ] || [ -n "${HOOK_SNAPS}" ]; then - echo "${ALL_SNAPS}" > config/seeded-snaps + if [ -n "${ALL_SNAPS}" ] || [ -n "${HOOK_SNAPS}" ]; then + echo "${ALL_SNAPS}" > config/seeded-snaps + fi fi -fi -# grab a list of packags to remove for a "minimal" installation from the seed -# mirror for this project -if [ -n "${BASE_SEED}" ] && [ -n "${BASE_SEED}" ]; then - minimal_packages_url=${SEEDMIRROR}/${SEED}/${BASE_SEED}.minimal-remove - echo -n "Checking ${minimal_packages_url} for a minimal installation list... " - minimal_packages_remove=$(wget -q -O- ${minimal_packages_url} | sed -e '/\s*#.*$/d' -e '/^\s*$/d') - if [ -n "${minimal_packages_remove}" ]; then - echo "${minimal_packages_remove}" > config/manifest-minimal-remove - echo "$(echo ${minimal_packages_remove} | tr '\n' ' ')" - else - echo "failed to retrieve, not including." + # grab a list of packags to remove for a "minimal" installation from the seed + # mirror for this project + if [ -n "${BASE_SEED}" ] && [ -n "${BASE_SEED}" ]; then + minimal_packages_url=${SEEDMIRROR}/${SEED}/${BASE_SEED}.minimal-remove + echo -n "Checking ${minimal_packages_url} for a minimal installation list... " + minimal_packages_remove=$(wget -q -O- ${minimal_packages_url} | sed -e '/\s*#.*$/d' -e '/^\s*$/d') + if [ -n "${minimal_packages_remove}" ]; then + echo "${minimal_packages_remove}" > config/manifest-minimal-remove + echo "$(echo ${minimal_packages_remove} | tr '\n' ' ')" + else + echo "failed to retrieve, not including." + fi fi fi @@ -857,6 +880,10 @@ lb config noauto \ echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/chroot +echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common +if [ -n "$PASSES" ]; then + echo "PASSES=\"$PASSES\"" >> config/common +fi echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary echo "BUILDSTAMP=\"$NOW\"" >> config/binary echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered new file mode 100755 index 00000000..2b7f13e3 --- /dev/null +++ b/live-build/lb_chroot_layered @@ -0,0 +1,141 @@ +#!/bin/sh + +## live-build(7) - System Build Scripts +## Copyright (C) 2006-2012 Daniel Baumann +## +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING. +## This is free software, and you are welcome to redistribute it +## under certain conditions; see COPYING for details. + +## This is a fork of lb_chroot for layered live system. +## We don't want leaking host configuratino in each layer, and so, +## we clean and setup the chroot each time. +## In addition, we create the squashfs for each layer, but top one (live) +## which still can be configured after lb chroot call. + +set -e + +# Including common functions +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh + +# Automatically populating config tree +if [ -x auto/config ] && [ ! -e .build/config ] +then + Echo_message "Automatically populating config tree." + lb config +fi + +# Setting static variables +DESCRIPTION="$(Echo 'customize the Debian system')" +HELP="" +USAGE="${PROGRAM} [--force]" + +Arguments "${@}" + +# Reading configuration files +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source +Set_defaults + +# Setup cleanup function +Setup_cleanup + +. config/functions + +teardownPath="config/teardown_chroot_layered" +cat >> $teardownPath << EOF +#!/bin/sh +set -e +EOF +chmod +x $teardownPath + +PASSES="${PASSES:-install live}" +CURPASS=1 +LASTPASS=$(echo $PASSES|wc -w) +for _PASS in $PASSES +do + + if [ $CURPASS -gt 1 ]; then + mkdir chroot.${_PASS} + mount_overlay chroot/ "chroot.${_PASS}/" chroot/ + cat >> $teardownPath << EOF +umount chroot/ +EOF + fi + + # Configuring chroot + lb chroot_cache restore ${*} + lb chroot_devpts install ${*} + lb chroot_proc install ${*} + lb chroot_selinuxfs install ${*} + lb chroot_sysfs install ${*} + lb chroot_debianchroot install ${*} + lb chroot_dpkg install ${*} + lb chroot_tmpfs install ${*} + lb chroot_sysv-rc install ${*} + lb chroot_upstart install ${*} + lb chroot_hosts install ${*} + lb chroot_resolv install ${*} + lb chroot_hostname install ${*} + lb chroot_apt install ${*} + lb chroot_archives chroot install ${*} + + # Customizing chroot + lb chroot_linux-image ${*} + lb chroot_preseed ${*} + lb chroot_early_hooks ${*} + + lb chroot_package-lists ${_PASS} ${*} + lb chroot_install-packages ${_PASS} ${*} + + # Kernel should be in first layer + if [ $CURPASS -eq 1 ]; then + Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ + | awk '/^i/ {print $2}' > chroot.headers + for i in $(cat chroot.headers); do + Chroot chroot "apt-mark auto $i" + done + fi + + Chroot chroot "apt-get --purge -y autoremove" + + # Add live packages to top layer + if [ $CURPASS -eq $LASTPASS ]; then + lb chroot_live-packages ${*} + fi + + lb chroot_includes ${*} + # TODO: look if this is what copies distro hooks + lb chroot_hooks ${*} + lb chroot_hacks ${*} + lb chroot_interactive ${*} + + Chroot chroot "dpkg-query -W" > chroot.packages.${_PASS} + + # Deconfiguring chroot + lb chroot_archives chroot remove ${*} + lb chroot_apt remove ${*} + lb chroot_hostname remove ${*} + lb chroot_resolv remove ${*} + lb chroot_hosts remove ${*} + lb chroot_sysv-rc remove ${*} + lb chroot_upstart remove ${*} + lb chroot_tmpfs remove ${*} + lb chroot_dpkg remove ${*} + lb chroot_debianchroot remove ${*} + lb chroot_sysfs remove ${*} + lb chroot_selinuxfs remove ${*} + lb chroot_proc remove ${*} + lb chroot_devpts remove ${*} + lb chroot_cache save ${*} + + if [ $CURPASS -eq 1 ]; then + cp -a chroot chroot.${_PASS} + fi + + CURPASS=$(( CURPASS + 1 )) +done + +# Remove unused chroot binary corresponding to bottom layer (once everything is unmount) +cat >> $teardownPath << EOF +rm -rf chroot/ +EOF From ce973ba07b71dc239ffdd5c8726c807d0361404a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 8 Nov 2018 11:05:40 +0100 Subject: [PATCH 03/44] auto/build: Removed useless comment --- live-build/auto/build | 1 - 1 file changed, 1 deletion(-) diff --git a/live-build/auto/build b/live-build/auto/build index 08d85c1f..00bb2602 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -492,7 +492,6 @@ EOF squashfs_f_manifest="${squashfs_f}.manifest" create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}" done - # We are skipping lb_binary_rootfs fi # Teardown mount points in layered mode From b6287916c2b96d8d45ddc86a961a47d0e46740eb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 8 Nov 2018 11:06:41 +0100 Subject: [PATCH 04/44] auto/build: Order generated squashfs filesystems --- live-build/auto/build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/live-build/auto/build b/live-build/auto/build index 00bb2602..7e939c2c 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -480,10 +480,11 @@ EOF lb binary "$@" # Create layered squashfs system + CURPASS=1 if [ "${IMAGEFORMAT:-}" = live-layered ]; then for _PASS in $PASSES do - squashfs_f="${PWD}/livecd.${PROJECT}.${_PASS}.squashfs" + squashfs_f="${PWD}/livecd.${PROJECT}.$(printf "%02g" $CURPASS)-${_PASS}.squashfs" (cd "chroot.${_PASS}/" && mksquashfs . ${squashfs_f} \ @@ -491,6 +492,7 @@ EOF squashfs_f_manifest="${squashfs_f}.manifest" create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}" + CURPASS=$(( CURPASS + 1 )) done fi From 940873e6cb8ca716b6619ad66a82f8a74c26e22c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 8 Nov 2018 18:33:41 +0100 Subject: [PATCH 05/44] live-build/lb_chroot_layered: overlayfs supports only 2 stacked layers. Changed the code to use several lower dirs instead like casper does --- live-build/lb_chroot_layered | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 2b7f13e3..089b3ceb 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -41,25 +41,16 @@ Setup_cleanup . config/functions -teardownPath="config/teardown_chroot_layered" -cat >> $teardownPath << EOF -#!/bin/sh -set -e -EOF -chmod +x $teardownPath - PASSES="${PASSES:-install live}" CURPASS=1 LASTPASS=$(echo $PASSES|wc -w) +LOWER_LAYERS="" for _PASS in $PASSES do if [ $CURPASS -gt 1 ]; then mkdir chroot.${_PASS} - mount_overlay chroot/ "chroot.${_PASS}/" chroot/ - cat >> $teardownPath << EOF -umount chroot/ -EOF + mount_overlay $LOWER_LAYERS "chroot.${_PASS}/" chroot/ fi # Configuring chroot @@ -129,13 +120,25 @@ EOF lb chroot_cache save ${*} if [ $CURPASS -eq 1 ]; then - cp -a chroot chroot.${_PASS} + mv chroot chroot.${_PASS} + mkdir chroot + elif [ $CURPASS -ne $LASTPASS ]; then + # we will unmount chroot/ at teardown for upper layer + umount chroot fi + LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" + LOWER_LAYERS="${LOWER_LAYERS%:}" CURPASS=$(( CURPASS + 1 )) done # Remove unused chroot binary corresponding to bottom layer (once everything is unmount) -cat >> $teardownPath << EOF + +TEARDOWNPATH="config/teardown_chroot_layered" +cat > $TEARDOWNPATH << EOF +#!/bin/sh +set -e +umount chroot rm -rf chroot/ EOF +chmod +x $TEARDOWNPATH From 7e2f4d4304eca5baf2c5311d8281de60fc6be381 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Fri, 9 Nov 2018 08:22:44 +0100 Subject: [PATCH 06/44] live-build/lb_chroot_layered: removed useless comment --- live-build/lb_chroot_layered | 1 - 1 file changed, 1 deletion(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 089b3ceb..5457e539 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -95,7 +95,6 @@ do fi lb chroot_includes ${*} - # TODO: look if this is what copies distro hooks lb chroot_hooks ${*} lb chroot_hacks ${*} lb chroot_interactive ${*} From ca63d43696c133215ded2b534863bb51ab718690 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Fri, 9 Nov 2018 16:39:14 +0100 Subject: [PATCH 07/44] live-build/auto/build: Add size of the filesystems. This is used by the installer to calculate the minimum partition size to reserve for installation. The installer will have to sum all the sizes to compute the final size. Thank didrocks --- live-build/auto/build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/live-build/auto/build b/live-build/auto/build index 7e939c2c..0b0a46a0 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -492,6 +492,9 @@ EOF squashfs_f_manifest="${squashfs_f}.manifest" create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}" + squashfs_f_size="${squashfs_f}.size" + du -B 1 -s "chroot.${_PASS}/" | cut -f1 > "${squashfs_f_size}" + CURPASS=$(( CURPASS + 1 )) done fi From d5a16296f49dc82f46ec29252aa117151aa81b13 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Mon, 12 Nov 2018 18:04:30 +0100 Subject: [PATCH 08/44] live-build/auto/build: Create one manifest by layer in diff format and one full manifest with all the layers in standard format --- live-build/auto/build | 18 ++++++++++++++---- live-build/auto/clean | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index 0b0a46a0..3332708a 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -481,18 +481,28 @@ EOF # Create layered squashfs system CURPASS=1 + LASTPASS=$(echo $PASSES|wc -w) + prev_manifest="" if [ "${IMAGEFORMAT:-}" = live-layered ]; then for _PASS in $PASSES do - squashfs_f="${PWD}/livecd.${PROJECT}.$(printf "%02g" $CURPASS)-${_PASS}.squashfs" + base="${PWD}/livecd.${PROJECT}.$(printf "%02g" $CURPASS)-${_PASS}" + squashfs_f="${base}.squashfs" (cd "chroot.${_PASS}/" && mksquashfs . ${squashfs_f} \ -no-progress -xattrs -comp xz ) - squashfs_f_manifest="${squashfs_f}.manifest" - create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}" - squashfs_f_size="${squashfs_f}.size" + squashfs_f_manifest="${base}.manifest" + create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}.full" + # We want manifest to only contain the diff of current pass + diff -NU0 ${prev_manifest}.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + prev_manifest=$squashfs_f_manifest + if [ $CURPASS -eq $LASTPASS ]; then + cp "${squashfs_f_manifest}.full" livecd.${PROJECT}.manifest + fi + + squashfs_f_size="${base}.size" du -B 1 -s "chroot.${_PASS}/" | cut -f1 > "${squashfs_f_size}" CURPASS=$(( CURPASS + 1 )) diff --git a/live-build/auto/clean b/live-build/auto/clean index 4794c311..82a7cce4 100755 --- a/live-build/auto/clean +++ b/live-build/auto/clean @@ -9,3 +9,4 @@ rm -f binary.manifest binary.manifest-desktop binary.log rm -f livecd.* rm -rf userdata rm -rf chroot.* +rm -rf *.manifest.full From 0f4fcbc7d31f0b3b67c63cfb093aa6ac2e37d9b3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 14 Nov 2018 09:57:45 +0100 Subject: [PATCH 09/44] lb_chroot_layered: Export _PASS as PASS to be available in hooks --- live-build/lb_chroot_layered | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 5457e539..9a3b3905 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -47,6 +47,7 @@ LASTPASS=$(echo $PASSES|wc -w) LOWER_LAYERS="" for _PASS in $PASSES do + export PASS=${_PASS} if [ $CURPASS -gt 1 ]; then mkdir chroot.${_PASS} From d4f7a0b23dadb35158245d625618d289d427f0ec Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 11:08:24 +0100 Subject: [PATCH 10/44] live-build/auto/config: Added ubuntu-desktop-minimal to install-minimal task --- live-build/auto/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/auto/config b/live-build/auto/config index 27a319e1..7f2cf0f1 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -334,7 +334,7 @@ case $PROJECT in case ${SUBPROJECT:-} in layered) PASSES="install-minimal install live" - add_task install-minimal minimal standard + add_task install-minimal minimal standard ubuntu-desktop-minimal add_task install ubuntu-desktop LIVE_TASK='ubuntu-live' ;; From 6a7368bf9557f1cac59549d2c0adb9c194bc7a92 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 11:32:40 +0100 Subject: [PATCH 11/44] Clean-up dead code for server live builds. --- live-build/auto/config | 3 --- 1 file changed, 3 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 7f2cf0f1..c6893eec 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -719,9 +719,6 @@ if [ "${IMAGEFORMAT:-}" != live-layered ]; then ;; ubuntu-server:live) BASE_SEED='server' - # subiquity is seeded but in a separate squashfs via hooks; set HOOK_SNAPS and ALL_SNAPS. - HOOK_SNAPS='subiquity' - ALL_SNAPS='' ;; esac From ef72cf5fe845ff4f156d62451474fc82f23aed0a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 11:43:09 +0100 Subject: [PATCH 12/44] Code refactoring: * Moved *inheritance functions from config to functions * Moved code to extract snap from seed to snap_from_seed function --- live-build/auto/config | 42 +----------------------------------- live-build/functions | 48 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index c6893eec..fa21cdce 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -261,30 +261,6 @@ if [ "${SUBPROJECT:-}" = minimized ]; then OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal --linux-packages=linux-image" fi -# cribbed from cdimage, perhaps this should be a small helper script in germinate? -add_inheritance () { - case " $inherit " in - *" $1 "*) - ;; - *) - inherit="${inherit:+$inherit }$1" - ;; - esac -} - -expand_inheritance () { - for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do - expand_inheritance "$seed" - done - add_inheritance "$1" -} - -inheritance () { - inherit= - expand_inheritance "$1" - echo "$inherit" -} - mkdir -p config/germinate-output case $PROJECT in kubuntu-active*) @@ -723,23 +699,7 @@ if [ "${IMAGEFORMAT:-}" != live-layered ]; then esac if [ -n "${BASE_SEED}" ]; then - SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) - for seed in ${SEEDS_EXPANDED}; do - echo "snap: considering ${seed}" - file=config/germinate-output/${seed}.snaps - [ -e "${file}" ] || continue - # extract the first column (snap package name) from germinate's output - # translate the human-readable "foo (classic)" into a - # more machine readable "foo/classic" - seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") - for snap in ${seed_snaps}; do - echo "snap: found ${snap}" - ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" - done - done - if [ -n "${ALL_SNAPS}" ] || [ -n "${HOOK_SNAPS}" ]; then - echo "${ALL_SNAPS}" > config/seeded-snaps - fi + snap_from_seed "${BASE_SEED}" config/seeded-snaps fi # grab a list of packags to remove for a "minimal" installation from the seed diff --git a/live-build/functions b/live-build/functions index fe6da6d9..c3fe2a35 100644 --- a/live-build/functions +++ b/live-build/functions @@ -373,6 +373,30 @@ release_ver() { distro-info --series="$LB_DISTRIBUTION" -r | awk '{ print $1 }' } +# cribbed from cdimage, perhaps this should be a small helper script in germinate? +add_inheritance () { + case " $inherit " in + *" $1 "*) + ;; + *) + inherit="${inherit:+$inherit }$1" + ;; + esac +} + +expand_inheritance () { + for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do + expand_inheritance "$seed" + done + add_inheritance "$1" +} + +inheritance () { + inherit= + expand_inheritance "$1" + echo "$inherit" +} + _snap_preseed() { # Download the snap/assertion and add to the preseed local CHROOT_ROOT=$1 @@ -502,6 +526,30 @@ snap_preseed() { _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL } +snap_from_seed() { + local base_seed=$1 + local out=$2 + local all_snaps + local seeds_expanded + + seeds_expanded=$(inheritance ${base_seed}) + for seed in ${seeds_expanded}; do + echo "snap: considering ${seed}" + file=config/germinate-output/${seed}.snaps + [ -e "${file}" ] || continue + # extract the first column (snap package name) from germinate's output + # translate the human-readable "foo (classic)" into a + # more machine readable "foo/classic" + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") + for snap in ${seed_snaps}; do + echo "snap: found ${snap}" + all_snaps="${all_snaps:+${all_snaps} }${snap}" + done + done + if [ -n "${all_snaps}" ]; then + echo "${all_snaps}" > $out + fi +} add_task () { From 00e474e2ff4247f09974b1a2f812a2331958549c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 12:08:07 +0100 Subject: [PATCH 13/44] Make snap functions idempotent _snap_preseed and snap_prepare_assertions can be called several times with the same snap and will only provision it once. --- live-build/functions | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/live-build/functions b/live-build/functions index c3fe2a35..9f910bcb 100644 --- a/live-build/functions +++ b/live-build/functions @@ -411,6 +411,12 @@ _snap_preseed() { # Download the snap & assertion local snap_download_failed=0 + + # Preseed a snap only once + if [ -f ${snaps_dir}/${SNAP_NAME}_[0-9]*.snap ]; then + return + fi + chroot $CHROOT_ROOT sh -c " set -x; cd /var/lib/snapd/seed; @@ -456,6 +462,10 @@ snap_prepare_assertions() { local account_key_assertion="$assertions_dir/account-key" local account_assertion="$assertions_dir/account" + if [ -d "$assertions_dir" ]; then + return + fi + mkdir -p "$assertions_dir" mkdir -p "$snaps_dir" @@ -500,15 +510,10 @@ snap_prepare() { # used for the image's model assertion local CUSTOM_BRAND_MODEL=${2:-generic:generic-classic} - local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" - local snaps_dir="$seed_dir/snaps" - snap_prepare_assertions "$CHROOT_ROOT" "$CUSTOM_BRAND_MODEL" # Download the core snap - if ! [ -f $snaps_dir/core_[0-9]*.snap ] ; then - _snap_preseed $CHROOT_ROOT core stable - fi + _snap_preseed $CHROOT_ROOT core stable } snap_preseed() { From 1ab78e783bb18e6e0734250f642e55a2fdeaf9f5 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 12:11:13 +0100 Subject: [PATCH 14/44] Generate snap list from task --- live-build/functions | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/live-build/functions b/live-build/functions index 9f910bcb..2bfe3f2d 100644 --- a/live-build/functions +++ b/live-build/functions @@ -556,11 +556,39 @@ snap_from_seed() { fi } +seed_from_task () +{ + # Retrieve the name of the seed from a task name + local task=$1 + local seed + local seedfile + local seedfiles + + seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)" + if [ -n "$seedfile" ]; then + basename $seedfile .seedtext + return + fi + + seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)" + if [ -n "$seedfiles" ]; then + for seed in $(echo $seedfiles | xargs basename -s .seedtext); do + if [ ${PROJECT}-${seed} = $task ]; then + echo ${seed} + return + fi + done + fi +} + add_task () { local pass="$1" shift local task + local snap_list_file + local snap_list_files + local curseed # The removal of direct task installation support from live-build # poses some problems. If the chroot has multiarch configured - for @@ -577,12 +605,32 @@ add_task () # probably a lurking timebomb that we need to fix. In the meantime, # the Architecture restriction at least saves us from abject # failure. + # + # We want as well to grab the snap list for each PASS. Resolve for all + # given task, and deduplicate them to generate snaps for the PASS. for task; do # We need a ridiculous number of backslashes to protect # parentheses from eval. echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" + + curseed=$(seed_from_task ${task}) + if [ -z "${curseed}" ]; then + echo "W: No seed matching task ${task}" + continue + fi + snap_list_file="config/package-lists/seed.${curseed}.snaplist.full" + snap_from_seed "${curseed}" $snap_list_file + if [ -e "$snap_list_file" ]; then + snap_list_files="${snap_list_files} $snap_list_file" + fi done + # The snap list is one line, and could be duplicated between seeds via inheritance. + # Uniquely sort them and store them back in one line. + if [ -n "${snap_list_files}" ]; then + cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full" + rm ${snap_list_files} + fi } add_package () From 903ba3b2760ac38e7c47f5bcc6472467fa26ac81 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 12:12:49 +0100 Subject: [PATCH 15/44] Actually install snap for each layer. --- live-build/lb_chroot_layered | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 9a3b3905..a8b152c4 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -41,8 +41,49 @@ Setup_cleanup . config/functions +# Create the snap list specific to this layer +lb_chroot_snap_lists() { + local pass=$1 + local prevpass=$2 + + # This assumes that the prefix is unique for a given project + local snap_for_pass=$(ls config/package-lists/*.snaplist.chroot_${pass}.full 2>/dev/null || true) + local snap_for_prevpass=$(ls config/package-lists/*.snaplist.chroot_${prevpass}.full 2>/dev/null || true) + + if [ -z "${snap_for_pass}" ]; then + return + fi + + if [ -z "${snap_for_prevpass}" ]; then + cp ${snap_for_pass} ${snap_for_pass%.full} + return + fi + + # Generate a list of snaps added to a layer. + diff -NU0 ${snap_for_prevpass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full} +} + +lb_chroot_install_snaps() { + # Prepare the snap environment and install snaps into a chroot + # + # $1: Name of the pass + + local snaplist_file=$(ls config/package-lists/*.snaplist.chroot_${1} 2>/dev/null || true) + + if [ -z "${snaplist_file}" ]; then + return + fi + + snap_prepare chroot + + while read snap; do + snap_preseed chroot "${snap}" + done < $snaplist_file +} + PASSES="${PASSES:-install live}" CURPASS=1 +PREVPASS="" LASTPASS=$(echo $PASSES|wc -w) LOWER_LAYERS="" for _PASS in $PASSES @@ -79,6 +120,10 @@ do lb chroot_package-lists ${_PASS} ${*} lb chroot_install-packages ${_PASS} ${*} + # Snap management + lb_chroot_snap_lists ${_PASS} ${PREVPASS} + lb_chroot_install_snaps ${_PASS} ${*} + # Kernel should be in first layer if [ $CURPASS -eq 1 ]; then Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ @@ -128,6 +173,7 @@ do fi LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" LOWER_LAYERS="${LOWER_LAYERS%:}" + PREVPASS=${_PASS} CURPASS=$(( CURPASS + 1 )) done From 78c196031b801d2babef10fe9d04e26de709dc00 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 15 Nov 2018 12:15:19 +0100 Subject: [PATCH 16/44] Add snaps to manifest Make sure that all the snaps are listed in the full manifest of each layer and as delta in the delta manifest. --- live-build/auto/build | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/live-build/auto/build b/live-build/auto/build index 3332708a..b2bfdcc5 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -481,8 +481,10 @@ EOF # Create layered squashfs system CURPASS=1 + PREVPASS="" LASTPASS=$(echo $PASSES|wc -w) prev_manifest="" + to_remove="" if [ "${IMAGEFORMAT:-}" = live-layered ]; then for _PASS in $PASSES do @@ -493,11 +495,25 @@ EOF mksquashfs . ${squashfs_f} \ -no-progress -xattrs -comp xz ) + # Full manifest until that PASS + # Note: snaps won't be listed in the new layer by snap-seed-parse + # if no newly snap was installed in that layer. Copy and then remove it + # just to create the correct full manifest and avoid a new layer + # chroot mount dance. + SNAPPATH=var/lib/snapd/seed/seed.yaml + if [ ! -f "chroot.${_PASS}/${SNAPPATH}" ] && [ -n "${PREVPASS}" ] && [ -f "chroot.${PREVPASS}/${SNAPPATH}" ]; then + mkdir -p chroot.${_PASS}/$(dirname ${SNAPPATH}) + cp "chroot.${PREVPASS}/${SNAPPATH}" "chroot.${_PASS}/${SNAPPATH}" + to_remove="chroot.${_PASS}/var/lib/snapd ${to_remove}" + fi squashfs_f_manifest="${base}.manifest" create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}.full" - # We want manifest to only contain the diff of current pass + + # Delta manifest diff -NU0 ${prev_manifest}.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest prev_manifest=$squashfs_f_manifest + + # Full ISO manifest if [ $CURPASS -eq $LASTPASS ]; then cp "${squashfs_f_manifest}.full" livecd.${PROJECT}.manifest fi @@ -505,8 +521,12 @@ EOF squashfs_f_size="${base}.size" du -B 1 -s "chroot.${_PASS}/" | cut -f1 > "${squashfs_f_size}" + PREVPASS=${_PASS} CURPASS=$(( CURPASS + 1 )) done + if [ -n "${to_remove}" ]; then + rm -r ${to_remove} + fi fi # Teardown mount points in layered mode From 53b06af93114db4339dd802044aa5fcea73751c4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 11:31:49 +0100 Subject: [PATCH 17/44] Added sublayers support Adds support for sublayers by defining package lists corresponding to each sublayer. The name of the package list defines the name of the layer. --- live-build/auto/build | 111 +++++++++++++++++++++-------------- live-build/lb_chroot_layered | 78 ++++++++++++++++-------- 2 files changed, 122 insertions(+), 67 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index b2bfdcc5..07025195 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -17,6 +17,57 @@ fi . config/functions +build_layered_squashfs() { + local pass=$1 # install|install_subpass|install_subpass_subsubpass|… + local prevpass=$2 # install|install_subpass|… + local prefix=$3 # 01-|02-|… + local lowerlayers=$4 + + base="${PWD}/livecd.${PROJECT}.${prefix}${pass}" + squashfs_f="${base}.squashfs" + + # We have already treated that pass + if [ -f "${squashfs_f}" ]; then + return + fi + + if [ -n "${lowerlayers}" ]; then + mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ + else + # first pass + ln -s "chroot.${pass}/" chroot + fi + + (cd "chroot.${pass}/" && + mksquashfs . ${squashfs_f} \ + -no-progress -xattrs -comp xz ) + + # Full manifest until that PASS + squashfs_f_manifest="${base}.manifest" + create_manifest "chroot" "${squashfs_f_manifest}.full" + + # Delta manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.${prefix}${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + + squashfs_f_size="${base}.size" + du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}" + + if [ -n "${lowerlayers}" ]; then + umount chroot + else + rm chroot + mkdir chroot/ + fi + + # Handle direct sublayer of current one + # Extract the name of the pass corresponding to the sublayer + for subpass in $(ls -d chroot.${pass}_* 2>/dev/null | sed -e "s/chroot\.\(${pass}_[^_]\+\).*/\1/"); do + lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" + lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" + build_layered_squashfs "${subpass}" "${pass}" "${prefix}" "${lowerlayers_for_subpass}" + done +} + # Link output files somewhere launchpad-buildd will be able to find them. PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" @@ -479,59 +530,31 @@ EOF lb binary "$@" + # Teardown mount points in layered mode + if [ -x config/teardown_chroot_layered ]; then + config/teardown_chroot_layered + fi + # Create layered squashfs system - CURPASS=1 - PREVPASS="" - LASTPASS=$(echo $PASSES|wc -w) - prev_manifest="" - to_remove="" if [ "${IMAGEFORMAT:-}" = live-layered ]; then + CURPASS=1 + PREVPASS="" + PREFIX="" + LOWER_LAYERS="" + for _PASS in $PASSES do - base="${PWD}/livecd.${PROJECT}.$(printf "%02g" $CURPASS)-${_PASS}" - squashfs_f="${base}.squashfs" - - (cd "chroot.${_PASS}/" && - mksquashfs . ${squashfs_f} \ - -no-progress -xattrs -comp xz ) - - # Full manifest until that PASS - # Note: snaps won't be listed in the new layer by snap-seed-parse - # if no newly snap was installed in that layer. Copy and then remove it - # just to create the correct full manifest and avoid a new layer - # chroot mount dance. - SNAPPATH=var/lib/snapd/seed/seed.yaml - if [ ! -f "chroot.${_PASS}/${SNAPPATH}" ] && [ -n "${PREVPASS}" ] && [ -f "chroot.${PREVPASS}/${SNAPPATH}" ]; then - mkdir -p chroot.${_PASS}/$(dirname ${SNAPPATH}) - cp "chroot.${PREVPASS}/${SNAPPATH}" "chroot.${_PASS}/${SNAPPATH}" - to_remove="chroot.${_PASS}/var/lib/snapd ${to_remove}" - fi - squashfs_f_manifest="${base}.manifest" - create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}.full" - - # Delta manifest - diff -NU0 ${prev_manifest}.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest - prev_manifest=$squashfs_f_manifest - - # Full ISO manifest - if [ $CURPASS -eq $LASTPASS ]; then - cp "${squashfs_f_manifest}.full" livecd.${PROJECT}.manifest - fi - - squashfs_f_size="${base}.size" - du -B 1 -s "chroot.${_PASS}/" | cut -f1 > "${squashfs_f_size}" + PREFIX="$(printf "%02g" $CURPASS)-" + build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PREFIX" ${LOWER_LAYERS} + LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" + LOWER_LAYERS="${LOWER_LAYERS%:}" PREVPASS=${_PASS} CURPASS=$(( CURPASS + 1 )) done - if [ -n "${to_remove}" ]; then - rm -r ${to_remove} - fi - fi - # Teardown mount points in layered mode - if [ -x config/teardown_chroot_layered ]; then - config/teardown_chroot_layered + # Full ISO manifest + cp "${PWD}/livecd.${PROJECT}.${PREFIX}${_PASS}.manifest.full" livecd.${PROJECT}.manifest fi touch binary.success diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index a8b152c4..4d401705 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -81,18 +81,23 @@ lb_chroot_install_snaps() { done < $snaplist_file } -PASSES="${PASSES:-install live}" -CURPASS=1 -PREVPASS="" -LASTPASS=$(echo $PASSES|wc -w) -LOWER_LAYERS="" -for _PASS in $PASSES -do - export PASS=${_PASS} +create_chroot_pass() { + local pass=$1 + local prevpass=$2 + local lowerlayers=$3 + local passtype=$4 # "first"|"last"|"" empty string + shift 4 # restore ${*} + + # We have already treated that pass + if [ -d "chroot.${pass}/" ]; then + return + fi - if [ $CURPASS -gt 1 ]; then - mkdir chroot.${_PASS} - mount_overlay $LOWER_LAYERS "chroot.${_PASS}/" chroot/ + export PASS=${pass} + + if [ "${passtype}" != "first" ]; then + mkdir chroot.${pass} + mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ fi # Configuring chroot @@ -117,15 +122,15 @@ do lb chroot_preseed ${*} lb chroot_early_hooks ${*} - lb chroot_package-lists ${_PASS} ${*} - lb chroot_install-packages ${_PASS} ${*} + lb chroot_package-lists ${pass} ${*} + lb chroot_install-packages ${pass} ${*} # Snap management - lb_chroot_snap_lists ${_PASS} ${PREVPASS} - lb_chroot_install_snaps ${_PASS} ${*} + lb_chroot_snap_lists ${pass} ${prevpass} + lb_chroot_install_snaps ${pass} ${*} # Kernel should be in first layer - if [ $CURPASS -eq 1 ]; then + if [ "${passtype}" = "first" ]; then Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ | awk '/^i/ {print $2}' > chroot.headers for i in $(cat chroot.headers); do @@ -136,7 +141,7 @@ do Chroot chroot "apt-get --purge -y autoremove" # Add live packages to top layer - if [ $CURPASS -eq $LASTPASS ]; then + if [ "${passtype}" = "last" ]; then lb chroot_live-packages ${*} fi @@ -145,7 +150,7 @@ do lb chroot_hacks ${*} lb chroot_interactive ${*} - Chroot chroot "dpkg-query -W" > chroot.packages.${_PASS} + Chroot chroot "dpkg-query -W" > chroot.packages.${pass} # Deconfiguring chroot lb chroot_archives chroot remove ${*} @@ -164,13 +169,38 @@ do lb chroot_devpts remove ${*} lb chroot_cache save ${*} - if [ $CURPASS -eq 1 ]; then - mv chroot chroot.${_PASS} + if [ "${passtype}" = "first" ]; then + mv chroot chroot.${pass} mkdir chroot - elif [ $CURPASS -ne $LASTPASS ]; then - # we will unmount chroot/ at teardown for upper layer + else umount chroot fi + + # Handle direct sublayer of current one + # Extract the name of the pass corresponding to the sublayer + for subpass in $(ls config/package-lists/*.list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do + lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" + lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" + create_chroot_pass "${subpass}" "${pass}" "${lowerlayers_for_subpass}" "" ${*} + done +} + +PASSES="${PASSES:-install live}" +CURPASS=1 +PREVPASS="" +LASTPASS=$(echo $PASSES|wc -w) +LOWER_LAYERS="" +for _PASS in $PASSES +do + PASSTYPE="" + if [ $CURPASS -eq 1 ]; then + PASSTYPE="first" + elif [ $CURPASS -eq $LASTPASS ]; then + PASSTYPE="last" + fi + + create_chroot_pass "$_PASS" "$PREVPASS" "$LOWER_LAYERS" "$PASSTYPE" ${*} + LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" LOWER_LAYERS="${LOWER_LAYERS%:}" PREVPASS=${_PASS} @@ -178,8 +208,10 @@ do CURPASS=$(( CURPASS + 1 )) done -# Remove unused chroot binary corresponding to bottom layer (once everything is unmount) +# remount last pass on chroot for lb binary +mount_overlay "${LOWER_LAYERS}" "chroot.${_PASS}/" chroot/ +# Remove unused chroot binary corresponding to bottom layer (once everything is unmount) TEARDOWNPATH="config/teardown_chroot_layered" cat > $TEARDOWNPATH << EOF #!/bin/sh From a9908e0c9d5fc54625db2179de4f88c8ae487f60 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 12:53:28 +0100 Subject: [PATCH 18/44] Add support for negative package lists --- live-build/lb_chroot_layered | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 4d401705..41ebef64 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -41,6 +41,18 @@ Setup_cleanup . config/functions +lb_chroot_remove_packages() { + # Remove packages from the chroot specific to this layer + # + # $1: Name of the pass* + local pass=$1 + + Expand_packagelist "$(basename config/package-lists/*.removal-list.chroot_${pass})" "config/package-lists" \ + >> chroot/root/packages.chroot.removal + Chroot chroot "xargs --arg-file=/root/packages.chroot.removal apt-get ${APT_OPTIONS} autoremove --purge" + rm -f chroot/root/packages.chroot.removal +} + # Create the snap list specific to this layer lb_chroot_snap_lists() { local pass=$1 @@ -124,6 +136,7 @@ create_chroot_pass() { lb chroot_package-lists ${pass} ${*} lb chroot_install-packages ${pass} ${*} + lb_chroot_remove_packages ${pass} ${*} # Snap management lb_chroot_snap_lists ${pass} ${prevpass} From f3dadf109196f26ffc340d3f07f6833e999dfaa6 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 13:01:35 +0100 Subject: [PATCH 19/44] Support for positive and negative sublayers Calculate positive and negative package lists for sublayers --- live-build/auto/config | 6 ++++ live-build/functions | 56 ++++++++++++++++++++++++++++++++++++ live-build/lb_chroot_layered | 2 +- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/live-build/auto/config b/live-build/auto/config index fa21cdce..0b413631 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -311,7 +311,13 @@ case $PROJECT in layered) PASSES="install-minimal install live" add_task install-minimal minimal standard ubuntu-desktop-minimal + # LANG PASS for minimal + add_layered_pass_delta install-minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' + add_layered_pass_delta install-minimal desktop-minimal-default-languages 'nolangpacks' # none (if no default langpack is selected) add_task install ubuntu-desktop + # LANG PASS for install + add_layered_pass_delta install desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' + add_layered_pass_delta install desktop-default-languages 'nolangpacks' # none (if no default langpack is selected) LIVE_TASK='ubuntu-live' ;; *) diff --git a/live-build/functions b/live-build/functions index 2bfe3f2d..8635d2c4 100644 --- a/live-build/functions +++ b/live-build/functions @@ -643,3 +643,59 @@ add_package () echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" done } + +list_packages_from_seed () { + # Store all packages for a given seed, including its seed dependency + # $1: Name of the seed to expand to a package list + + local all_seeds="$(inheritance $1)" + + for seed in $all_seeds; do + head -n-2 config/germinate-output/${seed}.seed|tail -n+3|awk '{print $1}' + done|sort -u +} + +add_layered_pass() { + # Add a layer to an existing pass based on seeds matching a regexp + # $1 base pass + # $2 seeds (regexp) + + for seed in $(ls config/germinate-output/|grep -P "$2"); do + pass=${1}_${seed} + list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass + done +} + +add_layered_pass_delta() { + # Add a layer to an existing pass based on delta between seeds matching a regexp and a base seed + # $1 base pass + # $2 base seed + # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. + + local seed_regexp="$3" + if [ -z "${seed_regexp}" ]; then + substract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-{$2} + return + fi + + for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do + pass=${1}_${seed} + substract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + done +} + +substract_package_lists() { + # Substract a package list from another + # + # $1 source package list + # $2 Package list to substract from source package list + local list1=$(mktemp) + local list2=$(mktemp) + + list_packages_from_seed $1 > list1 + list_packages_from_seed $2 > list2 + comm -23 list1 list2 + + rm list1 + rm list2 +} diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 41ebef64..fc3dc275 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -191,7 +191,7 @@ create_chroot_pass() { # Handle direct sublayer of current one # Extract the name of the pass corresponding to the sublayer - for subpass in $(ls config/package-lists/*.list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do + for subpass in $(ls config/package-lists/*list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" create_chroot_pass "${subpass}" "${pass}" "${lowerlayers_for_subpass}" "" ${*} From 42d1d7761206c12ab78acd2f33114f42bfddb9c8 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 13:09:05 +0100 Subject: [PATCH 20/44] Layered Ubuntu desktop by default Default Ubuntu desktop to the layered format Remove the requirement on setting IMAGEFORMAT to layered to build a layered image and relies on PASSED instead. --- live-build/auto/build | 4 ++-- live-build/auto/config | 33 +++++++++++++-------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index 07025195..d6642cd9 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -305,7 +305,7 @@ rm -f /usr/sbin/update-initramfs dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs EOF - if [ "${IMAGEFORMAT:-}" = live-layered ]; then + if [ -n "${PASSES}" ]; then PATH="config/:$PATH" lb chroot_layered "$@" else lb chroot "$@" @@ -536,7 +536,7 @@ EOF fi # Create layered squashfs system - if [ "${IMAGEFORMAT:-}" = live-layered ]; then + if [ -n "${PASSES}" ]; then CURPASS=1 PREVPASS="" PREFIX="" diff --git a/live-build/auto/config b/live-build/auto/config index 0b413631..df64f296 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -70,9 +70,6 @@ add_binary_hook () } if [ -z "${IMAGEFORMAT:-}" ]; then - if [ ${SUBPROJECT:-} = layered ]; then - IMAGEFORMAT=live-layered - fi case $PROJECT:${SUBPROJECT:-} in ubuntu-cpc:*) if [ "$SUBARCH" = "raspi3" ]; then @@ -193,12 +190,6 @@ case $IMAGEFORMAT in exit 0 ;; - live-layered) - # Stop lb creating filesystem.squashfs by skipping lb_binary_rootfs and lb_binary_manifest - skip_lb_stage binary_rootfs - skip_lb_stage binary_manifest - ;; - none) OPTS="${OPTS:+$OPTS }--chroot-filesystem $IMAGEFORMAT" ;; @@ -213,6 +204,12 @@ case $IMAGEFORMAT in ;; esac +if [ -n "${PASSES}" ]; then + # Stop lb creating filesystem.squashfs by skipping lb_binary_rootfs and lb_binary_manifest + skip_lb_stage binary_rootfs + skip_lb_stage binary_manifest +fi + if [ "$PREINSTALLED" = "true" ]; then # Touch a random file that we can refer back to during build, # cause that's wildly hackish @@ -307,22 +304,18 @@ esac case $PROJECT in ubuntu|ubuntu-dvd) + PASSES="install-minimal install live" + add_task install-minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages + add_task install ubuntu-desktop ubuntu-desktop-default-languages + LIVE_TASK='ubuntu-live' + case ${SUBPROJECT:-} in layered) - PASSES="install-minimal install live" - add_task install-minimal minimal standard ubuntu-desktop-minimal - # LANG PASS for minimal + # LANG PASS for minimal and install add_layered_pass_delta install-minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' add_layered_pass_delta install-minimal desktop-minimal-default-languages 'nolangpacks' # none (if no default langpack is selected) - add_task install ubuntu-desktop - # LANG PASS for install add_layered_pass_delta install desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' add_layered_pass_delta install desktop-default-languages 'nolangpacks' # none (if no default langpack is selected) - LIVE_TASK='ubuntu-live' - ;; - *) - add_task install minimal standard ubuntu-desktop - LIVE_TASK='ubuntu-live' ;; esac @@ -689,7 +682,7 @@ case $SUBPROJECT in ;; esac -if [ "${IMAGEFORMAT:-}" != live-layered ]; then +if [ -z "${PASSES}" ]; then # we'll expand the base seed given here according to the STRUCTURE file, and # then look in all of the seeds found to see which snaps are seeded case $PROJECT:${SUBPROJECT:-} in From 3b21bd9a2e791aa39ff6aaeb5b9328339e372e0d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 17:13:26 +0100 Subject: [PATCH 21/44] Moved back config helper functions to config Moved back config specific functions from functions to config --- live-build/auto/config | 92 ++++++++++++++++++++++++++++++++++++++++++ live-build/functions | 92 ------------------------------------------ 2 files changed, 92 insertions(+), 92 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index df64f296..1a74e87b 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -59,6 +59,98 @@ BINARY_HOOKS= APT_OPTIONS=" --yes -oDebug::pkgDepCache::AutoInstall=yes " +add_task () +{ + local pass="$1" + shift + local task + local snap_list_file + local snap_list_files + local curseed + + # The removal of direct task installation support from live-build + # poses some problems. If the chroot has multiarch configured - for + # example, if we're building for amd64 - then dumpavail will show + # foreign-architecture packages which will have their own Task + # lines, but which we don't want to install. (Compare + # PackageContainerInterface::FromTask in apt, which restricts task + # expansion to the native architecture.) We therefore restrict our + # search to stanzas with Architecture: $ARCH or all. + # + # However, even this may not be accurate enough. At the moment I + # have no idea what happens if an Architecture: all package has + # different Task fields on different architectures. This is + # probably a lurking timebomb that we need to fix. In the meantime, + # the Architecture restriction at least saves us from abject + # failure. + # + # We want as well to grab the snap list for each PASS. Resolve for all + # given task, and deduplicate them to generate snaps for the PASS. + + for task; do + # We need a ridiculous number of backslashes to protect + # parentheses from eval. + echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" + + curseed=$(seed_from_task ${task}) + if [ -z "${curseed}" ]; then + echo "W: No seed matching task ${task}" + continue + fi + snap_list_file="config/package-lists/seed.${curseed}.snaplist.full" + snap_from_seed "${curseed}" $snap_list_file + if [ -e "$snap_list_file" ]; then + snap_list_files="${snap_list_files} $snap_list_file" + fi + done + # The snap list is one line, and could be duplicated between seeds via inheritance. + # Uniquely sort them and store them back in one line. + if [ -n "${snap_list_files}" ]; then + cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full" + rm ${snap_list_files} + fi +} + +add_package () +{ + local pass="$1" + shift + local pkg + + for pkg; do + echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" + done +} + +add_layered_pass() { + # Add a layer to an existing pass based on seeds matching a regexp + # $1 base pass + # $2 seeds (regexp) + + for seed in $(ls config/germinate-output/|grep -P "$2"); do + pass=${1}_${seed} + list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass + done +} + +add_layered_pass_delta() { + # Add a layer to an existing pass based on delta between seeds matching a regexp and a base seed + # $1 base pass + # $2 base seed + # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. + + local seed_regexp="$3" + if [ -z "${seed_regexp}" ]; then + substract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-{$2} + return + fi + + for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do + pass=${1}_${seed} + substract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + done +} + add_chroot_hook () { CHROOT_HOOKS="${CHROOT_HOOKS:+$CHROOT_HOOKS }$1" diff --git a/live-build/functions b/live-build/functions index 8635d2c4..615acfe0 100644 --- a/live-build/functions +++ b/live-build/functions @@ -581,69 +581,6 @@ seed_from_task () fi } -add_task () -{ - local pass="$1" - shift - local task - local snap_list_file - local snap_list_files - local curseed - - # The removal of direct task installation support from live-build - # poses some problems. If the chroot has multiarch configured - for - # example, if we're building for amd64 - then dumpavail will show - # foreign-architecture packages which will have their own Task - # lines, but which we don't want to install. (Compare - # PackageContainerInterface::FromTask in apt, which restricts task - # expansion to the native architecture.) We therefore restrict our - # search to stanzas with Architecture: $ARCH or all. - # - # However, even this may not be accurate enough. At the moment I - # have no idea what happens if an Architecture: all package has - # different Task fields on different architectures. This is - # probably a lurking timebomb that we need to fix. In the meantime, - # the Architecture restriction at least saves us from abject - # failure. - # - # We want as well to grab the snap list for each PASS. Resolve for all - # given task, and deduplicate them to generate snaps for the PASS. - - for task; do - # We need a ridiculous number of backslashes to protect - # parentheses from eval. - echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" - - curseed=$(seed_from_task ${task}) - if [ -z "${curseed}" ]; then - echo "W: No seed matching task ${task}" - continue - fi - snap_list_file="config/package-lists/seed.${curseed}.snaplist.full" - snap_from_seed "${curseed}" $snap_list_file - if [ -e "$snap_list_file" ]; then - snap_list_files="${snap_list_files} $snap_list_file" - fi - done - # The snap list is one line, and could be duplicated between seeds via inheritance. - # Uniquely sort them and store them back in one line. - if [ -n "${snap_list_files}" ]; then - cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full" - rm ${snap_list_files} - fi -} - -add_package () -{ - local pass="$1" - shift - local pkg - - for pkg; do - echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" - done -} - list_packages_from_seed () { # Store all packages for a given seed, including its seed dependency # $1: Name of the seed to expand to a package list @@ -655,35 +592,6 @@ list_packages_from_seed () { done|sort -u } -add_layered_pass() { - # Add a layer to an existing pass based on seeds matching a regexp - # $1 base pass - # $2 seeds (regexp) - - for seed in $(ls config/germinate-output/|grep -P "$2"); do - pass=${1}_${seed} - list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass - done -} - -add_layered_pass_delta() { - # Add a layer to an existing pass based on delta between seeds matching a regexp and a base seed - # $1 base pass - # $2 base seed - # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. - - local seed_regexp="$3" - if [ -z "${seed_regexp}" ]; then - substract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-{$2} - return - fi - - for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do - pass=${1}_${seed} - substract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass - done -} - substract_package_lists() { # Substract a package list from another # From 9077289332d1c22b4f2533d38eb1aadc0c793c9e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 17:35:04 +0100 Subject: [PATCH 22/44] Add includes by pass lb_chroot_includes now takes the name of the pass as first argument and will includes files for the corresponding pass from the directory include.chroot. --- live-build/lb_chroot_layered | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index fc3dc275..8a064960 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -93,6 +93,19 @@ lb_chroot_install_snaps() { done < $snaplist_file } +lb_chroot_includes() { + # Copying includes from pass subdirectory + local pass="$1" + + if [ ! -d config/includes.chroot.${pass} ]; then + return + fi + + cd config/includes.chroot.${pass} + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot + cd "${OLDPWD}" +} + create_chroot_pass() { local pass=$1 local prevpass=$2 @@ -158,7 +171,9 @@ create_chroot_pass() { lb chroot_live-packages ${*} fi - lb chroot_includes ${*} + # Run includes by pass + lb_chroot_includes ${pass} ${*} + lb chroot_hooks ${*} lb chroot_hacks ${*} lb chroot_interactive ${*} From 047f883e92d5aa3195320e2ae89271b79f50793b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Tue, 20 Nov 2018 17:51:41 +0100 Subject: [PATCH 23/44] Various fixes Fixed prefix for passes to not conflict with global prefix Set layer to none for layer without language packs Copy size and manifest to the target directory Make sure the diff always diffes existing manifests --- live-build/auto/build | 13 +++++++------ live-build/auto/config | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index d6642cd9..fca3b0e9 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -47,7 +47,7 @@ build_layered_squashfs() { create_manifest "chroot" "${squashfs_f_manifest}.full" # Delta manifest - diff -NU0 ${PWD}/livecd.${PROJECT}.${prefix}${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + diff -U0 ${PWD}/livecd.${PROJECT}.[0-9]{2..2}-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest squashfs_f_size="${base}.size" du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}" @@ -539,13 +539,13 @@ EOF if [ -n "${PASSES}" ]; then CURPASS=1 PREVPASS="" - PREFIX="" + PASSPREFIX="" LOWER_LAYERS="" for _PASS in $PASSES do - PREFIX="$(printf "%02g" $CURPASS)-" - build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PREFIX" ${LOWER_LAYERS} + PASSPREFIX="$(printf "%02g" $CURPASS)-" + build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PASSPREFIX" ${LOWER_LAYERS} LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" LOWER_LAYERS="${LOWER_LAYERS%:}" @@ -553,8 +553,9 @@ EOF CURPASS=$(( CURPASS + 1 )) done - # Full ISO manifest - cp "${PWD}/livecd.${PROJECT}.${PREFIX}${_PASS}.manifest.full" livecd.${PROJECT}.manifest + # Full ISO manifest & size from last PASS + cp "${PWD}/livecd.${PROJECT}.${PASSPREFIX}${_PASS}.size" "binary/$INITFS/filesystem.size" + cp "${PWD}/livecd.${PROJECT}.${PASSPREFIX}${_PASS}.manifest.full" "binary/$INITFS/filesystem.manifest" fi touch binary.success diff --git a/live-build/auto/config b/live-build/auto/config index 1a74e87b..a75d4875 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -141,7 +141,7 @@ add_layered_pass_delta() { local seed_regexp="$3" if [ -z "${seed_regexp}" ]; then - substract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-{$2} + substract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-${2} return fi @@ -405,9 +405,9 @@ case $PROJECT in layered) # LANG PASS for minimal and install add_layered_pass_delta install-minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' - add_layered_pass_delta install-minimal desktop-minimal-default-languages 'nolangpacks' # none (if no default langpack is selected) + add_layered_pass_delta install-minimal desktop-minimal-default-languages '' # none (if no default langpack is selected) add_layered_pass_delta install desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' - add_layered_pass_delta install desktop-default-languages 'nolangpacks' # none (if no default langpack is selected) + add_layered_pass_delta install desktop-default-languages '' # none (if no default langpack is selected) ;; esac From 693e0a73606064d780653897dc90e8c90e9ad159 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 21 Nov 2018 10:36:06 +0100 Subject: [PATCH 24/44] Clean up chroot helpers Removed obsolete chroot helpers on Disco+: - selinux - sysv-rc - upstart Added a note about needlessly triggering chroot_archives on each pass. --- live-build/lb_chroot_layered | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 8a064960..c05d4507 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -129,17 +129,18 @@ create_chroot_pass() { lb chroot_cache restore ${*} lb chroot_devpts install ${*} lb chroot_proc install ${*} - lb chroot_selinuxfs install ${*} lb chroot_sysfs install ${*} lb chroot_debianchroot install ${*} lb chroot_dpkg install ${*} lb chroot_tmpfs install ${*} - lb chroot_sysv-rc install ${*} - lb chroot_upstart install ${*} lb chroot_hosts install ${*} lb chroot_resolv install ${*} lb chroot_hostname install ${*} lb chroot_apt install ${*} + # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more + # diff content than desired. However, we still need to setup the archive and teardown + # for each layer. + # We could modify livebuild if necessary to have conditional upgrade (first pass only). lb chroot_archives chroot install ${*} # Customizing chroot @@ -186,13 +187,10 @@ create_chroot_pass() { lb chroot_hostname remove ${*} lb chroot_resolv remove ${*} lb chroot_hosts remove ${*} - lb chroot_sysv-rc remove ${*} - lb chroot_upstart remove ${*} lb chroot_tmpfs remove ${*} lb chroot_dpkg remove ${*} lb chroot_debianchroot remove ${*} lb chroot_sysfs remove ${*} - lb chroot_selinuxfs remove ${*} lb chroot_proc remove ${*} lb chroot_devpts remove ${*} lb chroot_cache save ${*} From 43ec452fa01dc12ff2e62e48303e46cd73c6640b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 21 Nov 2018 17:16:50 +0100 Subject: [PATCH 25/44] Moved reusable functions to functions Moved functions to reuse in lb_binary and lb_chroot to functions --- live-build/auto/build | 290 ++++++++++++++++-------------------------- live-build/functions | 86 +++++++++++++ 2 files changed, 197 insertions(+), 179 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index fca3b0e9..1eafe95e 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -311,105 +311,51 @@ EOF lb chroot "$@" fi - if [ "${SUBPROJECT:-}" = minimized ]; then - # force removal of initramfs-tools, which we assert is not - # required for any minimized images but is still pulled in by - # default - # also remove landscape-common, which is heavyweight and - # in the server seed only to provide /etc/motd content which - # would only be seen by humans - Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ - apt-get -y purge initramfs-tools busybox-initramfs \ - busybox-static landscape-common" - # and if initramfs-tools was configured before our kernel, - # /etc/kernel/postinst.d/initramfs-tools will have created - # an initramfs despite the generic dpkg-divert; so remove it - # here. - rm -f chroot/boot/initrd.img-* - - # temporary workaround: don't remove linux-base which - # may have no other reverse-depends currently - Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ - apt-mark manual linux-base" - Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ - apt-get -y --purge autoremove" - fi + # Let all configuration non multi-layered project here. + # If those are moving to a multi-layer layout, this needs to be + # done in chroot hooks. + if [ -z "$PASSES" ]; then + if [ "${SUBPROJECT:-}" = minimized ]; then + # force removal of initramfs-tools, which we assert is not + # required for any minimized images but is still pulled in by + # default + # also remove landscape-common, which is heavyweight and + # in the server seed only to provide /etc/motd content which + # would only be seen by humans + Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ + apt-get -y purge initramfs-tools busybox-initramfs \ + busybox-static landscape-common" + # and if initramfs-tools was configured before our kernel, + # /etc/kernel/postinst.d/initramfs-tools will have created + # an initramfs despite the generic dpkg-divert; so remove it + # here. + rm -f chroot/boot/initrd.img-* + + # temporary workaround: don't remove linux-base which + # may have no other reverse-depends currently + Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ + apt-mark manual linux-base" + Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ + apt-get -y --purge autoremove" + fi - # remove crufty files that shouldn't be left in an image - rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old - Chroot chroot apt clean - if [ "${PROJECT}:${SUBPROJECT:-}" = "ubuntu-base:minimized" ]; then - # Save even more size by removing apt lists (that are currently removed - # downstream anyway) - rm -rf chroot/var/lib/apt/lists/* - # Having device notes in the docker image can cause problems - # (https://github.com/tianon/docker-brew-ubuntu-core/issues/62) - # so remove them. We only do this for docker out of an - # abundance of caution. - rm -rf chroot/dev/* - fi + clean_debian_chroot + + if [ "${PROJECT}:${SUBPROJECT:-}" = "ubuntu-base:minimized" ]; then + # Save even more size by removing apt lists (that are currently removed + # downstream anyway) + rm -rf chroot/var/lib/apt/lists/* + # Having device notes in the docker image can cause problems + # (https://github.com/tianon/docker-brew-ubuntu-core/issues/62) + # so remove them. We only do this for docker out of an + # abundance of caution. + rm -rf chroot/dev/* + fi - if [ -f config/universe-enabled ]; then - -# This is cargo-culted almost verbatim (with some syntax changes for -# preinstalled being slightly different in what it doesn't ask) from -# debian-installer's apt-setup: - -cat > chroot/etc/apt/sources.list << EOF -# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to -# newer versions of the distribution. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted - -## Major bug fix updates produced after the final release of the -## distribution. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted - -## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -## team. Also, please note that software in universe WILL NOT receive any -## review or updates from the Ubuntu security team. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe - -## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -## team, and may not be under a free licence. Please satisfy yourself as to -## your rights to use the software. Also, please note that software in -## multiverse WILL NOT receive any review or updates from the Ubuntu -## security team. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse - -## N.B. software from this repository may not have been tested as -## extensively as that contained in the main release, although it includes -## newer versions of some applications which may provide useful features. -## Also, please note that software in backports WILL NOT receive any review -## or updates from the Ubuntu security team. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse - -## Uncomment the following two lines to add software from Canonical's -## 'partner' repository. -## This software is not part of Ubuntu, but is offered by Canonical and the -## respective vendors as a service to Ubuntu users. -# deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner -# deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner - -deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted -# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted -deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe -# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe -deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse -# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse -EOF + configure_universe - fi - if [ -d chroot/var/lib/preinstalled-pool ]; then - cat > config/indices/apt.conf <<-EOF + if [ -d chroot/var/lib/preinstalled-pool ]; then + cat > config/indices/apt.conf <<-EOF Dir { ArchiveDir "chroot/var/lib/preinstalled-pool"; OverrideDir "config/indices"; @@ -426,11 +372,11 @@ Tree "dists/$LB_DISTRIBUTION" Contents " "; } EOF - for component in $LB_PARENT_ARCHIVE_AREAS; do - mkdir -p chroot/var/lib/preinstalled-pool/dists/$LB_DISTRIBUTION/$component/binary-$LB_ARCHITECTURES - done - apt-ftparchive generate config/indices/apt.conf - cat << @@EOF > chroot/etc/apt/sources.list.preinstall + for component in $LB_PARENT_ARCHIVE_AREAS; do + mkdir -p chroot/var/lib/preinstalled-pool/dists/$LB_DISTRIBUTION/$component/binary-$LB_ARCHITECTURES + done + apt-ftparchive generate config/indices/apt.conf + cat << @@EOF > chroot/etc/apt/sources.list.preinstall # This is a sources.list entry for a small pool of packages # provided on your preinstalled filesystem for your convenience. # @@ -442,97 +388,83 @@ deb file:/var/lib/preinstalled-pool/ $LB_DISTRIBUTION $LB_PARENT_ARCHIVE_AREAS # @@EOF - cp chroot/etc/apt/sources.list chroot/etc/apt/sources.list.orig - cp chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list - - echo "Waiting on gnupg ("$GPG_PROCESS") to finish generating a key." - wait $GPG_PROCESS - - R_ORIGIN=$(lsb_release -i -s) - R_CODENAME=$(lsb_release -c -s) - R_VERSION=$(lsb_release -r -s) - R_PRETTYNAME=$(echo $R_CODENAME | sed -e 's/^\(.\)/\U\1/') - - apt-ftparchive -o APT::FTPArchive::Release::Origin=$R_ORIGIN \ - -o APT::FTPArchive::Release::Label=$R_ORIGIN \ - -o APT::FTPArchive::Release::Suite=$R_CODENAME-local \ - -o APT::FTPArchive::Release::Version=$R_VERSION \ - -o APT::FTPArchive::Release::Codename=$R_CODENAME \ - -o APT::FTPArchive::Release::Description="$R_ORIGIN $R_PRETTYNAME Local" \ - release chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/ \ - > config/gnupg/Release - - gpg --home config/gnupg --detach-sign --armor config/gnupg/Release - mv config/gnupg/Release \ - chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release - mv config/gnupg/Release.asc \ - chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release.gpg - apt-key --keyring chroot/etc/apt/trusted.gpg add config/gnupg/pubring.gpg - find chroot/var/lib/preinstalled-pool/ -name Packages | xargs rm - - Chroot chroot "apt-get update" - cat chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig \ - > chroot/etc/apt/sources.list - rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig - fi - case $PROJECT:$SUBPROJECT in - *) - if [ -e "config/seeded-snaps" ]; then - snap_list=$(cat config/seeded-snaps) - preinstall_snaps $snap_list - fi - ;; - esac - - if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then - if [ "$ARCH" = "armhf" ]; then - INFO_DESC="$(lsb_release -d -s)" - echo "$INFO_DESC - $ARCH ($BUILDSTAMP)" >chroot/etc/media-info - mkdir -p chroot/var/log/installer - Chroot chroot "ln -s /etc/media-info /var/log/installer/media-info" + cp chroot/etc/apt/sources.list chroot/etc/apt/sources.list.orig + cp chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list + + echo "Waiting on gnupg ("$GPG_PROCESS") to finish generating a key." + wait $GPG_PROCESS + + R_ORIGIN=$(lsb_release -i -s) + R_CODENAME=$(lsb_release -c -s) + R_VERSION=$(lsb_release -r -s) + R_PRETTYNAME=$(echo $R_CODENAME | sed -e 's/^\(.\)/\U\1/') + + apt-ftparchive -o APT::FTPArchive::Release::Origin=$R_ORIGIN \ + -o APT::FTPArchive::Release::Label=$R_ORIGIN \ + -o APT::FTPArchive::Release::Suite=$R_CODENAME-local \ + -o APT::FTPArchive::Release::Version=$R_VERSION \ + -o APT::FTPArchive::Release::Codename=$R_CODENAME \ + -o APT::FTPArchive::Release::Description="$R_ORIGIN $R_PRETTYNAME Local" \ + release chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/ \ + > config/gnupg/Release + + gpg --home config/gnupg --detach-sign --armor config/gnupg/Release + mv config/gnupg/Release \ + chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release + mv config/gnupg/Release.asc \ + chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release.gpg + apt-key --keyring chroot/etc/apt/trusted.gpg add config/gnupg/pubring.gpg + find chroot/var/lib/preinstalled-pool/ -name Packages | xargs rm + + Chroot chroot "apt-get update" + cat chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig \ + > chroot/etc/apt/sources.list + rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig fi - fi - if [ "$PROJECT" = "ubuntu-cpc" ]; then - if [ "${SUBPROJECT:-}" = minimized ]; then - BUILD_NAME=minimal - else - BUILD_NAME=server + case $PROJECT:$SUBPROJECT in + *) + if [ -e "config/seeded-snaps" ]; then + snap_list=$(cat config/seeded-snaps) + preinstall_snaps $snap_list + fi + ;; + esac + + if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then + if [ "$ARCH" = "armhf" ]; then + INFO_DESC="$(lsb_release -d -s)" + echo "$INFO_DESC - $ARCH ($BUILDSTAMP)" >chroot/etc/media-info + mkdir -p chroot/var/log/installer + Chroot chroot "ln -s /etc/media-info /var/log/installer/media-info" + fi fi - cat > chroot/etc/cloud/build.info << EOF + if [ "$PROJECT" = "ubuntu-cpc" ]; then + if [ "${SUBPROJECT:-}" = minimized ]; then + BUILD_NAME=minimal + else + BUILD_NAME=server + fi + cat > chroot/etc/cloud/build.info << EOF build_name: $BUILD_NAME serial: $BUILDSTAMP EOF - fi + fi - # If the image pre-installs network-manager, let it manage all devices by - # default. Installing NM on an existing system only manages wifi and wwan via - # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting - # the global backend to NM, netplan overrides that file. - if [ -e chroot/usr/sbin/NetworkManager ]; then - echo "===== Enabling all devices in NetworkManager ====" - mkdir -p chroot/etc/netplan - cat < chroot/etc/netplan/01-network-manager-all.yaml -# Let NetworkManager manage all devices on this system -network: - version: 2 - renderer: NetworkManager -EOF - else - echo "==== NetworkManager not installed ====" - fi + configure_network_manager - echo "===== Checking size of /usr/share/doc =====" - echo BEGIN docdirs - (cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr) - echo END docdirs + echo "===== Checking size of /usr/share/doc =====" + echo BEGIN docdirs + (cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr) + echo END docdirs - /usr/share/livecd-rootfs/minimize-manual chroot + /usr/share/livecd-rootfs/minimize-manual chroot + fi lb binary "$@" # Teardown mount points in layered mode if [ -x config/teardown_chroot_layered ]; then - config/teardown_chroot_layered + config/teardown_chroot_layered fi # Create layered squashfs system diff --git a/live-build/functions b/live-build/functions index 615acfe0..33ef79e2 100644 --- a/live-build/functions +++ b/live-build/functions @@ -607,3 +607,89 @@ substract_package_lists() { rm list1 rm list2 } + +clean_debian_chroot() { + # remove crufty files that shouldn't be left in an image + rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old + Chroot chroot apt clean +} + +configure_universe() { + if [ -f config/universe-enabled ]; then + # This is cargo-culted almost verbatim (with some syntax changes for + # preinstalled being slightly different in what it doesn't ask) from + # debian-installer's apt-setup: + + cat > chroot/etc/apt/sources.list << EOF +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner +# deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner + +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse +EOF + +fi +} + +configure_network_manager() { + # If the image pre-installs network-manager, let it manage all devices by + # default. Installing NM on an existing system only manages wifi and wwan via + # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting + # the global backend to NM, netplan overrides that file. + if [ -e chroot/usr/sbin/NetworkManager ]; then + echo "===== Enabling all devices in NetworkManager ====" + mkdir -p chroot/etc/netplan + cat < chroot/etc/netplan/01-network-manager-all.yaml +# Let NetworkManager manage all devices on this system +network: + version: 2 + renderer: NetworkManager +EOF + else + echo "==== NetworkManager not installed ====" + fi +} From 9dba350834ded2ea0dbbf8cf1c268e0bf12be85f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 21 Nov 2018 17:30:32 +0100 Subject: [PATCH 26/44] Added lb_binary_layered Created lb_binary_layered based on lb_binary Factorized several helpers These helpers are called from lb_chroot --- live-build/auto/build | 81 +---------------------- live-build/auto/config | 8 +-- live-build/lb_binary_layered | 125 +++++++++++++++++++++++++++++++++++ live-build/lb_chroot_layered | 22 +++--- 4 files changed, 139 insertions(+), 97 deletions(-) create mode 100755 live-build/lb_binary_layered diff --git a/live-build/auto/build b/live-build/auto/build index 1eafe95e..9be0af20 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -17,57 +17,6 @@ fi . config/functions -build_layered_squashfs() { - local pass=$1 # install|install_subpass|install_subpass_subsubpass|… - local prevpass=$2 # install|install_subpass|… - local prefix=$3 # 01-|02-|… - local lowerlayers=$4 - - base="${PWD}/livecd.${PROJECT}.${prefix}${pass}" - squashfs_f="${base}.squashfs" - - # We have already treated that pass - if [ -f "${squashfs_f}" ]; then - return - fi - - if [ -n "${lowerlayers}" ]; then - mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ - else - # first pass - ln -s "chroot.${pass}/" chroot - fi - - (cd "chroot.${pass}/" && - mksquashfs . ${squashfs_f} \ - -no-progress -xattrs -comp xz ) - - # Full manifest until that PASS - squashfs_f_manifest="${base}.manifest" - create_manifest "chroot" "${squashfs_f_manifest}.full" - - # Delta manifest - diff -U0 ${PWD}/livecd.${PROJECT}.[0-9]{2..2}-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest - - squashfs_f_size="${base}.size" - du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}" - - if [ -n "${lowerlayers}" ]; then - umount chroot - else - rm chroot - mkdir chroot/ - fi - - # Handle direct sublayer of current one - # Extract the name of the pass corresponding to the sublayer - for subpass in $(ls -d chroot.${pass}_* 2>/dev/null | sed -e "s/chroot\.\(${pass}_[^_]\+\).*/\1/"); do - lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" - lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" - build_layered_squashfs "${subpass}" "${pass}" "${prefix}" "${lowerlayers_for_subpass}" - done -} - # Link output files somewhere launchpad-buildd will be able to find them. PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" @@ -460,34 +409,10 @@ EOF /usr/share/livecd-rootfs/minimize-manual chroot fi - lb binary "$@" - - # Teardown mount points in layered mode - if [ -x config/teardown_chroot_layered ]; then - config/teardown_chroot_layered - fi - - # Create layered squashfs system if [ -n "${PASSES}" ]; then - CURPASS=1 - PREVPASS="" - PASSPREFIX="" - LOWER_LAYERS="" - - for _PASS in $PASSES - do - PASSPREFIX="$(printf "%02g" $CURPASS)-" - build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PASSPREFIX" ${LOWER_LAYERS} - - LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" - LOWER_LAYERS="${LOWER_LAYERS%:}" - PREVPASS=${_PASS} - CURPASS=$(( CURPASS + 1 )) - done - - # Full ISO manifest & size from last PASS - cp "${PWD}/livecd.${PROJECT}.${PASSPREFIX}${_PASS}.size" "binary/$INITFS/filesystem.size" - cp "${PWD}/livecd.${PROJECT}.${PASSPREFIX}${_PASS}.manifest.full" "binary/$INITFS/filesystem.manifest" + PATH="config/:$PATH" lb binary_layered "$@" + else + lb binary "$@" fi touch binary.success diff --git a/live-build/auto/config b/live-build/auto/config index a75d4875..bc309f6b 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -33,7 +33,7 @@ fi mkdir -p config cp -af /usr/share/livecd-rootfs/live-build/functions config/functions -cp -af /usr/share/livecd-rootfs/live-build/lb_chroot_layered config/lb_chroot_layered +cp -af /usr/share/livecd-rootfs/live-build/lb_*_layered config/ cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse mkdir -p config/package-lists @@ -296,12 +296,6 @@ case $IMAGEFORMAT in ;; esac -if [ -n "${PASSES}" ]; then - # Stop lb creating filesystem.squashfs by skipping lb_binary_rootfs and lb_binary_manifest - skip_lb_stage binary_rootfs - skip_lb_stage binary_manifest -fi - if [ "$PREINSTALLED" = "true" ]; then # Touch a random file that we can refer back to during build, # cause that's wildly hackish diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered new file mode 100755 index 00000000..848cc9ab --- /dev/null +++ b/live-build/lb_binary_layered @@ -0,0 +1,125 @@ +#!/bin/sh + +## live-build(7) - System Build Scripts +## Copyright (C) 2006-2012 Daniel Baumann +## +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING. +## This is free software, and you are welcome to redistribute it +## under certain conditions; see COPYING for details. + + +set -e + +# Including common functions +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh + + +# Automatically populating config tree +if [ -x auto/config ] && [ ! -e .build/config ] +then + Echo_message "Automatically populating config tree." + lb config +fi + +# Setting static variables +DESCRIPTION="$(Echo 'build binary images')" +HELP="" +USAGE="${PROGRAM} [--force]" + +Arguments "${@}" + +# Reading configuration files +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source +Set_defaults + +# Setup cleanup function +Setup_cleanup + +. config/functions + +build_layered_squashfs() { + local pass=$1 # install|install_subpass|install_subpass_subsubpass|… + local prevpass=$2 # install|install_subpass|… + local prefix=$3 # 01-|02-|… + local lowerlayers=$4 + shift 4 # restore ${*} + + # Cleanup root filesystem + lb binary_chroot ${*} + + # Building squashfs filesystem & manifest + base="${PWD}/livecd.${PROJECT}.${prefix}${pass}" + squashfs_f="${base}.squashfs" + + # We have already treated that pass + if [ -f "${squashfs_f}" ]; then + return + fi + + if [ -n "${lowerlayers}" ]; then + mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ + else + # first pass + rmdir chroot||true + ln -s "chroot.${pass}/" chroot + fi + + # Full manifest until that PASS + squashfs_f_manifest="${base}.manifest" + create_manifest "chroot" "${squashfs_f_manifest}.full" + + # Delta manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.[0-9]{2..2}-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + + squashfs_f_size="${base}.size" + du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}" + + (cd "chroot.${pass}/" && + mksquashfs . ${squashfs_f} \ + -no-progress -xattrs -comp xz ) + + if [ -n "${lowerlayers}" ]; then + umount chroot + else + rm chroot + mkdir chroot/ + fi + + # Handle direct sublayer of current one + # Extract the name of the pass corresponding to the sublayer + for subpass in $(ls -d chroot.${pass}_* 2>/dev/null | sed -e "s/chroot\.\(${pass}_[^_]\+\).*/\1/"); do + lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" + lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" + build_layered_squashfs "${subpass}" "${pass}" "${prefix}" "${lowerlayers_for_subpass}" ${*} + done +} + +CURPASS=1 +PREVPASS="" +PASSPREFIX="" +LOWER_LAYERS="" +for _PASS in $PASSES +do + PASSPREFIX="$(printf "%02g" $CURPASS)-" + + build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PASSPREFIX" "${LOWER_LAYERS}" ${*} + + LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" + LOWER_LAYERS="${LOWER_LAYERS%:}" + PREVPASS=${_PASS} + CURPASS=$(( CURPASS + 1 )) +done + +# remount last "main" pass on chroot for lb binary +mount_overlay "${LOWER_LAYERS}" "chroot.${_PASS}/" chroot/ + +# Prepare initrd + kernel +lb binary_linux-image ${*} + +umount chroot/ + +# Full ISO manifest & size from last main PASS +PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" +cp "livecd.${PROJECT}.${PASSPREFIX}${_PASS}.size" "$PREFIX.size" +cp "livecd.${PROJECT}.${PASSPREFIX}${_PASS}.manifest.full" "$PREFIX.manifest" +chmod 644 *.squashfs *.manifest *.manifest.full *.size diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index c05d4507..8ec915b8 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -143,6 +143,11 @@ create_chroot_pass() { # We could modify livebuild if necessary to have conditional upgrade (first pass only). lb chroot_archives chroot install ${*} + if [ "${passtype}" = "first" ]; then + configure_universe + configure_network_manager + fi + # Customizing chroot lb chroot_linux-image ${*} lb chroot_preseed ${*} @@ -179,6 +184,10 @@ create_chroot_pass() { lb chroot_hacks ${*} lb chroot_interactive ${*} + # Misc ubuntu cleanup and post-layer configuration + clean_debian_chroot + /usr/share/livecd-rootfs/minimize-manual chroot + Chroot chroot "dpkg-query -W" > chroot.packages.${pass} # Deconfiguring chroot @@ -234,15 +243,4 @@ do CURPASS=$(( CURPASS + 1 )) done -# remount last pass on chroot for lb binary -mount_overlay "${LOWER_LAYERS}" "chroot.${_PASS}/" chroot/ - -# Remove unused chroot binary corresponding to bottom layer (once everything is unmount) -TEARDOWNPATH="config/teardown_chroot_layered" -cat > $TEARDOWNPATH << EOF -#!/bin/sh -set -e -umount chroot -rm -rf chroot/ -EOF -chmod +x $TEARDOWNPATH +rmdir chroot From 3075d6557885a20f8c91ddf90f574733016d88ec Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 22 Nov 2018 10:19:59 +0100 Subject: [PATCH 27/44] Cleanup and handle manifest-remove files manifest-remove files are generated in all cases and removed for projects/subprojects that don't need it like the new ubiquity. --- live-build/auto/build | 6 ++++ live-build/auto/config | 59 +++++++++++++++++------------------- live-build/lb_binary_layered | 11 +++++-- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index 9be0af20..edd9e98c 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -447,6 +447,12 @@ for OUTPUT in ext2 ext3 ext4 manifest manifest-remove size squashfs; do chmod 644 "$PREFIX.$OUTPUT" done +# we don't need a manifest-remove for a layered-aware installer +if [ "$SUBPROJECT" = "ubiquity-ng" ]; then + rm -f livecd.${PROJECT}-manifest-remove + rm -f config/manifest-minimal-remove +fi + if [ -e config/manifest-minimal-remove ]; then cp config/manifest-minimal-remove "$PREFIX.manifest-minimal-remove" fi diff --git a/live-build/auto/config b/live-build/auto/config index bc309f6b..d6d62cbb 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -396,7 +396,7 @@ case $PROJECT in LIVE_TASK='ubuntu-live' case ${SUBPROJECT:-} in - layered) + ubiquity-ng) # LANG PASS for minimal and install add_layered_pass_delta install-minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' add_layered_pass_delta install-minimal desktop-minimal-default-languages '' # none (if no default langpack is selected) @@ -768,37 +768,35 @@ case $SUBPROJECT in ;; esac -if [ -z "${PASSES}" ]; then - # we'll expand the base seed given here according to the STRUCTURE file, and - # then look in all of the seeds found to see which snaps are seeded - case $PROJECT:${SUBPROJECT:-} in - ubuntu:*|kubuntu*:*|lubuntu*:*|xubuntu*:*|ubuntu-mate*:*|ubuntustudio*:*|ubuntukylin*:*|ubuntu-budgie*:*) - BASE_SEED='desktop' - ;; - ubuntu-cpc:*) - BASE_SEED='server' - ;; - ubuntu-server:live) - BASE_SEED='server' - ;; - esac +# we'll expand the base seed given here according to the STRUCTURE file, and +# then look in all of the seeds found to see which snaps are seeded +case $PROJECT:${SUBPROJECT:-} in + ubuntu:*|kubuntu*:*|lubuntu*:*|xubuntu*:*|ubuntu-mate*:*|ubuntustudio*:*|ubuntukylin*:*|ubuntu-budgie*:*) + BASE_SEED='desktop' + ;; + ubuntu-cpc:*) + BASE_SEED='server' + ;; + ubuntu-server:live) + BASE_SEED='server' + ;; +esac - if [ -n "${BASE_SEED}" ]; then - snap_from_seed "${BASE_SEED}" config/seeded-snaps - fi +if [ -z "$PASSES" ] && [ -n "${BASE_SEED}" ]; then + snap_from_seed "${BASE_SEED}" config/seeded-snaps +fi - # grab a list of packags to remove for a "minimal" installation from the seed - # mirror for this project - if [ -n "${BASE_SEED}" ] && [ -n "${BASE_SEED}" ]; then - minimal_packages_url=${SEEDMIRROR}/${SEED}/${BASE_SEED}.minimal-remove - echo -n "Checking ${minimal_packages_url} for a minimal installation list... " - minimal_packages_remove=$(wget -q -O- ${minimal_packages_url} | sed -e '/\s*#.*$/d' -e '/^\s*$/d') - if [ -n "${minimal_packages_remove}" ]; then - echo "${minimal_packages_remove}" > config/manifest-minimal-remove - echo "$(echo ${minimal_packages_remove} | tr '\n' ' ')" - else - echo "failed to retrieve, not including." - fi +# grab a list of packags to remove for a "minimal" installation from the seed +# mirror for this project +if [ -n "${BASE_SEED}" ] && [ -n "${BASE_SEED}" ]; then + minimal_packages_url=${SEEDMIRROR}/${SEED}/${BASE_SEED}.minimal-remove + echo -n "Checking ${minimal_packages_url} for a minimal installation list... " + minimal_packages_remove=$(wget -q -O- ${minimal_packages_url} | sed -e '/\s*#.*$/d' -e '/^\s*$/d') + if [ -n "${minimal_packages_remove}" ]; then + echo "${minimal_packages_remove}" > config/manifest-minimal-remove + echo "$(echo ${minimal_packages_remove} | tr '\n' ' ')" + else + echo "failed to retrieve, not including." fi fi @@ -922,7 +920,6 @@ lb config noauto \ echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/chroot -echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common if [ -n "$PASSES" ]; then echo "PASSES=\"$PASSES\"" >> config/common fi diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index 848cc9ab..0411f319 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -60,7 +60,7 @@ build_layered_squashfs() { mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ else # first pass - rmdir chroot||true + rmdir chroot 2>/dev/null||true ln -s "chroot.${pass}/" chroot fi @@ -122,4 +122,11 @@ umount chroot/ PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" cp "livecd.${PROJECT}.${PASSPREFIX}${_PASS}.size" "$PREFIX.size" cp "livecd.${PROJECT}.${PASSPREFIX}${_PASS}.manifest.full" "$PREFIX.manifest" -chmod 644 *.squashfs *.manifest *.manifest.full *.size + +# Ubiquity-compatible removal manifest for ISO not using a layered-aware installer +if [ -n "$(ls livecd.${PROJECT}.[0-9][0-9]-live.manifest.full 2>/dev/null)" ] && \ + [ -n "$(ls livecd.${PROJECT}.[0-9][0-9]-install.manifest.full 2>/dev/null)" ]; then + echo "$(diff livecd.${PROJECT}.[0-9][0-9]-live.manifest.full livecd.${PROJECT}.[0-9][0-9]-install.manifest.full | awk '/^< / { print $2 }')" > livecd.${PROJECT}-manifest-remove +fi + +chmod 644 *.squashfs *.manifest* *.size From e0ed85c4009041c52d9bbd3cffcfda577f43c638 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Fri, 23 Nov 2018 09:57:30 +0100 Subject: [PATCH 28/44] Fixed pattern for matching manifest --- live-build/lb_binary_layered | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index 0411f319..e6b762ab 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -69,7 +69,7 @@ build_layered_squashfs() { create_manifest "chroot" "${squashfs_f_manifest}.full" # Delta manifest - diff -NU0 ${PWD}/livecd.${PROJECT}.[0-9]{2..2}-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.[0-9][0-9]-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest squashfs_f_size="${base}.size" du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}" From 41becb0af74bcb2c2a1dca5fc2e62f019fd46a72 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Fri, 23 Nov 2018 15:34:51 +0100 Subject: [PATCH 29/44] Ensure snaps are available on ubuntu-server live --- live-build/auto/build | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/live-build/auto/build b/live-build/auto/build index edd9e98c..79d3d7a4 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -371,6 +371,11 @@ deb file:/var/lib/preinstalled-pool/ $LB_DISTRIBUTION $LB_PARENT_ARCHIVE_AREAS rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig fi case $PROJECT:$SUBPROJECT in + ubuntu-server:live) + lb chroot_resolv install + snap_prepare chroot + lb chroot_resolv remove + ;; *) if [ -e "config/seeded-snaps" ]; then snap_list=$(cat config/seeded-snaps) From 1bfac23725a10e3c8c4bbdcce6f107e78c67db2f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Mon, 26 Nov 2018 10:01:25 +0100 Subject: [PATCH 30/44] Add ubiquity-ng image autopkgtest --- debian/tests/default-bootstraps | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps index 0c0062a6..3399932c 100755 --- a/debian/tests/default-bootstraps +++ b/debian/tests/default-bootstraps @@ -23,6 +23,7 @@ ALL_TRIPLETS=" lubuntu:: mythbuntu:: ubuntu:: + ubuntu:ubiquity-ng: ubuntu-base:: ubuntu-budgie:: ubuntu-budgie-desktop:: From 15cacaee96bf9e5c51579cfabea1f13dfd7edc69 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 28 Nov 2018 09:44:52 +0100 Subject: [PATCH 31/44] configure network manager _after_ installing network-manager --- live-build/lb_chroot_layered | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 8ec915b8..722fa619 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -145,7 +145,6 @@ create_chroot_pass() { if [ "${passtype}" = "first" ]; then configure_universe - configure_network_manager fi # Customizing chroot @@ -163,6 +162,7 @@ create_chroot_pass() { # Kernel should be in first layer if [ "${passtype}" = "first" ]; then + configure_network_manager Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ | awk '/^i/ {print $2}' > chroot.headers for i in $(cat chroot.headers); do From e0d30dd2d36591de486d134e2eb07578d334bd32 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 17 Jan 2019 16:26:39 +0100 Subject: [PATCH 32/44] Fixed substract -> subtract --- live-build/auto/config | 4 ++-- live-build/functions | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index d6d62cbb..3d155228 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -141,13 +141,13 @@ add_layered_pass_delta() { local seed_regexp="$3" if [ -z "${seed_regexp}" ]; then - substract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-${2} + subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-${2} return fi for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do pass=${1}_${seed} - substract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + subtract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass done } diff --git a/live-build/functions b/live-build/functions index 33ef79e2..1127897e 100644 --- a/live-build/functions +++ b/live-build/functions @@ -592,11 +592,11 @@ list_packages_from_seed () { done|sort -u } -substract_package_lists() { - # Substract a package list from another +subtract_package_lists() { + # Subtract a package list from another # # $1 source package list - # $2 Package list to substract from source package list + # $2 Package list to subtract from source package list local list1=$(mktemp) local list2=$(mktemp) From 2d2ce6eea41fe43c3be09fdcd2ab775c36456867 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 17 Jan 2019 16:40:11 +0100 Subject: [PATCH 33/44] Rename chroot.$PASS -> overlay.$PASS --- live-build/auto/clean | 1 + live-build/lb_binary_layered | 16 ++++++++-------- live-build/lb_chroot_layered | 12 ++++++------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/live-build/auto/clean b/live-build/auto/clean index 82a7cce4..ab4ea402 100755 --- a/live-build/auto/clean +++ b/live-build/auto/clean @@ -9,4 +9,5 @@ rm -f binary.manifest binary.manifest-desktop binary.log rm -f livecd.* rm -rf userdata rm -rf chroot.* +rm -rf overlay.* rm -rf *.manifest.full diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index e6b762ab..b100339a 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -57,11 +57,11 @@ build_layered_squashfs() { fi if [ -n "${lowerlayers}" ]; then - mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ + mount_overlay ${lowerlayers} "overlay.${pass}/" chroot/ else # first pass rmdir chroot 2>/dev/null||true - ln -s "chroot.${pass}/" chroot + ln -s "overlay.${pass}/" chroot fi # Full manifest until that PASS @@ -72,9 +72,9 @@ build_layered_squashfs() { diff -NU0 ${PWD}/livecd.${PROJECT}.[0-9][0-9]-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest squashfs_f_size="${base}.size" - du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}" + du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" - (cd "chroot.${pass}/" && + (cd "overlay.${pass}/" && mksquashfs . ${squashfs_f} \ -no-progress -xattrs -comp xz ) @@ -87,8 +87,8 @@ build_layered_squashfs() { # Handle direct sublayer of current one # Extract the name of the pass corresponding to the sublayer - for subpass in $(ls -d chroot.${pass}_* 2>/dev/null | sed -e "s/chroot\.\(${pass}_[^_]\+\).*/\1/"); do - lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" + for subpass in $(ls -d overlay.${pass}_* 2>/dev/null | sed -e "s/overlay\.\(${pass}_[^_]\+\).*/\1/"); do + lowerlayers_for_subpass="overlay.${pass}:${lowerlayers}" lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" build_layered_squashfs "${subpass}" "${pass}" "${prefix}" "${lowerlayers_for_subpass}" ${*} done @@ -104,14 +104,14 @@ do build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PASSPREFIX" "${LOWER_LAYERS}" ${*} - LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" + LOWER_LAYERS="overlay.${_PASS}:$LOWER_LAYERS" LOWER_LAYERS="${LOWER_LAYERS%:}" PREVPASS=${_PASS} CURPASS=$(( CURPASS + 1 )) done # remount last "main" pass on chroot for lb binary -mount_overlay "${LOWER_LAYERS}" "chroot.${_PASS}/" chroot/ +mount_overlay "${LOWER_LAYERS}" "overlay.${_PASS}/" chroot/ # Prepare initrd + kernel lb binary_linux-image ${*} diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 722fa619..89b84724 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -114,15 +114,15 @@ create_chroot_pass() { shift 4 # restore ${*} # We have already treated that pass - if [ -d "chroot.${pass}/" ]; then + if [ -d "overlay.${pass}/" ]; then return fi export PASS=${pass} if [ "${passtype}" != "first" ]; then - mkdir chroot.${pass} - mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/ + mkdir overlay.${pass} + mount_overlay ${lowerlayers} "overlay.${pass}/" chroot/ fi # Configuring chroot @@ -205,7 +205,7 @@ create_chroot_pass() { lb chroot_cache save ${*} if [ "${passtype}" = "first" ]; then - mv chroot chroot.${pass} + mv chroot overlay.${pass} mkdir chroot else umount chroot @@ -214,7 +214,7 @@ create_chroot_pass() { # Handle direct sublayer of current one # Extract the name of the pass corresponding to the sublayer for subpass in $(ls config/package-lists/*list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do - lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}" + lowerlayers_for_subpass="overlay.${pass}:${lowerlayers}" lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" create_chroot_pass "${subpass}" "${pass}" "${lowerlayers_for_subpass}" "" ${*} done @@ -236,7 +236,7 @@ do create_chroot_pass "$_PASS" "$PREVPASS" "$LOWER_LAYERS" "$PASSTYPE" ${*} - LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS" + LOWER_LAYERS="overlay.${_PASS}:$LOWER_LAYERS" LOWER_LAYERS="${LOWER_LAYERS%:}" PREVPASS=${_PASS} From addef92e23db1c9798fb151ad28ee3e836c01bef Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 17 Jan 2019 16:43:10 +0100 Subject: [PATCH 34/44] Decouple from ubuntu-cdimage & debian-cd landings We don't want to regress the ubuntu desktop traditional image when livecd-rootfs lands uncordinated with ubuntu-cdimage & debian-cd. Decoupling enables us to: - land livecd-rootfs (no change for ubuntu-desktop project, ubuntu-desktop:ubiquity-ng can be tested once the 2 others branches lands) - land ubuntu-cdimage & debian-cd for publishing any arbitrary squashfs names - finally, revert that commit and land it in livecd-rootfs to transform the traditional ubuntu-desktop image to a layered approach. --- live-build/auto/config | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 3d155228..7f63163b 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -390,19 +390,23 @@ esac case $PROJECT in ubuntu|ubuntu-dvd) - PASSES="install-minimal install live" - add_task install-minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages - add_task install ubuntu-desktop ubuntu-desktop-default-languages LIVE_TASK='ubuntu-live' case ${SUBPROJECT:-} in ubiquity-ng) + PASSES="install-minimal install live" + add_task install-minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages + add_task install ubuntu-desktop ubuntu-desktop-default-languages + # LANG PASS for minimal and install add_layered_pass_delta install-minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' add_layered_pass_delta install-minimal desktop-minimal-default-languages '' # none (if no default langpack is selected) add_layered_pass_delta install desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' add_layered_pass_delta install desktop-default-languages '' # none (if no default langpack is selected) ;; + *) + add_task install minimal standard ubuntu-desktop ubuntu-desktop-minimal-default-languages ubuntu-desktop-default-languages + ;; esac case $ARCH in From 283871573db33af8bb1a77d452d17ac6e18ef9e2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 24 Jan 2019 10:46:56 +0100 Subject: [PATCH 35/44] Changed the naming scheme for multi layer images to dot format --- live-build/auto/build | 21 +---- live-build/auto/config | 159 +++++++++++++++++++++++++++++------ live-build/functions | 68 ++++++++++++++- live-build/lb_binary_layered | 106 +++++++++++------------ live-build/lb_chroot_layered | 134 +++++++++++++++-------------- 5 files changed, 323 insertions(+), 165 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index 79d3d7a4..e9f20961 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -235,28 +235,11 @@ echo "command. You will still need to ensure the 'man-db' package is installed." EOF chmod +x chroot/usr/bin/man fi - Chroot chroot "dpkg-divert --quiet --add \ - --divert /usr/sbin/update-initramfs.REAL --rename \ - /usr/sbin/update-initramfs" - cat > chroot/usr/sbin/update-initramfs <<'EOF' -#! /bin/sh -if [ $# != 1 ] || [ "$1" != -u ]; then - exec update-initramfs.REAL "$@" -fi -echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2 -exit 0 -EOF - chmod +x chroot/usr/sbin/update-initramfs - - cat > config/hooks/999-undivert-update-initramfs.chroot <<'EOF' -#! /bin/sh -rm -f /usr/sbin/update-initramfs -dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs -EOF if [ -n "${PASSES}" ]; then PATH="config/:$PATH" lb chroot_layered "$@" else + divert_update_initramfs lb chroot "$@" fi @@ -453,7 +436,7 @@ for OUTPUT in ext2 ext3 ext4 manifest manifest-remove size squashfs; do done # we don't need a manifest-remove for a layered-aware installer -if [ "$SUBPROJECT" = "ubiquity-ng" ]; then +if [ "$PROJECT" = "ubuntu" ] && [ "$SUBPROJECT" = "canary" ]; then rm -f livecd.${PROJECT}-manifest-remove rm -f config/manifest-minimal-remove fi diff --git a/live-build/auto/config b/live-build/auto/config index 7f63163b..21d173ac 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -59,6 +59,40 @@ BINARY_HOOKS= APT_OPTIONS=" --yes -oDebug::pkgDepCache::AutoInstall=yes " +PASSES_TO_LAYERS=false +_PASSES_TO_LAYERS= # Stores the initial value of PASSES_TO_LAYERS +PASSES= + +_check_immutable_passes_to_layers () { + # Check that PASSES_TO_LAYERS didn't change ie we didn't switch from layer + # to non-layer image format and vice versa. + if [ -z "$_PASSES_TO_LAYERS" ]; then + _PASSES_TO_LAYERS=$PASSES_TO_LAYERS + fi + + if [ "$_PASSES_TO_LAYERS" != "$PASSES_TO_LAYERS" ]; then + echo "ERROR: PASSES_TO_LAYERS has changed from $_PASSES_TO_LAYERS to $PASSES_TO_LAYERS between 2 API calls. Please set it first." + exit 1 + fi +} + +_check_layers_only_API () { + # Check if a function is designed only for layered mode + # $1 name of the function + if [ "$PASSES_TO_LAYERS" != "true" ]; then + echo "$1 should only be used in layered mode, with PASSES_TO_LAYERS set to true" + exit 1 + fi +} + +_register_pass () { + # In layer mode, record a pass into the list of passes + # $1 Name of the pass + [ "$PASSES_TO_LAYERS" != "true" ] && return + + PASSES="$PASSES $1" +} + add_task () { local pass="$1" @@ -68,6 +102,9 @@ add_task () local snap_list_files local curseed + _check_immutable_passes_to_layers + _register_pass "$pass" + # The removal of direct task installation support from live-build # poses some problems. If the chroot has multiarch configured - for # example, if we're building for amd64 - then dumpavail will show @@ -113,40 +150,80 @@ add_task () add_package () { + # Adds a pass named pass_name composed of packages to install + # $1 pass + # $@ list of packages + local pass="$1" shift local pkg + _check_immutable_passes_to_layers + _register_pass "$pass" + for pkg; do echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" done } -add_layered_pass() { - # Add a layer to an existing pass based on seeds matching a regexp +remove_package () +{ + # Adds a pass named pass_name composed of packages to remove + # $1 pass + # $@ list of packages + + local pass="$1" + shift + local pkg + + _check_immutable_passes_to_layers + _check_layers_only_API "remove_package" + _register_pass "$pass" + + for pkg; do + echo "$pkg" >> "config/package-lists/livecd-rootfs.removal-list.chroot_$pass" + done +} + +add_packages_from_seed_regexp () { + # Creates one or more passes, depending on base_pass_name, from any seeds matching seed_regexp. # $1 base pass # $2 seeds (regexp) + local pass + + _check_immutable_passes_to_layers + _check_layers_only_API "add_packages_from_seed_regexp" for seed in $(ls config/germinate-output/|grep -P "$2"); do - pass=${1}_${seed} + pass=${1}.${seed} + _register_pass "$pass" list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass done } -add_layered_pass_delta() { - # Add a layer to an existing pass based on delta between seeds matching a regexp and a base seed +remove_packages_from_seed_regexp() { + # Creates one or more passes, based on base_pass_name, from any seed matching seed_regexp. + # This package list is a list of packages to remove (and included in a previous dependent + # pass then), resulting from base_seed - {current_seed_match_from_regexp}. # $1 base pass # $2 base seed # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. + local pass + + _check_immutable_passes_to_layers + _check_layers_only_API "remove_packages_from_seed_regexp" local seed_regexp="$3" if [ -z "${seed_regexp}" ]; then - subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_${1}_no-${2} + pass="${1}.no-${2}" + _register_pass "$pass" + subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass return fi for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do - pass=${1}_${seed} + pass="${1}.${seed}" + _register_pass "$pass" subtract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass done } @@ -161,6 +238,28 @@ add_binary_hook () BINARY_HOOKS="${BINARY_HOOKS:+$BINARY_HOOKS }$1" } +_sanitize_passes () +{ + # Returns an uniquely ordered list of passes and ensure dependency tree is coherent + # $1 list of passes + local passes="$1" + [ -z "$passes" ] && return + + passes=$(echo $passes | tr ' ' '\n' | sort -u) + for pass in $passes; do + parent=$(get_parent_pass $pass) + # if root pass, no parent to find + [ -z "$parent" ] && continue + if [ $(echo "$passes"|grep -cE "^$parent\$") -ne 1 ]; then + echo "ERROR: '$parent' is required by '$pass' but is missing. Registered passes are:\n$passes" + exit 1 + fi + done + + # return the list of passes + echo $passes +} + if [ -z "${IMAGEFORMAT:-}" ]; then case $PROJECT:${SUBPROJECT:-} in ubuntu-cpc:*) @@ -289,6 +388,11 @@ case $IMAGEFORMAT in case $PROJECT in ubuntu-server|ubuntu-touch|ubuntu-touch-custom) ;; + ubuntu) + if [ $SUBPROJECT != "canary" ]; then + add_package live lupin-casper + fi + ;; *) add_package live lupin-casper ;; @@ -390,28 +494,33 @@ esac case $PROJECT in ubuntu|ubuntu-dvd) - LIVE_TASK='ubuntu-live' case ${SUBPROJECT:-} in - ubiquity-ng) - PASSES="install-minimal install live" - add_task install-minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages - add_task install ubuntu-desktop ubuntu-desktop-default-languages - - # LANG PASS for minimal and install - add_layered_pass_delta install-minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' - add_layered_pass_delta install-minimal desktop-minimal-default-languages '' # none (if no default langpack is selected) - add_layered_pass_delta install desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' - add_layered_pass_delta install desktop-default-languages '' # none (if no default langpack is selected) + canary) + PASSES_TO_LAYERS="true" + add_task minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages + add_task minimal.standard ubuntu-desktop ubuntu-desktop-default-languages + add_task minimal.standard.live ubuntu-live + add_package minimal.standard.live lupin-casper + + case $ARCH in + amd64) add_package minimal.standard.live $SIGNED_KERNEL_PACKAGE ;; + esac + + # LANG PASS for minimal and standard + remove_packages_from_seed_regexp minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' + remove_packages_from_seed_regexp minimal desktop-minimal-default-languages '' # none (if no default langpack is selected) + remove_packages_from_seed_regexp minimal.standard desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' + remove_packages_from_seed_regexp minimal.standard desktop-default-languages '' # none (if no default langpack is selected) ;; *) + LIVE_TASK='ubuntu-live' add_task install minimal standard ubuntu-desktop ubuntu-desktop-minimal-default-languages ubuntu-desktop-default-languages + case $ARCH in + amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; + esac ;; esac - - case $ARCH in - amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; - esac ;; ubuntu-desktop-next) @@ -786,7 +895,7 @@ case $PROJECT:${SUBPROJECT:-} in ;; esac -if [ -z "$PASSES" ] && [ -n "${BASE_SEED}" ]; then +if [ "$PASSES_TO_LAYERS" != "true" ] && [ -n "${BASE_SEED}" ]; then snap_from_seed "${BASE_SEED}" config/seeded-snaps fi @@ -806,7 +915,7 @@ fi export APT_OPTIONS -if [ "$PREINSTALLED" != "true" ] && [ "$LIVE_TASK" ]; then +if [ "$PREINSTALLED" != "true" ] && [ "$PASSES_TO_LAYERS" != "true" ] && [ "$LIVE_TASK" ]; then add_task live "$LIVE_TASK" fi @@ -921,6 +1030,8 @@ lb config noauto \ $OPTS \ "$@" +PASSES=$(_sanitize_passes "$PASSES") + echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/chroot diff --git a/live-build/functions b/live-build/functions index 1127897e..64984e35 100644 --- a/live-build/functions +++ b/live-build/functions @@ -137,6 +137,22 @@ mount_overlay() { "$path" } +get_lowerdirs_for_pass () { + # Returns the name of the lowerdir from the name of a pass + # $1 Name of the pass + local curpass="$1" + local lowerlayers="" + + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + lowerlayers="${lowerlayers}:overlay.${curpass}" + done + echo "${lowerlayers#:}" +} + mount_disk_image() { local disk_image=${1} local mountpoint=${2} @@ -680,7 +696,7 @@ configure_network_manager() { # default. Installing NM on an existing system only manages wifi and wwan via # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting # the global backend to NM, netplan overrides that file. - if [ -e chroot/usr/sbin/NetworkManager ]; then + if [ -e chroot/usr/sbin/NetworkManager -a ! -f chroot/etc/netplan/01-network-manager-all.yaml ]; then echo "===== Enabling all devices in NetworkManager ====" mkdir -p chroot/etc/netplan cat < chroot/etc/netplan/01-network-manager-all.yaml @@ -693,3 +709,53 @@ EOF echo "==== NetworkManager not installed ====" fi } + +get_parent_pass () { + # return parent pass + # $1 name of the pass + # return parent pass name or '' if pass is root pass. + local pass="$1" + + parent_pass=${pass%.*} + if [ "${parent_pass}" = "${pass}" ]; then + return + fi + echo ${pass%.*} +} + +setenv_file () { + # Exposes an environment variable in a chroot + # $1 Name of the variable + # $2 Value of the variable + # $3 Path to the environment file of the chroot + local var="$1" + local val="$2" + local file="$3" + + grep -v "^$var" $file || true > $file.new + echo "${var}=${val}" >> $file.new + mv $file.new $file +} + +divert_update_initramfs () { + Chroot chroot "dpkg-divert --quiet --add \ + --divert /usr/sbin/update-initramfs.REAL --rename \ + /usr/sbin/update-initramfs" + cat > chroot/usr/sbin/update-initramfs <<'EOF' +#! /bin/sh +if [ $# != 1 ] || [ "$1" != -u ]; then + exec update-initramfs.REAL "$@" +fi +echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2 + +exit 0 +EOF + chmod +x chroot/usr/sbin/update-initramfs + + cat > config/hooks/999-undivert-update-initramfs.chroot <<'EOF' +#! /bin/sh +[ ! -f /usr/sbin/update-initramfs.REAL ] && exit 0 +rm -f /usr/sbin/update-initramfs +dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs +EOF +} diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index b100339a..3b3b0b7c 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -37,18 +37,28 @@ Setup_cleanup . config/functions -build_layered_squashfs() { - local pass=$1 # install|install_subpass|install_subpass_subsubpass|… - local prevpass=$2 # install|install_subpass|… - local prefix=$3 # 01-|02-|… - local lowerlayers=$4 - shift 4 # restore ${*} +lb_binary_includes () { + # Copying includes from pass subdirectory + local pass="$1" - # Cleanup root filesystem - lb binary_chroot ${*} + if [ ! -d config/includes.binary.${pass} ]; then + return + fi + + cd config/includes.binary.${pass} + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot + cd "${OLDPWD}" +} + +build_layered_squashfs () { + local pass=$1 + shift 1 # restore ${*} + + Echo_message "lb_binary_layered: treating pass $pass" # Building squashfs filesystem & manifest - base="${PWD}/livecd.${PROJECT}.${prefix}${pass}" + local overlay_dir="overlay.${pass}" + base="${PWD}/livecd.${PROJECT}.${pass}" squashfs_f="${base}.squashfs" # We have already treated that pass @@ -56,77 +66,67 @@ build_layered_squashfs() { return fi - if [ -n "${lowerlayers}" ]; then - mount_overlay ${lowerlayers} "overlay.${pass}/" chroot/ + rm -f .build/binary_chroot + + mkdir -p "$overlay_dir/" + lowerdirs=$(get_lowerdirs_for_pass $pass) + if [ -n "$lowerdirs" ]; then + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ else - # first pass - rmdir chroot 2>/dev/null||true - ln -s "overlay.${pass}/" chroot + ln -s "$overlay_dir/" chroot fi + export PASS=${pass} + setenv_file PASS "${pass}" config/environment.chroot + + # Cleanup root filesystem + lb binary_chroot ${*} + + lb_binary_includes $pass ${*} + lb binary_hooks ${*} + # Full manifest until that PASS squashfs_f_manifest="${base}.manifest" create_manifest "chroot" "${squashfs_f_manifest}.full" # Delta manifest - diff -NU0 ${PWD}/livecd.${PROJECT}.[0-9][0-9]-${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest squashfs_f_size="${base}.size" du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" + # We take first live path for "global" ISO properties (used by installers and checkers): + # Prepare initrd + kernel + # Main manifest and size files + prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" + if echo $pass | grep -Eq '\.live*' && test ! -e "${prefix}.manifest"; then + lb binary_linux-image ${*} + cp ${squashfs_f_size} "${prefix}.size" + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + fi + (cd "overlay.${pass}/" && mksquashfs . ${squashfs_f} \ -no-progress -xattrs -comp xz ) - if [ -n "${lowerlayers}" ]; then + if [ -n "$lowerdirs" ]; then umount chroot + rmdir chroot else rm chroot - mkdir chroot/ fi - - # Handle direct sublayer of current one - # Extract the name of the pass corresponding to the sublayer - for subpass in $(ls -d overlay.${pass}_* 2>/dev/null | sed -e "s/overlay\.\(${pass}_[^_]\+\).*/\1/"); do - lowerlayers_for_subpass="overlay.${pass}:${lowerlayers}" - lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" - build_layered_squashfs "${subpass}" "${pass}" "${prefix}" "${lowerlayers_for_subpass}" ${*} - done } -CURPASS=1 -PREVPASS="" -PASSPREFIX="" -LOWER_LAYERS="" for _PASS in $PASSES do - PASSPREFIX="$(printf "%02g" $CURPASS)-" - - build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PASSPREFIX" "${LOWER_LAYERS}" ${*} - - LOWER_LAYERS="overlay.${_PASS}:$LOWER_LAYERS" - LOWER_LAYERS="${LOWER_LAYERS%:}" - PREVPASS=${_PASS} - CURPASS=$(( CURPASS + 1 )) + build_layered_squashfs "${_PASS}" ${*} done -# remount last "main" pass on chroot for lb binary -mount_overlay "${LOWER_LAYERS}" "overlay.${_PASS}/" chroot/ - -# Prepare initrd + kernel -lb binary_linux-image ${*} - -umount chroot/ - -# Full ISO manifest & size from last main PASS -PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" -cp "livecd.${PROJECT}.${PASSPREFIX}${_PASS}.size" "$PREFIX.size" -cp "livecd.${PROJECT}.${PASSPREFIX}${_PASS}.manifest.full" "$PREFIX.manifest" - # Ubiquity-compatible removal manifest for ISO not using a layered-aware installer -if [ -n "$(ls livecd.${PROJECT}.[0-9][0-9]-live.manifest.full 2>/dev/null)" ] && \ - [ -n "$(ls livecd.${PROJECT}.[0-9][0-9]-install.manifest.full 2>/dev/null)" ]; then - echo "$(diff livecd.${PROJECT}.[0-9][0-9]-live.manifest.full livecd.${PROJECT}.[0-9][0-9]-install.manifest.full | awk '/^< / { print $2 }')" > livecd.${PROJECT}-manifest-remove +if [ -n "$(ls livecd.${PROJECT}.*install.live.manifest.full 2>/dev/null)" ] && \ + [ -n "$(ls livecd.${PROJECT}.*install.manifest.full 2>/dev/null)" ]; then + echo "$(diff livecd.${PROJECT}.*install.live.manifest.full livecd.${PROJECT}.*install.manifest.full | awk '/^< / { print $2 }')" > livecd.${PROJECT}-manifest-remove fi chmod 644 *.squashfs *.manifest* *.size diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 89b84724..c65a6af4 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -41,7 +41,7 @@ Setup_cleanup . config/functions -lb_chroot_remove_packages() { +lb_chroot_remove_packages () { # Remove packages from the chroot specific to this layer # # $1: Name of the pass* @@ -54,28 +54,28 @@ lb_chroot_remove_packages() { } # Create the snap list specific to this layer -lb_chroot_snap_lists() { +lb_chroot_snap_lists () { local pass=$1 - local prevpass=$2 # This assumes that the prefix is unique for a given project local snap_for_pass=$(ls config/package-lists/*.snaplist.chroot_${pass}.full 2>/dev/null || true) - local snap_for_prevpass=$(ls config/package-lists/*.snaplist.chroot_${prevpass}.full 2>/dev/null || true) + parent_pass=$(get_parent_pass $pass) + local snap_for_parent_pass=$(ls config/package-lists/*.snaplist.chroot_${parent_pass}.full 2>/dev/null || true) if [ -z "${snap_for_pass}" ]; then return fi - if [ -z "${snap_for_prevpass}" ]; then + if [ -z "${snap_for_parent_pass}" ]; then cp ${snap_for_pass} ${snap_for_pass%.full} return fi # Generate a list of snaps added to a layer. - diff -NU0 ${snap_for_prevpass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full} + diff -NU0 ${snap_for_parent_pass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full} } -lb_chroot_install_snaps() { +lb_chroot_install_snaps () { # Prepare the snap environment and install snaps into a chroot # # $1: Name of the pass @@ -93,7 +93,7 @@ lb_chroot_install_snaps() { done < $snaplist_file } -lb_chroot_includes() { +lb_chroot_includes () { # Copying includes from pass subdirectory local pass="$1" @@ -106,30 +106,47 @@ lb_chroot_includes() { cd "${OLDPWD}" } -create_chroot_pass() { +create_chroot_pass () { local pass=$1 - local prevpass=$2 - local lowerlayers=$3 - local passtype=$4 # "first"|"last"|"" empty string - shift 4 # restore ${*} + shift 1 # restore ${*} - # We have already treated that pass - if [ -d "overlay.${pass}/" ]; then + Echo_message "lb_chroot_layered: treating pass $pass" + + # We have already treated that pass just return. + local overlay_dir="overlay.${pass}" + if [ -d "$overlay_dir/" ]; then return fi - export PASS=${pass} - - if [ "${passtype}" != "first" ]; then - mkdir overlay.${pass} - mount_overlay ${lowerlayers} "overlay.${pass}/" chroot/ + # Only get some function executed on root passes + # Copy bootstrap on root layers + if [ -z "$(get_parent_pass $pass)" ]; then + rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks + cp -a chroot.bootstrap/ "$overlay_dir/" + fi + # Others have to be executed on every pass + rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive + + mkdir -p "$overlay_dir/" + lowerdirs=$(get_lowerdirs_for_pass $pass) + if [ -n "$lowerdirs" ]; then + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ + else + ln -s "$overlay_dir/" chroot fi + export PASS=${pass} + setenv_file PASS "${pass}" config/environment.chroot + # Configuring chroot - lb chroot_cache restore ${*} lb chroot_devpts install ${*} lb chroot_proc install ${*} lb chroot_sysfs install ${*} + # We run chroot_hacks only on root layers (update-initramfs diverted) + if [ -z "$(get_parent_pass $pass)" ]; then + divert_update_initramfs + fi lb chroot_debianchroot install ${*} lb chroot_dpkg install ${*} lb chroot_tmpfs install ${*} @@ -140,10 +157,11 @@ create_chroot_pass() { # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more # diff content than desired. However, we still need to setup the archive and teardown # for each layer. - # We could modify livebuild if necessary to have conditional upgrade (first pass only). + # We could modify livebuild if necessary to have conditional upgrade (root pass only). lb chroot_archives chroot install ${*} - if [ "${passtype}" = "first" ]; then + # Only configure universe on root passes + if [ -z "$(get_parent_pass $pass)" ]; then configure_universe fi @@ -157,23 +175,22 @@ create_chroot_pass() { lb_chroot_remove_packages ${pass} ${*} # Snap management - lb_chroot_snap_lists ${pass} ${prevpass} + lb_chroot_snap_lists ${pass} ${*} lb_chroot_install_snaps ${pass} ${*} - # Kernel should be in first layer - if [ "${passtype}" = "first" ]; then - configure_network_manager - Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ + configure_network_manager + + # Mark kernel headers as autoremovable + Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ | awk '/^i/ {print $2}' > chroot.headers - for i in $(cat chroot.headers); do - Chroot chroot "apt-mark auto $i" - done - fi + for i in $(cat chroot.headers); do + Chroot chroot "apt-mark auto $i" + done Chroot chroot "apt-get --purge -y autoremove" - # Add live packages to top layer - if [ "${passtype}" = "last" ]; then + # Add live packages to live layers + if echo $pass | grep -Eq '\.live*'; then lb chroot_live-packages ${*} fi @@ -181,7 +198,14 @@ create_chroot_pass() { lb_chroot_includes ${pass} ${*} lb chroot_hooks ${*} + + # Run chroot_hacks only on root layers. + # chroot_hacks changes the mode of boot/initrd*. The side effect in + # layered mode is to create an initrd on each layer with a significant + # impact on image size (+30MB per layer). This is an issue even with + # update-initramfs disabled. lb chroot_hacks ${*} + lb chroot_interactive ${*} # Misc ubuntu cleanup and post-layer configuration @@ -202,45 +226,19 @@ create_chroot_pass() { lb chroot_sysfs remove ${*} lb chroot_proc remove ${*} lb chroot_devpts remove ${*} - lb chroot_cache save ${*} - if [ "${passtype}" = "first" ]; then - mv chroot overlay.${pass} - mkdir chroot - else + if [ -n "$lowerdirs" ]; then umount chroot + rmdir chroot + else + rm chroot fi - - # Handle direct sublayer of current one - # Extract the name of the pass corresponding to the sublayer - for subpass in $(ls config/package-lists/*list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do - lowerlayers_for_subpass="overlay.${pass}:${lowerlayers}" - lowerlayers_for_subpass="${lowerlayers_for_subpass%:}" - create_chroot_pass "${subpass}" "${pass}" "${lowerlayers_for_subpass}" "" ${*} - done } -PASSES="${PASSES:-install live}" -CURPASS=1 -PREVPASS="" -LASTPASS=$(echo $PASSES|wc -w) -LOWER_LAYERS="" +if [ ! -d chroot.bootstrap/ ]; then + mv chroot/ chroot.bootstrap/ +fi for _PASS in $PASSES do - PASSTYPE="" - if [ $CURPASS -eq 1 ]; then - PASSTYPE="first" - elif [ $CURPASS -eq $LASTPASS ]; then - PASSTYPE="last" - fi - - create_chroot_pass "$_PASS" "$PREVPASS" "$LOWER_LAYERS" "$PASSTYPE" ${*} - - LOWER_LAYERS="overlay.${_PASS}:$LOWER_LAYERS" - LOWER_LAYERS="${LOWER_LAYERS%:}" - PREVPASS=${_PASS} - - CURPASS=$(( CURPASS + 1 )) + create_chroot_pass "$_PASS" ${*} done - -rmdir chroot From 402cdd15bed5f2dfb6442a77d6aa943178d91d87 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 24 Jan 2019 16:13:41 +0100 Subject: [PATCH 36/44] Adds is_root_layer Factored the code to detect if a layer is a root layer in function is_root_layer and execute lb_chroot_archives only on root layers. --- live-build/functions | 8 ++++++++ live-build/lb_chroot_layered | 18 +++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/live-build/functions b/live-build/functions index 64984e35..30aaadff 100644 --- a/live-build/functions +++ b/live-build/functions @@ -759,3 +759,11 @@ rm -f /usr/sbin/update-initramfs dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs EOF } + +is_root_layer () { + local pass=$1 + if [ -z "$(get_parent_pass $pass)" ]; then + return 0 + fi + return 1 +} diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index c65a6af4..c81bec5f 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -120,7 +120,7 @@ create_chroot_pass () { # Only get some function executed on root passes # Copy bootstrap on root layers - if [ -z "$(get_parent_pass $pass)" ]; then + if $(is_root_layer $pass); then rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks cp -a chroot.bootstrap/ "$overlay_dir/" fi @@ -144,7 +144,7 @@ create_chroot_pass () { lb chroot_proc install ${*} lb chroot_sysfs install ${*} # We run chroot_hacks only on root layers (update-initramfs diverted) - if [ -z "$(get_parent_pass $pass)" ]; then + if $(is_root_layer $pass); then divert_update_initramfs fi lb chroot_debianchroot install ${*} @@ -155,13 +155,11 @@ create_chroot_pass () { lb chroot_hostname install ${*} lb chroot_apt install ${*} # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more - # diff content than desired. However, we still need to setup the archive and teardown - # for each layer. - # We could modify livebuild if necessary to have conditional upgrade (root pass only). - lb chroot_archives chroot install ${*} - + # diff content than desired. So only running this on root pass. + # Also configure universe # Only configure universe on root passes - if [ -z "$(get_parent_pass $pass)" ]; then + if $(is_root_layer $pass); then + lb chroot_archives chroot install ${*} configure_universe fi @@ -215,7 +213,9 @@ create_chroot_pass () { Chroot chroot "dpkg-query -W" > chroot.packages.${pass} # Deconfiguring chroot - lb chroot_archives chroot remove ${*} + if $(is_root_layer $pass); then + lb chroot_archives chroot remove ${*} + fi lb chroot_apt remove ${*} lb chroot_hostname remove ${*} lb chroot_resolv remove ${*} From d1a2e24c411ac888b07432f79def699269864597 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 24 Jan 2019 16:15:55 +0100 Subject: [PATCH 37/44] Adds reduce_pass_size Adds a function reduce_pass_size which removes duplicated files between layers. A duplicate file is a file with same path, size, mode, owners and content on 2 related layers. --- live-build/lb_chroot_layered | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index c81bec5f..65c81910 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -106,6 +106,38 @@ lb_chroot_includes () { cd "${OLDPWD}" } +reduce_pass_size () { + # Remove duplicated files between parent and currant pass + # Note the empty directories created in a child pass are not removed + local pass=$1 + local parent="$(get_parent_pass $pass)" + + $(is_root_layer $pass) && return + + pass_dir="overlay.${pass}" + parent_pass_dir="overlay.${parent}" + + local list_pass=$(mktemp) + local list_parent=$(mktemp) + + (cd $pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_pass) + (cd $parent_pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_parent) + + # Only iterate on common files with same type, owner, permission and size + comm -12 $list_pass $list_parent|cut -d'|' -f1|while read f; do + # If file contents are different, keep it + if ! diff --brief --no-dereference "$pass_dir/$f" "$parent_pass_dir/$f" >/dev/null; then + continue + fi + # Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta. + Echo_message "reduce_pass_size: '$f' is strictly identical between $parent and $pass. Removing." + rm "$pass_dir/$f" + done + + rm $list_pass + rm $list_parent +} + create_chroot_pass () { local pass=$1 shift 1 # restore ${*} @@ -233,6 +265,8 @@ create_chroot_pass () { else rm chroot fi + + reduce_pass_size $pass } if [ ! -d chroot.bootstrap/ ]; then From 52d9bd3eadfb6a6650664922861e2fd1f90489e7 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 24 Jan 2019 16:21:41 +0100 Subject: [PATCH 38/44] Run live specific scripts on non live layers Allow execution of lb_binary_linux-image and lb_chroot_live-packages on any layer by setting LIVE_PASSES. --- live-build/auto/config | 25 +++++++++++++++++++++++++ live-build/lb_binary_layered | 12 ++++++++---- live-build/lb_chroot_layered | 6 ++++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 21d173ac..5d464e26 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -260,6 +260,22 @@ _sanitize_passes () echo $passes } +_get_live_passes () +{ + # Returns a list of all passes that ends with .live for automated live passes detection + # $1 list of passes + local passes="$1" + local livepasses="" + [ -z "$passes" ] && return + + for pass in $passes; do + if echo $pass | grep -Eq '\.live*'; then + livepasses="$pass $livepasses" + fi + done + echo $livepasses +} + if [ -z "${IMAGEFORMAT:-}" ]; then case $PROJECT:${SUBPROJECT:-} in ubuntu-cpc:*) @@ -1031,6 +1047,12 @@ lb config noauto \ "$@" PASSES=$(_sanitize_passes "$PASSES") +LIVE_PASSES=${LIVE_PASSES:-$(_get_live_passes "$PASSES")} + +if [ -n "$PASSES" ] && [ -z "$LIVE_PASSES" ]; then + Echo_warning "Multi-layered mode is enabled, but we didn't find any live pass." \ + "Either set \$LIVE_PASSES or add a pass ending with '.live'." +fi echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot @@ -1038,6 +1060,9 @@ echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/chroot if [ -n "$PASSES" ]; then echo "PASSES=\"$PASSES\"" >> config/common fi +if [ -n "$LIVE_PASSES" ]; then + echo "LIVE_PASSES=\"$LIVE_PASSES\"" >> config/common +fi echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary echo "BUILDSTAMP=\"$NOW\"" >> config/binary echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index 3b3b0b7c..91c64046 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -100,10 +100,14 @@ build_layered_squashfs () { # Prepare initrd + kernel # Main manifest and size files prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" - if echo $pass | grep -Eq '\.live*' && test ! -e "${prefix}.manifest"; then - lb binary_linux-image ${*} - cp ${squashfs_f_size} "${prefix}.size" - cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + if [ ! -e "${prefix}.manifest" ]; then + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue + lb binary_linux-image ${*} + cp ${squashfs_f_size} "${prefix}.size" + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + break + done fi (cd "overlay.${pass}/" && diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 65c81910..b1bf31fe 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -220,9 +220,11 @@ create_chroot_pass () { Chroot chroot "apt-get --purge -y autoremove" # Add live packages to live layers - if echo $pass | grep -Eq '\.live*'; then + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue lb chroot_live-packages ${*} - fi + break + done # Run includes by pass lb_chroot_includes ${pass} ${*} From 3db920c40c7c933b72f92999033f6f2eefdf6128 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 24 Jan 2019 16:27:38 +0100 Subject: [PATCH 39/44] Fixed quoting of subproject variable --- live-build/auto/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/auto/config b/live-build/auto/config index 5d464e26..c391eb14 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -405,7 +405,7 @@ case $IMAGEFORMAT in ubuntu-server|ubuntu-touch|ubuntu-touch-custom) ;; ubuntu) - if [ $SUBPROJECT != "canary" ]; then + if [ "$SUBPROJECT" != "canary" ]; then add_package live lupin-casper fi ;; From f84d92d76f8713dff67a077e0a896ad5fa73995d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Wed, 30 Jan 2019 12:28:03 +0100 Subject: [PATCH 40/44] Calculate total size of the image + typos --- debian/tests/default-bootstraps | 2 +- live-build/auto/config | 44 ++++++++++++++++----------------- live-build/functions | 8 +++--- live-build/lb_binary_layered | 15 +++++++++-- live-build/lb_chroot_layered | 7 +++--- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps index 3399932c..ea042caa 100755 --- a/debian/tests/default-bootstraps +++ b/debian/tests/default-bootstraps @@ -23,7 +23,7 @@ ALL_TRIPLETS=" lubuntu:: mythbuntu:: ubuntu:: - ubuntu:ubiquity-ng: + ubuntu:canary: ubuntu-base:: ubuntu-budgie:: ubuntu-budgie-desktop:: diff --git a/live-build/auto/config b/live-build/auto/config index c391eb14..0f7d2b0b 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -186,46 +186,46 @@ remove_package () } add_packages_from_seed_regexp () { - # Creates one or more passes, depending on base_pass_name, from any seeds matching seed_regexp. - # $1 base pass - # $2 seeds (regexp) + # Creates one or more passes, depending on base_pass_name, from any seeds matching seed_regexp. + # $1 base pass + # $2 seeds (regexp) local pass _check_immutable_passes_to_layers _check_layers_only_API "add_packages_from_seed_regexp" - for seed in $(ls config/germinate-output/|grep -P "$2"); do - pass=${1}.${seed} - _register_pass "$pass" - list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass - done + for seed in $(ls config/germinate-output/|grep -P "$2"); do + pass=${1}.${seed} + _register_pass "$pass" + list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass + done } remove_packages_from_seed_regexp() { - # Creates one or more passes, based on base_pass_name, from any seed matching seed_regexp. + # Creates one or more passes, based on base_pass_name, from any seed matching seed_regexp. # This package list is a list of packages to remove (and included in a previous dependent # pass then), resulting from base_seed - {current_seed_match_from_regexp}. - # $1 base pass - # $2 base seed - # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. + # $1 base pass + # $2 base seed + # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. local pass _check_immutable_passes_to_layers _check_layers_only_API "remove_packages_from_seed_regexp" - local seed_regexp="$3" - if [ -z "${seed_regexp}" ]; then + local seed_regexp="$3" + if [ -z "${seed_regexp}" ]; then pass="${1}.no-${2}" _register_pass "$pass" - subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass - return - fi + subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + return + fi - for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do - pass="${1}.${seed}" + for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do + pass="${1}.${seed}" _register_pass "$pass" - subtract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass - done + subtract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + done } add_chroot_hook () @@ -269,7 +269,7 @@ _get_live_passes () [ -z "$passes" ] && return for pass in $passes; do - if echo $pass | grep -Eq '\.live*'; then + if echo $pass | grep -Eq '\.live$'; then livepasses="$pass $livepasses" fi done diff --git a/live-build/functions b/live-build/functions index 30aaadff..4bf85af5 100644 --- a/live-build/functions +++ b/live-build/functions @@ -724,10 +724,10 @@ get_parent_pass () { } setenv_file () { - # Exposes an environment variable in a chroot - # $1 Name of the variable - # $2 Value of the variable - # $3 Path to the environment file of the chroot + # Exposes an environment variable in a chroot + # $1 Name of the variable + # $2 Value of the variable + # $3 Path to the environment file of the chroot local var="$1" local val="$2" local file="$3" diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index 91c64046..cf1837ed 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -96,7 +96,7 @@ build_layered_squashfs () { squashfs_f_size="${base}.size" du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" - # We take first live path for "global" ISO properties (used by installers and checkers): + # We take first live pass for "global" ISO properties (used by installers and checkers): # Prepare initrd + kernel # Main manifest and size files prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" @@ -104,7 +104,18 @@ build_layered_squashfs () { for livepass in $LIVE_PASSES; do [ "$livepass" != "$pass" ] && continue lb binary_linux-image ${*} - cp ${squashfs_f_size} "${prefix}.size" + + totalsize=$(cat ${squashfs_f_size}) + curpass="$pass" + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) + done + echo ${totalsize} > "${prefix}.size" + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" break done diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index b1bf31fe..7f436528 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -8,7 +8,7 @@ ## under certain conditions; see COPYING for details. ## This is a fork of lb_chroot for layered live system. -## We don't want leaking host configuratino in each layer, and so, +## We don't want leaking host configuration in each layer, and so, ## we clean and setup the chroot each time. ## In addition, we create the squashfs for each layer, but top one (live) ## which still can be configured after lb chroot call. @@ -107,7 +107,7 @@ lb_chroot_includes () { } reduce_pass_size () { - # Remove duplicated files between parent and currant pass + # Remove duplicated files between parent and current pass # Note the empty directories created in a child pass are not removed local pass=$1 local parent="$(get_parent_pass $pass)" @@ -118,7 +118,7 @@ reduce_pass_size () { parent_pass_dir="overlay.${parent}" local list_pass=$(mktemp) - local list_parent=$(mktemp) + local list_parent=$(mktemp) (cd $pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_pass) (cd $parent_pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_parent) @@ -188,7 +188,6 @@ create_chroot_pass () { lb chroot_apt install ${*} # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more # diff content than desired. So only running this on root pass. - # Also configure universe # Only configure universe on root passes if $(is_root_layer $pass); then lb chroot_archives chroot install ${*} From f5d3233c64e1f751f455a85a972f20d23b4ae5cf Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 31 Jan 2019 09:39:22 +0100 Subject: [PATCH 41/44] Fixed wonky indentation --- live-build/lb_chroot_layered | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered index 7f436528..3a3d50c0 100755 --- a/live-build/lb_chroot_layered +++ b/live-build/lb_chroot_layered @@ -124,7 +124,7 @@ reduce_pass_size () { (cd $parent_pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_parent) # Only iterate on common files with same type, owner, permission and size - comm -12 $list_pass $list_parent|cut -d'|' -f1|while read f; do + comm -12 $list_pass $list_parent|cut -d'|' -f1|while read f; do # If file contents are different, keep it if ! diff --brief --no-dereference "$pass_dir/$f" "$parent_pass_dir/$f" >/dev/null; then continue @@ -134,8 +134,8 @@ reduce_pass_size () { rm "$pass_dir/$f" done - rm $list_pass - rm $list_parent + rm $list_pass + rm $list_parent } create_chroot_pass () { From 3dce125d720f8c5818ba51854cdacad03784dd2d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lallement Date: Thu, 31 Jan 2019 10:35:27 +0100 Subject: [PATCH 42/44] Remove inidrd and vmlinuz from live layers after they've been moved to casper --- live-build/functions | 9 +++++++++ live-build/lb_binary_layered | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/live-build/functions b/live-build/functions index 4bf85af5..53f6c1a9 100644 --- a/live-build/functions +++ b/live-build/functions @@ -767,3 +767,12 @@ is_root_layer () { fi return 1 } + +is_live_layer () { + local pass=$1 + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue + return 0 + done + return 1 +} \ No newline at end of file diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered index cf1837ed..2cf56dbb 100755 --- a/live-build/lb_binary_layered +++ b/live-build/lb_binary_layered @@ -86,6 +86,12 @@ build_layered_squashfs () { lb_binary_includes $pass ${*} lb binary_hooks ${*} + # Copy initrd and vmlinuz outside of chroot and remove them from the layer squashfs + if $(is_live_layer "$pass"); then + lb binary_linux-image ${*} + rm -f chroot/boot/initrd.img-* chroot/boot/vmlinu{x,z}-* + fi + # Full manifest until that PASS squashfs_f_manifest="${base}.manifest" create_manifest "chroot" "${squashfs_f_manifest}.full" @@ -100,25 +106,19 @@ build_layered_squashfs () { # Prepare initrd + kernel # Main manifest and size files prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" - if [ ! -e "${prefix}.manifest" ]; then - for livepass in $LIVE_PASSES; do - [ "$livepass" != "$pass" ] && continue - lb binary_linux-image ${*} - - totalsize=$(cat ${squashfs_f_size}) - curpass="$pass" - while :; do - curpass=$(get_parent_pass $curpass) - # We climbed up the tree to the root layer, we are done - [ -z "$curpass" ] && break - - totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) - done - echo ${totalsize} > "${prefix}.size" - - cp "${squashfs_f_manifest}.full" "${prefix}.manifest" - break + if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then + totalsize=$(cat ${squashfs_f_size}) + curpass="$pass" + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) done + echo ${totalsize} > "${prefix}.size" + + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" fi (cd "overlay.${pass}/" && From a9991a908e057b06731df52173d17257b8c1631f Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 1 Feb 2019 17:22:01 +1300 Subject: [PATCH 43/44] revert change to ubuntu-server:live snap seeding --- live-build/auto/build | 5 ----- 1 file changed, 5 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index e9f20961..c6a68171 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -354,11 +354,6 @@ deb file:/var/lib/preinstalled-pool/ $LB_DISTRIBUTION $LB_PARENT_ARCHIVE_AREAS rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig fi case $PROJECT:$SUBPROJECT in - ubuntu-server:live) - lb chroot_resolv install - snap_prepare chroot - lb chroot_resolv remove - ;; *) if [ -e "config/seeded-snaps" ]; then snap_list=$(cat config/seeded-snaps) From ae04fee7c205299744bcb539684311378a219c22 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 1 Feb 2019 09:20:41 +1300 Subject: [PATCH 44/44] changelog entry --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 37a8ed1a..4d49b94f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +livecd-rootfs (2.559) UNRELEASED; urgency=medium + + [ Jean-Baptiste Lallement ] + * Add support for multi layer filesystem by generating one squashfs per layer + based on PASSES defined by project. + + -- Michael Hudson-Doyle Fri, 01 Feb 2019 09:19:41 +1300 + livecd-rootfs (2.558) disco; urgency=medium * Use the new linux-firmware-raspi2 for pi3 boot binary blobs as