diff --git a/debian/changelog b/debian/changelog index 37a8ed1a..36bf3537 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +livecd-rootfs (2.559) disco; urgency=medium + + [ Jean-Baptiste Lallement ] + [ Didier Roche ] + * Add support for multi layer filesystem by generating one squashfs per layer + based on PASSES defined by project. + Thanks Michael Hudson-Doyle for the reviews. + + -- Didier Roche Fri, 01 Feb 2019 08:04:35 +0100 + livecd-rootfs (2.558) disco; urgency=medium * Use the new linux-firmware-raspi2 for pi3 boot binary blobs as diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps index 0c0062a6..ea042caa 100755 --- a/debian/tests/default-bootstraps +++ b/debian/tests/default-bootstraps @@ -23,6 +23,7 @@ ALL_TRIPLETS=" lubuntu:: mythbuntu:: ubuntu:: + ubuntu:canary: ubuntu-base:: ubuntu-budgie:: ubuntu-budgie-desktop:: diff --git a/live-build/auto/build b/live-build/auto/build index 70287baa..c6a68171 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -235,126 +235,59 @@ echo "command. You will still need to ensure the 'man-db' package is installed." EOF chmod +x chroot/usr/bin/man fi - Chroot chroot "dpkg-divert --quiet --add \ - --divert /usr/sbin/update-initramfs.REAL --rename \ - /usr/sbin/update-initramfs" - cat > chroot/usr/sbin/update-initramfs <<'EOF' -#! /bin/sh -if [ $# != 1 ] || [ "$1" != -u ]; then - exec update-initramfs.REAL "$@" -fi -echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2 -exit 0 -EOF - chmod +x chroot/usr/sbin/update-initramfs - cat > config/hooks/999-undivert-update-initramfs.chroot <<'EOF' -#! /bin/sh -rm -f /usr/sbin/update-initramfs -dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs -EOF - - lb chroot "$@" - - if [ "${SUBPROJECT:-}" = minimized ]; then - # force removal of initramfs-tools, which we assert is not - # required for any minimized images but is still pulled in by - # default - # also remove landscape-common, which is heavyweight and - # in the server seed only to provide /etc/motd content which - # would only be seen by humans - Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ - apt-get -y purge initramfs-tools busybox-initramfs \ - busybox-static landscape-common" - # and if initramfs-tools was configured before our kernel, - # /etc/kernel/postinst.d/initramfs-tools will have created - # an initramfs despite the generic dpkg-divert; so remove it - # here. - rm -f chroot/boot/initrd.img-* - - # temporary workaround: don't remove linux-base which - # may have no other reverse-depends currently - Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ - apt-mark manual linux-base" - Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ - apt-get -y --purge autoremove" + if [ -n "${PASSES}" ]; then + PATH="config/:$PATH" lb chroot_layered "$@" + else + divert_update_initramfs + lb chroot "$@" fi - # remove crufty files that shouldn't be left in an image - rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old - Chroot chroot apt clean - if [ "${PROJECT}:${SUBPROJECT:-}" = "ubuntu-base:minimized" ]; then - # Save even more size by removing apt lists (that are currently removed - # downstream anyway) - rm -rf chroot/var/lib/apt/lists/* - # Having device notes in the docker image can cause problems - # (https://github.com/tianon/docker-brew-ubuntu-core/issues/62) - # so remove them. We only do this for docker out of an - # abundance of caution. - rm -rf chroot/dev/* - fi + # Let all configuration non multi-layered project here. + # If those are moving to a multi-layer layout, this needs to be + # done in chroot hooks. + if [ -z "$PASSES" ]; then + if [ "${SUBPROJECT:-}" = minimized ]; then + # force removal of initramfs-tools, which we assert is not + # required for any minimized images but is still pulled in by + # default + # also remove landscape-common, which is heavyweight and + # in the server seed only to provide /etc/motd content which + # would only be seen by humans + Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ + apt-get -y purge initramfs-tools busybox-initramfs \ + busybox-static landscape-common" + # and if initramfs-tools was configured before our kernel, + # /etc/kernel/postinst.d/initramfs-tools will have created + # an initramfs despite the generic dpkg-divert; so remove it + # here. + rm -f chroot/boot/initrd.img-* + + # temporary workaround: don't remove linux-base which + # may have no other reverse-depends currently + Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ + apt-mark manual linux-base" + Chroot chroot "env DEBIAN_FRONTEND=noninteractive \ + apt-get -y --purge autoremove" + fi - if [ -f config/universe-enabled ]; then - -# This is cargo-culted almost verbatim (with some syntax changes for -# preinstalled being slightly different in what it doesn't ask) from -# debian-installer's apt-setup: - -cat > chroot/etc/apt/sources.list << EOF -# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to -# newer versions of the distribution. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted - -## Major bug fix updates produced after the final release of the -## distribution. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted - -## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -## team. Also, please note that software in universe WILL NOT receive any -## review or updates from the Ubuntu security team. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe - -## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -## team, and may not be under a free licence. Please satisfy yourself as to -## your rights to use the software. Also, please note that software in -## multiverse WILL NOT receive any review or updates from the Ubuntu -## security team. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse - -## N.B. software from this repository may not have been tested as -## extensively as that contained in the main release, although it includes -## newer versions of some applications which may provide useful features. -## Also, please note that software in backports WILL NOT receive any review -## or updates from the Ubuntu security team. -deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse -# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse - -## Uncomment the following two lines to add software from Canonical's -## 'partner' repository. -## This software is not part of Ubuntu, but is offered by Canonical and the -## respective vendors as a service to Ubuntu users. -# deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner -# deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner - -deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted -# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted -deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe -# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe -deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse -# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse -EOF + clean_debian_chroot + + if [ "${PROJECT}:${SUBPROJECT:-}" = "ubuntu-base:minimized" ]; then + # Save even more size by removing apt lists (that are currently removed + # downstream anyway) + rm -rf chroot/var/lib/apt/lists/* + # Having device notes in the docker image can cause problems + # (https://github.com/tianon/docker-brew-ubuntu-core/issues/62) + # so remove them. We only do this for docker out of an + # abundance of caution. + rm -rf chroot/dev/* + fi - fi - if [ -d chroot/var/lib/preinstalled-pool ]; then - cat > config/indices/apt.conf <<-EOF + configure_universe + + if [ -d chroot/var/lib/preinstalled-pool ]; then + cat > config/indices/apt.conf <<-EOF Dir { ArchiveDir "chroot/var/lib/preinstalled-pool"; OverrideDir "config/indices"; @@ -371,11 +304,11 @@ Tree "dists/$LB_DISTRIBUTION" Contents " "; } EOF - for component in $LB_PARENT_ARCHIVE_AREAS; do - mkdir -p chroot/var/lib/preinstalled-pool/dists/$LB_DISTRIBUTION/$component/binary-$LB_ARCHITECTURES - done - apt-ftparchive generate config/indices/apt.conf - cat << @@EOF > chroot/etc/apt/sources.list.preinstall + for component in $LB_PARENT_ARCHIVE_AREAS; do + mkdir -p chroot/var/lib/preinstalled-pool/dists/$LB_DISTRIBUTION/$component/binary-$LB_ARCHITECTURES + done + apt-ftparchive generate config/indices/apt.conf + cat << @@EOF > chroot/etc/apt/sources.list.preinstall # This is a sources.list entry for a small pool of packages # provided on your preinstalled filesystem for your convenience. # @@ -387,93 +320,84 @@ deb file:/var/lib/preinstalled-pool/ $LB_DISTRIBUTION $LB_PARENT_ARCHIVE_AREAS # @@EOF - cp chroot/etc/apt/sources.list chroot/etc/apt/sources.list.orig - cp chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list - - echo "Waiting on gnupg ("$GPG_PROCESS") to finish generating a key." - wait $GPG_PROCESS - - R_ORIGIN=$(lsb_release -i -s) - R_CODENAME=$(lsb_release -c -s) - R_VERSION=$(lsb_release -r -s) - R_PRETTYNAME=$(echo $R_CODENAME | sed -e 's/^\(.\)/\U\1/') - - apt-ftparchive -o APT::FTPArchive::Release::Origin=$R_ORIGIN \ - -o APT::FTPArchive::Release::Label=$R_ORIGIN \ - -o APT::FTPArchive::Release::Suite=$R_CODENAME-local \ - -o APT::FTPArchive::Release::Version=$R_VERSION \ - -o APT::FTPArchive::Release::Codename=$R_CODENAME \ - -o APT::FTPArchive::Release::Description="$R_ORIGIN $R_PRETTYNAME Local" \ - release chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/ \ - > config/gnupg/Release - - gpg --home config/gnupg --detach-sign --armor config/gnupg/Release - mv config/gnupg/Release \ - chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release - mv config/gnupg/Release.asc \ - chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release.gpg - apt-key --keyring chroot/etc/apt/trusted.gpg add config/gnupg/pubring.gpg - find chroot/var/lib/preinstalled-pool/ -name Packages | xargs rm - - Chroot chroot "apt-get update" - cat chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig \ - > chroot/etc/apt/sources.list - rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig - fi - case $PROJECT:$SUBPROJECT in - *) - if [ -e "config/seeded-snaps" ]; then - snap_list=$(cat config/seeded-snaps) - preinstall_snaps $snap_list - fi - ;; - esac - - if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then - if [ "$ARCH" = "armhf" ]; then - INFO_DESC="$(lsb_release -d -s)" - echo "$INFO_DESC - $ARCH ($BUILDSTAMP)" >chroot/etc/media-info - mkdir -p chroot/var/log/installer - Chroot chroot "ln -s /etc/media-info /var/log/installer/media-info" + cp chroot/etc/apt/sources.list chroot/etc/apt/sources.list.orig + cp chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list + + echo "Waiting on gnupg ("$GPG_PROCESS") to finish generating a key." + wait $GPG_PROCESS + + R_ORIGIN=$(lsb_release -i -s) + R_CODENAME=$(lsb_release -c -s) + R_VERSION=$(lsb_release -r -s) + R_PRETTYNAME=$(echo $R_CODENAME | sed -e 's/^\(.\)/\U\1/') + + apt-ftparchive -o APT::FTPArchive::Release::Origin=$R_ORIGIN \ + -o APT::FTPArchive::Release::Label=$R_ORIGIN \ + -o APT::FTPArchive::Release::Suite=$R_CODENAME-local \ + -o APT::FTPArchive::Release::Version=$R_VERSION \ + -o APT::FTPArchive::Release::Codename=$R_CODENAME \ + -o APT::FTPArchive::Release::Description="$R_ORIGIN $R_PRETTYNAME Local" \ + release chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/ \ + > config/gnupg/Release + + gpg --home config/gnupg --detach-sign --armor config/gnupg/Release + mv config/gnupg/Release \ + chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release + mv config/gnupg/Release.asc \ + chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release.gpg + apt-key --keyring chroot/etc/apt/trusted.gpg add config/gnupg/pubring.gpg + find chroot/var/lib/preinstalled-pool/ -name Packages | xargs rm + + Chroot chroot "apt-get update" + cat chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig \ + > chroot/etc/apt/sources.list + rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig fi - fi - if [ "$PROJECT" = "ubuntu-cpc" ]; then - if [ "${SUBPROJECT:-}" = minimized ]; then - BUILD_NAME=minimal - else - BUILD_NAME=server + case $PROJECT:$SUBPROJECT in + *) + if [ -e "config/seeded-snaps" ]; then + snap_list=$(cat config/seeded-snaps) + preinstall_snaps $snap_list + fi + ;; + esac + + if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then + if [ "$ARCH" = "armhf" ]; then + INFO_DESC="$(lsb_release -d -s)" + echo "$INFO_DESC - $ARCH ($BUILDSTAMP)" >chroot/etc/media-info + mkdir -p chroot/var/log/installer + Chroot chroot "ln -s /etc/media-info /var/log/installer/media-info" + fi fi - cat > chroot/etc/cloud/build.info << EOF + if [ "$PROJECT" = "ubuntu-cpc" ]; then + if [ "${SUBPROJECT:-}" = minimized ]; then + BUILD_NAME=minimal + else + BUILD_NAME=server + fi + cat > chroot/etc/cloud/build.info << EOF build_name: $BUILD_NAME serial: $BUILDSTAMP EOF + fi + + configure_network_manager + + echo "===== Checking size of /usr/share/doc =====" + echo BEGIN docdirs + (cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr) + echo END docdirs + + /usr/share/livecd-rootfs/minimize-manual chroot fi - # If the image pre-installs network-manager, let it manage all devices by - # default. Installing NM on an existing system only manages wifi and wwan via - # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting - # the global backend to NM, netplan overrides that file. - if [ -e chroot/usr/sbin/NetworkManager ]; then - echo "===== Enabling all devices in NetworkManager ====" - mkdir -p chroot/etc/netplan - cat < chroot/etc/netplan/01-network-manager-all.yaml -# Let NetworkManager manage all devices on this system -network: - version: 2 - renderer: NetworkManager -EOF + if [ -n "${PASSES}" ]; then + PATH="config/:$PATH" lb binary_layered "$@" else - echo "==== NetworkManager not installed ====" + lb binary "$@" fi - echo "===== Checking size of /usr/share/doc =====" - echo BEGIN docdirs - (cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr) - echo END docdirs - - /usr/share/livecd-rootfs/minimize-manual chroot - - lb binary "$@" touch binary.success ) 2>&1 | tee binary.log @@ -506,6 +430,12 @@ for OUTPUT in ext2 ext3 ext4 manifest manifest-remove size squashfs; do chmod 644 "$PREFIX.$OUTPUT" done +# we don't need a manifest-remove for a layered-aware installer +if [ "$PROJECT" = "ubuntu" ] && [ "$SUBPROJECT" = "canary" ]; then + rm -f livecd.${PROJECT}-manifest-remove + rm -f config/manifest-minimal-remove +fi + if [ -e config/manifest-minimal-remove ]; then cp config/manifest-minimal-remove "$PREFIX.manifest-minimal-remove" fi diff --git a/live-build/auto/clean b/live-build/auto/clean index 9ddf752e..ab4ea402 100755 --- a/live-build/auto/clean +++ b/live-build/auto/clean @@ -8,3 +8,6 @@ rm -rf config rm -f binary.manifest binary.manifest-desktop binary.log rm -f livecd.* rm -rf userdata +rm -rf chroot.* +rm -rf overlay.* +rm -rf *.manifest.full diff --git a/live-build/auto/config b/live-build/auto/config index 65f5df56..0f7d2b0b 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -33,15 +33,77 @@ fi mkdir -p config cp -af /usr/share/livecd-rootfs/live-build/functions config/functions +cp -af /usr/share/livecd-rootfs/live-build/lb_*_layered config/ cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse mkdir -p config/package-lists +. config/functions + +OPTS= +COMPONENTS= +BINARY_REMOVE_LINUX=: +BINARY_IMAGES=none +MEMTEST=none +SOURCE='--source false' +BOOTLOADER=none +BOOTAPPEND_LIVE= +LIVE_TASK= +PREINSTALLED=false +PREINSTALL_POOL= +PREINSTALL_POOL_SEEDS= +PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" + +CHROOT_HOOKS= +BINARY_HOOKS= + +APT_OPTIONS=" --yes -oDebug::pkgDepCache::AutoInstall=yes " + +PASSES_TO_LAYERS=false +_PASSES_TO_LAYERS= # Stores the initial value of PASSES_TO_LAYERS +PASSES= + +_check_immutable_passes_to_layers () { + # Check that PASSES_TO_LAYERS didn't change ie we didn't switch from layer + # to non-layer image format and vice versa. + if [ -z "$_PASSES_TO_LAYERS" ]; then + _PASSES_TO_LAYERS=$PASSES_TO_LAYERS + fi + + if [ "$_PASSES_TO_LAYERS" != "$PASSES_TO_LAYERS" ]; then + echo "ERROR: PASSES_TO_LAYERS has changed from $_PASSES_TO_LAYERS to $PASSES_TO_LAYERS between 2 API calls. Please set it first." + exit 1 + fi +} + +_check_layers_only_API () { + # Check if a function is designed only for layered mode + # $1 name of the function + if [ "$PASSES_TO_LAYERS" != "true" ]; then + echo "$1 should only be used in layered mode, with PASSES_TO_LAYERS set to true" + exit 1 + fi +} + +_register_pass () { + # In layer mode, record a pass into the list of passes + # $1 Name of the pass + [ "$PASSES_TO_LAYERS" != "true" ] && return + + PASSES="$PASSES $1" +} + add_task () { local pass="$1" shift local task + local snap_list_file + local snap_list_files + local curseed + + _check_immutable_passes_to_layers + _register_pass "$pass" # The removal of direct task installation support from live-build # poses some problems. If the chroot has multiarch configured - for @@ -58,43 +120,113 @@ add_task () # probably a lurking timebomb that we need to fix. In the meantime, # the Architecture restriction at least saves us from abject # failure. + # + # We want as well to grab the snap list for each PASS. Resolve for all + # given task, and deduplicate them to generate snaps for the PASS. for task; do # We need a ridiculous number of backslashes to protect # parentheses from eval. echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" + + curseed=$(seed_from_task ${task}) + if [ -z "${curseed}" ]; then + echo "W: No seed matching task ${task}" + continue + fi + snap_list_file="config/package-lists/seed.${curseed}.snaplist.full" + snap_from_seed "${curseed}" $snap_list_file + if [ -e "$snap_list_file" ]; then + snap_list_files="${snap_list_files} $snap_list_file" + fi done + # The snap list is one line, and could be duplicated between seeds via inheritance. + # Uniquely sort them and store them back in one line. + if [ -n "${snap_list_files}" ]; then + cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full" + rm ${snap_list_files} + fi } add_package () { + # Adds a pass named pass_name composed of packages to install + # $1 pass + # $@ list of packages + local pass="$1" shift local pkg + _check_immutable_passes_to_layers + _register_pass "$pass" + for pkg; do echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" done } -OPTS= -COMPONENTS= -BINARY_REMOVE_LINUX=: -BINARY_IMAGES=none -MEMTEST=none -SOURCE='--source false' -BOOTLOADER=none -BOOTAPPEND_LIVE= -LIVE_TASK= -PREINSTALLED=false -PREINSTALL_POOL= -PREINSTALL_POOL_SEEDS= -PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" +remove_package () +{ + # Adds a pass named pass_name composed of packages to remove + # $1 pass + # $@ list of packages -CHROOT_HOOKS= -BINARY_HOOKS= + local pass="$1" + shift + local pkg -APT_OPTIONS=" --yes -oDebug::pkgDepCache::AutoInstall=yes " + _check_immutable_passes_to_layers + _check_layers_only_API "remove_package" + _register_pass "$pass" + + for pkg; do + echo "$pkg" >> "config/package-lists/livecd-rootfs.removal-list.chroot_$pass" + done +} + +add_packages_from_seed_regexp () { + # Creates one or more passes, depending on base_pass_name, from any seeds matching seed_regexp. + # $1 base pass + # $2 seeds (regexp) + local pass + + _check_immutable_passes_to_layers + _check_layers_only_API "add_packages_from_seed_regexp" + + for seed in $(ls config/germinate-output/|grep -P "$2"); do + pass=${1}.${seed} + _register_pass "$pass" + list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass + done +} + +remove_packages_from_seed_regexp() { + # Creates one or more passes, based on base_pass_name, from any seed matching seed_regexp. + # This package list is a list of packages to remove (and included in a previous dependent + # pass then), resulting from base_seed - {current_seed_match_from_regexp}. + # $1 base pass + # $2 base seed + # $3 seeds to remove from base seed (regexp). If empty, a no- sublayer is generated. + local pass + + _check_immutable_passes_to_layers + _check_layers_only_API "remove_packages_from_seed_regexp" + + local seed_regexp="$3" + if [ -z "${seed_regexp}" ]; then + pass="${1}.no-${2}" + _register_pass "$pass" + subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + return + fi + + for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do + pass="${1}.${seed}" + _register_pass "$pass" + subtract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass + done +} add_chroot_hook () { @@ -106,6 +238,44 @@ add_binary_hook () BINARY_HOOKS="${BINARY_HOOKS:+$BINARY_HOOKS }$1" } +_sanitize_passes () +{ + # Returns an uniquely ordered list of passes and ensure dependency tree is coherent + # $1 list of passes + local passes="$1" + [ -z "$passes" ] && return + + passes=$(echo $passes | tr ' ' '\n' | sort -u) + for pass in $passes; do + parent=$(get_parent_pass $pass) + # if root pass, no parent to find + [ -z "$parent" ] && continue + if [ $(echo "$passes"|grep -cE "^$parent\$") -ne 1 ]; then + echo "ERROR: '$parent' is required by '$pass' but is missing. Registered passes are:\n$passes" + exit 1 + fi + done + + # return the list of passes + echo $passes +} + +_get_live_passes () +{ + # Returns a list of all passes that ends with .live for automated live passes detection + # $1 list of passes + local passes="$1" + local livepasses="" + [ -z "$passes" ] && return + + for pass in $passes; do + if echo $pass | grep -Eq '\.live$'; then + livepasses="$pass $livepasses" + fi + done + echo $livepasses +} + if [ -z "${IMAGEFORMAT:-}" ]; then case $PROJECT:${SUBPROJECT:-} in ubuntu-cpc:*) @@ -234,6 +404,11 @@ case $IMAGEFORMAT in case $PROJECT in ubuntu-server|ubuntu-touch|ubuntu-touch-custom) ;; + ubuntu) + if [ "$SUBPROJECT" != "canary" ]; then + add_package live lupin-casper + fi + ;; *) add_package live lupin-casper ;; @@ -289,30 +464,6 @@ if [ "${SUBPROJECT:-}" = minimized ]; then OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal --linux-packages=linux-image" fi -# cribbed from cdimage, perhaps this should be a small helper script in germinate? -add_inheritance () { - case " $inherit " in - *" $1 "*) - ;; - *) - inherit="${inherit:+$inherit }$1" - ;; - esac -} - -expand_inheritance () { - for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do - expand_inheritance "$seed" - done - add_inheritance "$1" -} - -inheritance () { - inherit= - expand_inheritance "$1" - echo "$inherit" -} - mkdir -p config/germinate-output case $PROJECT in kubuntu-active*) @@ -359,10 +510,32 @@ esac case $PROJECT in ubuntu|ubuntu-dvd) - add_task install minimal standard ubuntu-desktop - LIVE_TASK='ubuntu-live' - case $ARCH in - amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; + + case ${SUBPROJECT:-} in + canary) + PASSES_TO_LAYERS="true" + add_task minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages + add_task minimal.standard ubuntu-desktop ubuntu-desktop-default-languages + add_task minimal.standard.live ubuntu-live + add_package minimal.standard.live lupin-casper + + case $ARCH in + amd64) add_package minimal.standard.live $SIGNED_KERNEL_PACKAGE ;; + esac + + # LANG PASS for minimal and standard + remove_packages_from_seed_regexp minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$' + remove_packages_from_seed_regexp minimal desktop-minimal-default-languages '' # none (if no default langpack is selected) + remove_packages_from_seed_regexp minimal.standard desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$' + remove_packages_from_seed_regexp minimal.standard desktop-default-languages '' # none (if no default langpack is selected) + ;; + *) + LIVE_TASK='ubuntu-live' + add_task install minimal standard ubuntu-desktop ubuntu-desktop-minimal-default-languages ubuntu-desktop-default-languages + case $ARCH in + amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; + esac + ;; esac ;; @@ -735,30 +908,11 @@ case $PROJECT:${SUBPROJECT:-} in ;; ubuntu-server:live) BASE_SEED='server' - # subiquity is seeded but in a separate squashfs via hooks; set HOOK_SNAPS and ALL_SNAPS. - HOOK_SNAPS='subiquity' - ALL_SNAPS='' ;; esac -if [ -n "${BASE_SEED}" ]; then - SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) - for seed in ${SEEDS_EXPANDED}; do - echo "snap: considering ${seed}" - file=config/germinate-output/${seed}.snaps - [ -e "${file}" ] || continue - # extract the first column (snap package name) from germinate's output - # translate the human-readable "foo (classic)" into a - # more machine readable "foo/classic" - seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") - for snap in ${seed_snaps}; do - echo "snap: found ${snap}" - ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" - done - done - if [ -n "${ALL_SNAPS}" ] || [ -n "${HOOK_SNAPS}" ]; then - echo "${ALL_SNAPS}" > config/seeded-snaps - fi +if [ "$PASSES_TO_LAYERS" != "true" ] && [ -n "${BASE_SEED}" ]; then + snap_from_seed "${BASE_SEED}" config/seeded-snaps fi # grab a list of packags to remove for a "minimal" installation from the seed @@ -777,7 +931,7 @@ fi export APT_OPTIONS -if [ "$PREINSTALLED" != "true" ] && [ "$LIVE_TASK" ]; then +if [ "$PREINSTALLED" != "true" ] && [ "$PASSES_TO_LAYERS" != "true" ] && [ "$LIVE_TASK" ]; then add_task live "$LIVE_TASK" fi @@ -892,9 +1046,23 @@ lb config noauto \ $OPTS \ "$@" +PASSES=$(_sanitize_passes "$PASSES") +LIVE_PASSES=${LIVE_PASSES:-$(_get_live_passes "$PASSES")} + +if [ -n "$PASSES" ] && [ -z "$LIVE_PASSES" ]; then + Echo_warning "Multi-layered mode is enabled, but we didn't find any live pass." \ + "Either set \$LIVE_PASSES or add a pass ending with '.live'." +fi + echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/chroot +if [ -n "$PASSES" ]; then + echo "PASSES=\"$PASSES\"" >> config/common +fi +if [ -n "$LIVE_PASSES" ]; then + echo "LIVE_PASSES=\"$LIVE_PASSES\"" >> config/common +fi echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary echo "BUILDSTAMP=\"$NOW\"" >> config/binary echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary diff --git a/live-build/functions b/live-build/functions index 2cbe3e02..53f6c1a9 100644 --- a/live-build/functions +++ b/live-build/functions @@ -137,6 +137,22 @@ mount_overlay() { "$path" } +get_lowerdirs_for_pass () { + # Returns the name of the lowerdir from the name of a pass + # $1 Name of the pass + local curpass="$1" + local lowerlayers="" + + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + lowerlayers="${lowerlayers}:overlay.${curpass}" + done + echo "${lowerlayers#:}" +} + mount_disk_image() { local disk_image=${1} local mountpoint=${2} @@ -373,6 +389,30 @@ release_ver() { distro-info --series="$LB_DISTRIBUTION" -r | awk '{ print $1 }' } +# cribbed from cdimage, perhaps this should be a small helper script in germinate? +add_inheritance () { + case " $inherit " in + *" $1 "*) + ;; + *) + inherit="${inherit:+$inherit }$1" + ;; + esac +} + +expand_inheritance () { + for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do + expand_inheritance "$seed" + done + add_inheritance "$1" +} + +inheritance () { + inherit= + expand_inheritance "$1" + echo "$inherit" +} + _snap_preseed() { # Download the snap/assertion and add to the preseed local CHROOT_ROOT=$1 @@ -387,6 +427,12 @@ _snap_preseed() { # Download the snap & assertion local snap_download_failed=0 + + # Preseed a snap only once + if [ -f ${snaps_dir}/${SNAP_NAME}_[0-9]*.snap ]; then + return + fi + chroot $CHROOT_ROOT sh -c " set -x; cd /var/lib/snapd/seed; @@ -432,6 +478,10 @@ snap_prepare_assertions() { local account_key_assertion="$assertions_dir/account-key" local account_assertion="$assertions_dir/account" + if [ -d "$assertions_dir" ]; then + return + fi + mkdir -p "$assertions_dir" mkdir -p "$snaps_dir" @@ -476,15 +526,10 @@ snap_prepare() { # used for the image's model assertion local CUSTOM_BRAND_MODEL=${2:-generic:generic-classic} - local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" - local snaps_dir="$seed_dir/snaps" - snap_prepare_assertions "$CHROOT_ROOT" "$CUSTOM_BRAND_MODEL" # Download the core snap - if ! [ -f $snaps_dir/core_[0-9]*.snap ] ; then - _snap_preseed $CHROOT_ROOT core stable - fi + _snap_preseed $CHROOT_ROOT core stable } snap_preseed() { @@ -501,3 +546,233 @@ snap_preseed() { fi _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL } + +snap_from_seed() { + local base_seed=$1 + local out=$2 + local all_snaps + local seeds_expanded + + seeds_expanded=$(inheritance ${base_seed}) + for seed in ${seeds_expanded}; do + echo "snap: considering ${seed}" + file=config/germinate-output/${seed}.snaps + [ -e "${file}" ] || continue + # extract the first column (snap package name) from germinate's output + # translate the human-readable "foo (classic)" into a + # more machine readable "foo/classic" + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") + for snap in ${seed_snaps}; do + echo "snap: found ${snap}" + all_snaps="${all_snaps:+${all_snaps} }${snap}" + done + done + if [ -n "${all_snaps}" ]; then + echo "${all_snaps}" > $out + fi +} + +seed_from_task () +{ + # Retrieve the name of the seed from a task name + local task=$1 + local seed + local seedfile + local seedfiles + + seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)" + if [ -n "$seedfile" ]; then + basename $seedfile .seedtext + return + fi + + seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)" + if [ -n "$seedfiles" ]; then + for seed in $(echo $seedfiles | xargs basename -s .seedtext); do + if [ ${PROJECT}-${seed} = $task ]; then + echo ${seed} + return + fi + done + fi +} + +list_packages_from_seed () { + # Store all packages for a given seed, including its seed dependency + # $1: Name of the seed to expand to a package list + + local all_seeds="$(inheritance $1)" + + for seed in $all_seeds; do + head -n-2 config/germinate-output/${seed}.seed|tail -n+3|awk '{print $1}' + done|sort -u +} + +subtract_package_lists() { + # Subtract a package list from another + # + # $1 source package list + # $2 Package list to subtract from source package list + local list1=$(mktemp) + local list2=$(mktemp) + + list_packages_from_seed $1 > list1 + list_packages_from_seed $2 > list2 + comm -23 list1 list2 + + rm list1 + rm list2 +} + +clean_debian_chroot() { + # remove crufty files that shouldn't be left in an image + rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old + Chroot chroot apt clean +} + +configure_universe() { + if [ -f config/universe-enabled ]; then + # This is cargo-culted almost verbatim (with some syntax changes for + # preinstalled being slightly different in what it doesn't ask) from + # debian-installer's apt-setup: + + cat > chroot/etc/apt/sources.list << EOF +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner +# deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner + +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse +EOF + +fi +} + +configure_network_manager() { + # If the image pre-installs network-manager, let it manage all devices by + # default. Installing NM on an existing system only manages wifi and wwan via + # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting + # the global backend to NM, netplan overrides that file. + if [ -e chroot/usr/sbin/NetworkManager -a ! -f chroot/etc/netplan/01-network-manager-all.yaml ]; then + echo "===== Enabling all devices in NetworkManager ====" + mkdir -p chroot/etc/netplan + cat < chroot/etc/netplan/01-network-manager-all.yaml +# Let NetworkManager manage all devices on this system +network: + version: 2 + renderer: NetworkManager +EOF + else + echo "==== NetworkManager not installed ====" + fi +} + +get_parent_pass () { + # return parent pass + # $1 name of the pass + # return parent pass name or '' if pass is root pass. + local pass="$1" + + parent_pass=${pass%.*} + if [ "${parent_pass}" = "${pass}" ]; then + return + fi + echo ${pass%.*} +} + +setenv_file () { + # Exposes an environment variable in a chroot + # $1 Name of the variable + # $2 Value of the variable + # $3 Path to the environment file of the chroot + local var="$1" + local val="$2" + local file="$3" + + grep -v "^$var" $file || true > $file.new + echo "${var}=${val}" >> $file.new + mv $file.new $file +} + +divert_update_initramfs () { + Chroot chroot "dpkg-divert --quiet --add \ + --divert /usr/sbin/update-initramfs.REAL --rename \ + /usr/sbin/update-initramfs" + cat > chroot/usr/sbin/update-initramfs <<'EOF' +#! /bin/sh +if [ $# != 1 ] || [ "$1" != -u ]; then + exec update-initramfs.REAL "$@" +fi +echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2 + +exit 0 +EOF + chmod +x chroot/usr/sbin/update-initramfs + + cat > config/hooks/999-undivert-update-initramfs.chroot <<'EOF' +#! /bin/sh +[ ! -f /usr/sbin/update-initramfs.REAL ] && exit 0 +rm -f /usr/sbin/update-initramfs +dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs +EOF +} + +is_root_layer () { + local pass=$1 + if [ -z "$(get_parent_pass $pass)" ]; then + return 0 + fi + return 1 +} + +is_live_layer () { + local pass=$1 + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue + return 0 + done + return 1 +} \ No newline at end of file diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered new file mode 100755 index 00000000..2cf56dbb --- /dev/null +++ b/live-build/lb_binary_layered @@ -0,0 +1,147 @@ +#!/bin/sh + +## live-build(7) - System Build Scripts +## Copyright (C) 2006-2012 Daniel Baumann +## +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING. +## This is free software, and you are welcome to redistribute it +## under certain conditions; see COPYING for details. + + +set -e + +# Including common functions +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh + + +# Automatically populating config tree +if [ -x auto/config ] && [ ! -e .build/config ] +then + Echo_message "Automatically populating config tree." + lb config +fi + +# Setting static variables +DESCRIPTION="$(Echo 'build binary images')" +HELP="" +USAGE="${PROGRAM} [--force]" + +Arguments "${@}" + +# Reading configuration files +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source +Set_defaults + +# Setup cleanup function +Setup_cleanup + +. config/functions + +lb_binary_includes () { + # Copying includes from pass subdirectory + local pass="$1" + + if [ ! -d config/includes.binary.${pass} ]; then + return + fi + + cd config/includes.binary.${pass} + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot + cd "${OLDPWD}" +} + +build_layered_squashfs () { + local pass=$1 + shift 1 # restore ${*} + + Echo_message "lb_binary_layered: treating pass $pass" + + # Building squashfs filesystem & manifest + local overlay_dir="overlay.${pass}" + base="${PWD}/livecd.${PROJECT}.${pass}" + squashfs_f="${base}.squashfs" + + # We have already treated that pass + if [ -f "${squashfs_f}" ]; then + return + fi + + rm -f .build/binary_chroot + + mkdir -p "$overlay_dir/" + lowerdirs=$(get_lowerdirs_for_pass $pass) + if [ -n "$lowerdirs" ]; then + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ + else + ln -s "$overlay_dir/" chroot + fi + + export PASS=${pass} + setenv_file PASS "${pass}" config/environment.chroot + + # Cleanup root filesystem + lb binary_chroot ${*} + + lb_binary_includes $pass ${*} + lb binary_hooks ${*} + + # Copy initrd and vmlinuz outside of chroot and remove them from the layer squashfs + if $(is_live_layer "$pass"); then + lb binary_linux-image ${*} + rm -f chroot/boot/initrd.img-* chroot/boot/vmlinu{x,z}-* + fi + + # Full manifest until that PASS + squashfs_f_manifest="${base}.manifest" + create_manifest "chroot" "${squashfs_f_manifest}.full" + + # Delta manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + + squashfs_f_size="${base}.size" + du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" + + # We take first live pass for "global" ISO properties (used by installers and checkers): + # Prepare initrd + kernel + # Main manifest and size files + prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" + if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then + totalsize=$(cat ${squashfs_f_size}) + curpass="$pass" + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) + done + echo ${totalsize} > "${prefix}.size" + + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + fi + + (cd "overlay.${pass}/" && + mksquashfs . ${squashfs_f} \ + -no-progress -xattrs -comp xz ) + + if [ -n "$lowerdirs" ]; then + umount chroot + rmdir chroot + else + rm chroot + fi +} + +for _PASS in $PASSES +do + build_layered_squashfs "${_PASS}" ${*} +done + +# Ubiquity-compatible removal manifest for ISO not using a layered-aware installer +if [ -n "$(ls livecd.${PROJECT}.*install.live.manifest.full 2>/dev/null)" ] && \ + [ -n "$(ls livecd.${PROJECT}.*install.manifest.full 2>/dev/null)" ]; then + echo "$(diff livecd.${PROJECT}.*install.live.manifest.full livecd.${PROJECT}.*install.manifest.full | awk '/^< / { print $2 }')" > livecd.${PROJECT}-manifest-remove +fi + +chmod 644 *.squashfs *.manifest* *.size diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered new file mode 100755 index 00000000..3a3d50c0 --- /dev/null +++ b/live-build/lb_chroot_layered @@ -0,0 +1,279 @@ +#!/bin/sh + +## live-build(7) - System Build Scripts +## Copyright (C) 2006-2012 Daniel Baumann +## +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING. +## This is free software, and you are welcome to redistribute it +## under certain conditions; see COPYING for details. + +## This is a fork of lb_chroot for layered live system. +## We don't want leaking host configuration in each layer, and so, +## we clean and setup the chroot each time. +## In addition, we create the squashfs for each layer, but top one (live) +## which still can be configured after lb chroot call. + +set -e + +# Including common functions +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh + +# Automatically populating config tree +if [ -x auto/config ] && [ ! -e .build/config ] +then + Echo_message "Automatically populating config tree." + lb config +fi + +# Setting static variables +DESCRIPTION="$(Echo 'customize the Debian system')" +HELP="" +USAGE="${PROGRAM} [--force]" + +Arguments "${@}" + +# Reading configuration files +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source +Set_defaults + +# Setup cleanup function +Setup_cleanup + +. config/functions + +lb_chroot_remove_packages () { + # Remove packages from the chroot specific to this layer + # + # $1: Name of the pass* + local pass=$1 + + Expand_packagelist "$(basename config/package-lists/*.removal-list.chroot_${pass})" "config/package-lists" \ + >> chroot/root/packages.chroot.removal + Chroot chroot "xargs --arg-file=/root/packages.chroot.removal apt-get ${APT_OPTIONS} autoremove --purge" + rm -f chroot/root/packages.chroot.removal +} + +# Create the snap list specific to this layer +lb_chroot_snap_lists () { + local pass=$1 + + # This assumes that the prefix is unique for a given project + local snap_for_pass=$(ls config/package-lists/*.snaplist.chroot_${pass}.full 2>/dev/null || true) + parent_pass=$(get_parent_pass $pass) + local snap_for_parent_pass=$(ls config/package-lists/*.snaplist.chroot_${parent_pass}.full 2>/dev/null || true) + + if [ -z "${snap_for_pass}" ]; then + return + fi + + if [ -z "${snap_for_parent_pass}" ]; then + cp ${snap_for_pass} ${snap_for_pass%.full} + return + fi + + # Generate a list of snaps added to a layer. + diff -NU0 ${snap_for_parent_pass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full} +} + +lb_chroot_install_snaps () { + # Prepare the snap environment and install snaps into a chroot + # + # $1: Name of the pass + + local snaplist_file=$(ls config/package-lists/*.snaplist.chroot_${1} 2>/dev/null || true) + + if [ -z "${snaplist_file}" ]; then + return + fi + + snap_prepare chroot + + while read snap; do + snap_preseed chroot "${snap}" + done < $snaplist_file +} + +lb_chroot_includes () { + # Copying includes from pass subdirectory + local pass="$1" + + if [ ! -d config/includes.chroot.${pass} ]; then + return + fi + + cd config/includes.chroot.${pass} + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot + cd "${OLDPWD}" +} + +reduce_pass_size () { + # Remove duplicated files between parent and current pass + # Note the empty directories created in a child pass are not removed + local pass=$1 + local parent="$(get_parent_pass $pass)" + + $(is_root_layer $pass) && return + + pass_dir="overlay.${pass}" + parent_pass_dir="overlay.${parent}" + + local list_pass=$(mktemp) + local list_parent=$(mktemp) + + (cd $pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_pass) + (cd $parent_pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_parent) + + # Only iterate on common files with same type, owner, permission and size + comm -12 $list_pass $list_parent|cut -d'|' -f1|while read f; do + # If file contents are different, keep it + if ! diff --brief --no-dereference "$pass_dir/$f" "$parent_pass_dir/$f" >/dev/null; then + continue + fi + # Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta. + Echo_message "reduce_pass_size: '$f' is strictly identical between $parent and $pass. Removing." + rm "$pass_dir/$f" + done + + rm $list_pass + rm $list_parent +} + +create_chroot_pass () { + local pass=$1 + shift 1 # restore ${*} + + Echo_message "lb_chroot_layered: treating pass $pass" + + # We have already treated that pass just return. + local overlay_dir="overlay.${pass}" + if [ -d "$overlay_dir/" ]; then + return + fi + + # Only get some function executed on root passes + # Copy bootstrap on root layers + if $(is_root_layer $pass); then + rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks + cp -a chroot.bootstrap/ "$overlay_dir/" + fi + # Others have to be executed on every pass + rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive + + mkdir -p "$overlay_dir/" + lowerdirs=$(get_lowerdirs_for_pass $pass) + if [ -n "$lowerdirs" ]; then + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ + else + ln -s "$overlay_dir/" chroot + fi + + export PASS=${pass} + setenv_file PASS "${pass}" config/environment.chroot + + # Configuring chroot + lb chroot_devpts install ${*} + lb chroot_proc install ${*} + lb chroot_sysfs install ${*} + # We run chroot_hacks only on root layers (update-initramfs diverted) + if $(is_root_layer $pass); then + divert_update_initramfs + fi + lb chroot_debianchroot install ${*} + lb chroot_dpkg install ${*} + lb chroot_tmpfs install ${*} + lb chroot_hosts install ${*} + lb chroot_resolv install ${*} + lb chroot_hostname install ${*} + lb chroot_apt install ${*} + # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more + # diff content than desired. So only running this on root pass. + # Only configure universe on root passes + if $(is_root_layer $pass); then + lb chroot_archives chroot install ${*} + configure_universe + fi + + # Customizing chroot + lb chroot_linux-image ${*} + lb chroot_preseed ${*} + lb chroot_early_hooks ${*} + + lb chroot_package-lists ${pass} ${*} + lb chroot_install-packages ${pass} ${*} + lb_chroot_remove_packages ${pass} ${*} + + # Snap management + lb_chroot_snap_lists ${pass} ${*} + lb_chroot_install_snaps ${pass} ${*} + + configure_network_manager + + # Mark kernel headers as autoremovable + Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ + | awk '/^i/ {print $2}' > chroot.headers + for i in $(cat chroot.headers); do + Chroot chroot "apt-mark auto $i" + done + + Chroot chroot "apt-get --purge -y autoremove" + + # Add live packages to live layers + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue + lb chroot_live-packages ${*} + break + done + + # Run includes by pass + lb_chroot_includes ${pass} ${*} + + lb chroot_hooks ${*} + + # Run chroot_hacks only on root layers. + # chroot_hacks changes the mode of boot/initrd*. The side effect in + # layered mode is to create an initrd on each layer with a significant + # impact on image size (+30MB per layer). This is an issue even with + # update-initramfs disabled. + lb chroot_hacks ${*} + + lb chroot_interactive ${*} + + # Misc ubuntu cleanup and post-layer configuration + clean_debian_chroot + /usr/share/livecd-rootfs/minimize-manual chroot + + Chroot chroot "dpkg-query -W" > chroot.packages.${pass} + + # Deconfiguring chroot + if $(is_root_layer $pass); then + lb chroot_archives chroot remove ${*} + fi + lb chroot_apt remove ${*} + lb chroot_hostname remove ${*} + lb chroot_resolv remove ${*} + lb chroot_hosts remove ${*} + lb chroot_tmpfs remove ${*} + lb chroot_dpkg remove ${*} + lb chroot_debianchroot remove ${*} + lb chroot_sysfs remove ${*} + lb chroot_proc remove ${*} + lb chroot_devpts remove ${*} + + if [ -n "$lowerdirs" ]; then + umount chroot + rmdir chroot + else + rm chroot + fi + + reduce_pass_size $pass +} + +if [ ! -d chroot.bootstrap/ ]; then + mv chroot/ chroot.bootstrap/ +fi +for _PASS in $PASSES +do + create_chroot_pass "$_PASS" ${*} +done