diff --git a/config/functions b/config/functions new file mode 100644 index 00000000..d86f27d6 --- /dev/null +++ b/config/functions @@ -0,0 +1,922 @@ +# vi: ts=4 expandtab syntax=sh + +# default imagesize = 2252*1024**2 = 2.2G (the current size we ship) +imagesize=${IMAGE_SIZE:-2361393152} +fs_label="${FS_LABEL:-rootfs}" + +rootfs_dev_mapper= +loop_device= +loop_raw= +backing_img= + +clean_loops() { + local kpartx_ret + local kpartx_stdout + + if [ -n "${backing_img}" ]; then + # sync before removing loop to avoid "Device or resource busy" errors + sync + kpartx_ret="" + kpartx_stdout=$(kpartx -v -d "${backing_img}") || kpartx_ret=$? + echo "$kpartx_stdout" + if [ -n "$kpartx_ret" ]; then + if echo "$kpartx_stdout" | grep -q "loop deleted"; then + echo "Suppressing kpartx returning error (#860894)" + else + exit $kpartx_ret + fi + fi + unset backing_img + fi + + if [ -z "${rootfs_dev_mapper}" ]; then + return 0 + fi + + unset loop_device + unset loop_raw + unset rootfs_dev_mapper +} + +create_empty_disk_image() { + # Prepare an empty disk image + dd if=/dev/zero of="$1" bs=1 count=0 seek="${imagesize}" +} + +create_manifest() { + local chroot_root=${1} + local target_file=${2} + echo "create_manifest chroot_root: ${chroot_root}" + dpkg-query --show --admindir="${chroot_root}/var/lib/dpkg" > ${target_file} + echo "create_manifest call to dpkg-query finished." + ./config/snap-seed-parse "${chroot_root}" "${target_file}" + echo "create_manifest call to snap_seed_parse finished." + if [ "$PROJECT" = ubuntu-cpc ]; then + echo "create_manifest creating file listing." + local target_filelist=${2%.manifest}.filelist + (cd "${chroot_root}" && find -xdev) > "${target_filelist}" + fi + echo "create_manifest finished" +} + +make_ext4_partition() { + device="$1" + label=${fs_label:+-L "${fs_label}"} + mkfs.ext4 -F -b 4096 -i 8192 -m 0 ${label} -E resize=536870912 "$device" +} + +mount_image() { + trap clean_loops EXIT + backing_img="$1" + local rootpart="$2" + kpartx_mapping="$(kpartx -s -v -a ${backing_img})" + + # Find the loop device + loop_p1="$(echo -e ${kpartx_mapping} | head -n1 | awk '{print$3}')" + loop_device="/dev/${loop_p1%p[0-9]*}" + if [ ! -b ${loop_device} ]; then + echo "unable to find loop device for ${backing_img}" + exit 1 + fi + + # Find the rootfs location + rootfs_dev_mapper="/dev/mapper/${loop_p1%%[0-9]}${rootpart}" + if [ ! -b "${rootfs_dev_mapper}" ]; then + echo "${rootfs_dev_mapper} is not a block device"; + exit 1 + fi + + # Add some information to the debug logs + echo "Mounted disk image ${backing_img} to ${rootfs_dev_mapper}" + blkid ${rootfs_dev_mapper} + + return 0 +} + +setup_mountpoint() { + local mountpoint="$1" + + mount --rbind /dev "$mountpoint/dev" + mount proc-live -t proc "$mountpoint/proc" + mount sysfs-live -t sysfs "$mountpoint/sys" + mount -t tmpfs none "$mountpoint/tmp" + mount -t tmpfs none "$mountpoint/var/lib/apt" + mount -t tmpfs none "$mountpoint/var/cache/apt" + mv "$mountpoint/etc/resolv.conf" resolv.conf.tmp + cp /etc/resolv.conf "$mountpoint/etc/resolv.conf" + chroot "$mountpoint" apt-get update + +} + +teardown_mountpoint() { + # Reverse the operations from setup_mountpoint + local mountpoint="$1" + + # ensure we have exactly one trailing slash, and escape all slashes for awk + mountpoint_match=$(echo "$mountpoint" | sed -e's,/$,,; s,/,\\/,g;')'\/' + # sort -r ensures that deeper mountpoints are unmounted first + for submount in $(awk $mountpoint/usr/sbin/policy-rc.d << EOF +#!/bin/sh +# ${IMAGE_STR} +echo "All runlevel operations denied by policy" >&2 +exit 101 +EOF + chmod 0755 $mountpoint/usr/sbin/policy-rc.d + +} + +umount_partition() { + local mountpoint=${1} + teardown_mountpoint $mountpoint + mount --make-private $mountpoint + umount $mountpoint + udevadm settle + + if [ -n "${rootfs_dev_mapper}" -a -b "${rootfs_dev_mapper}" ]; then + # buildd's don't have /etc/mtab symlinked + # /etc/mtab is needed in order zerofree space for ext4 filesystems + [ -e /etc/mtab ] || ln -s /proc/mounts /etc/mtab + + # both of these are likely overkill, but it does result in slightly + # smaller ext4 filesystem + e2fsck -y -E discard ${rootfs_dev_mapper} + zerofree ${rootfs_dev_mapper} + fi +} + +umount_disk_image() { + mountpoint="$1" + + local uefi_dev="/dev/mapper${loop_device///dev/}p15" + if [ -e "$mountpoint/boot/efi" -a -b "$uefi_dev" ]; then + # zero fill free space in UEFI partition + cat < /dev/zero > "$mountpoint/boot/efi/bloat_file" 2> /dev/null || true + rm "$mountpoint/boot/efi/bloat_file" + mount --make-private "$mountpoint/boot/efi" + umount --detach-loop "$mountpoint/boot/efi" + fi + + if [ -e $mountpoint/usr/sbin/policy-rc.d ]; then + rm $mountpoint/usr/sbin/policy-rc.d + fi + umount_partition $mountpoint + clean_loops +} + +modify_vmdk_header() { + # Modify the VMDK headers so that both VirtualBox _and_ VMware can + # read the vmdk and import them. + + vmdk_name="${1}" + descriptor=$(mktemp) + newdescriptor=$(mktemp) + + # Extract the vmdk header for manipulation + dd if="${vmdk_name}" of="${descriptor}" bs=1 skip=512 count=1024 + + # The sed lines below is where the magic is. Specifically: + # ddb.toolsVersion: sets the open-vm-tools so that VMware shows + # the tooling as current + # ddb.virtualHWVersion: set the version to 7, which covers most + # current versions of VMware + # createType: make sure its set to stream Optimized + # remove the vmdk-stream-converter comment and replace with + # # Disk DescriptorFile. This is needed for Virtualbox + # remove the comments from vmdk-stream-converter which causes + # VirtualBox and others to fail VMDK validation + + sed -e 's|# Description file.*|# Disk DescriptorFile|' \ + -e '/# Believe this is random*/d' \ + -e '/# Indicates no parent/d' \ + -e '/# The Disk Data Base/d' \ + -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ + "${descriptor}" > "${newdescriptor}" + + # The header is cannot be bigger than 1024 + expr $(stat --format=%s ${newdescriptor}) \< 1024 > /dev/null 2>&1 || { + echo "descriptor is too large, VMDK will be invalid!"; exit 1; } + + # Overwrite the vmdk header with our new, modified one + dd conv=notrunc,nocreat \ + if="${newdescriptor}" of="${vmdk_name}" \ + bs=1 seek=512 count=1024 + + rm ${descriptor} ${newdescriptor} +} + +create_vmdk() { + # There is no real good way to create a _compressed_ VMDK using open source + # tooling that works across multiple VMDK-capable platforms. This functions + # uses vmdk-stream-converter and then calls modify_vmdk_header to produce a + # compatible VMDK. + + src="$1" + destination="$2" + size="${3:-10240}" + + streamconverter="VMDKstream" + scratch_d=$(mktemp -d) + cp ${src} ${scratch_d}/resize.img + + truncate --size=${size}M ${scratch_d}/resize.img + python3 -m ${streamconverter} ${scratch_d}/resize.img ${destination} + modify_vmdk_header ${destination} + + qemu-img info ${destination} + rm -rf ${scratch_d} +} + +create_derivative() { + # arg1 is the disk type + # arg2 is the new name + unset derivative_img + case ${1} in + uefi) disk_image="binary/boot/disk-uefi.ext4"; + dname="${disk_image//-uefi/-$2-uefi}";; + *) disk_image="binary/boot/disk.ext4"; + dname="${disk_image//.ext4/-$2.ext4}";; + esac + + if [ ! -e ${disk_image} ]; then + echo "Did not find ${disk_image}!"; exit 1; + fi + + cp ${disk_image} ${dname} + export derivative_img=${dname} +} + +convert_to_qcow2() { + src="$1" + destination="$2" + qemu-img convert -c -O qcow2 -o compat=0.10 "$src" "$destination" + qemu-img info "$destination" +} + +replace_grub_root_with_label() { + # When update-grub is run, it will detect the disks in the build system. + # Instead, we want grub to use the right labelled disk + CHROOT_ROOT="$1" + + # If boot by partuuid has been requested, don't override. + if [ -f $CHROOT_ROOT/etc/default/grub.d/40-force-partuuid.cfg ] && \ + grep -q ^GRUB_FORCE_PARTUUID= $CHROOT_ROOT/etc/default/grub.d/40-force-partuuid.cfg + then + return 0 + fi + sed -i -e "s,root=[^ ]*,root=LABEL=${fs_label}," \ + "$CHROOT_ROOT/boot/grub/grub.cfg" +} + + +# When running update-grub in a chroot on a build host, we don't want it to +# probe for disks or probe for other installed OSes. Extract common +# diversion wrappers, so this isn't reinvented differently for each image. +divert_grub() { + CHROOT_ROOT="$1" + + # Don't divert all of grub-probe here; just the scripts we don't want + # running. Otherwise, you may be missing part-uuids for the search + # command, for example. ~cyphermox + + chroot "$CHROOT_ROOT" dpkg-divert --local \ + --divert /etc/grub.d/30_os-prober.dpkg-divert \ + --rename /etc/grub.d/30_os-prober + + # Divert systemd-detect-virt; /etc/kernel/postinst.d/zz-update-grub + # no-ops if we are in a container, and the launchpad farm runs builds + # in lxd. We therefore pretend that we're never in a container (by + # exiting 1). + chroot "$CHROOT_ROOT" dpkg-divert --local \ + --rename /usr/bin/systemd-detect-virt + echo "exit 1" > "$CHROOT_ROOT"/usr/bin/systemd-detect-virt + chmod +x "$CHROOT_ROOT"/usr/bin/systemd-detect-virt +} + +undivert_grub() { + CHROOT_ROOT="$1" + + chroot "$CHROOT_ROOT" dpkg-divert --remove --local \ + --divert /etc/grub.d/30_os-prober.dpkg-divert \ + --rename /etc/grub.d/30_os-prober + + rm "$CHROOT_ROOT"/usr/bin/systemd-detect-virt + chroot "$CHROOT_ROOT" dpkg-divert --remove --local \ + --rename /usr/bin/systemd-detect-virt +} + +recreate_initramfs() { + # Regenerate the initramfs by running update-initramfs in the + # chroot at $1 and copying the generated initramfs + # around. Beware that this was written for a single use case + # (live-server) and may not work in all cases without + # tweaking... + # config/common must be sourced before calling this function. + CHROOT="$1" + # Start by cargo culting bits of lb_chroot_hacks: + if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then + echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$CHROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf + fi + chroot "$CHROOT" sh -c "${UPDATE_INITRAMFS_OPTIONS:-} update-initramfs -k all -t -u" + rm -rf "$CHROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf + # Then bits of lb_binary_linux-image: + case "${LB_INITRAMFS}" in + casper) + DESTDIR="binary/casper" + ;; + + live-boot) + DESTDIR="binary/live" + ;; + + *) + DESTDIR="binary/boot" + ;; + esac + mv "$CHROOT"/boot/initrd.img-* $DESTDIR +} + +release_ver() { + # Return the release version number + distro-info --series="$LB_DISTRIBUTION" -r | awk '{ print $1 }' +} + +# cribbed from cdimage, perhaps this should be a small helper script in germinate? +add_inheritance () { + case " $inherit " in + *" $1 "*) + ;; + *) + inherit="${inherit:+$inherit }$1" + ;; + esac +} + +expand_inheritance () { + for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do + expand_inheritance "$seed" + done + add_inheritance "$1" +} + +inheritance () { + inherit= + expand_inheritance "$1" + echo "$inherit" +} + +_snap_post_process() { + # Look for the 'core' snap. If it is not present, assume that the image + # contains only snaps with bases >= core18. In that case snapd is + # preseeded. However, when 'core' is being installed and snapd has not + # been installed by a call to 'snap_preseed' (see below) then it is + # removed again. + local CHROOT_ROOT=$1 + local SNAP_NAME=$2 + + local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" + local snaps_dir="$seed_dir/snaps" + local seed_yaml="$seed_dir/seed.yaml" + local assertions_dir="$seed_dir/assertions" + local snapd_install_stamp="$seed_dir/.snapd-explicit-install-stamp" + + case $SNAP_NAME in + core[0-9]*) + # If the 'core' snap is not present, assume we are coreXX-only and + # install the snapd snap. + if [ ! -f ${snaps_dir}/core_[0-9]*.snap ]; then + _snap_preseed $CHROOT_ROOT snapd stable + fi + ;; + core) + # If the snapd snap has been seeded, but not marked as explicitly + # installed (see snap_preseed below), then remove it. + if [ -f ${snaps_dir}/snapd_[0-9]*.snap ] && \ + [ ! -f "$snapd_install_stamp" ] + then + # Remove snap, assertions and entry in seed.yaml + rm -f ${snaps_dir}/snapd_[0-9]*.snap + rm -f ${assertions_dir}/snapd_[0-9]*.assert + sed -i -e'N;/name: snapd/,+2d' $seed_yaml + fi + ;; + *) + # ignore + ;; + esac +} + +_snap_preseed() { + # Download the snap/assertion and add to the preseed + local CHROOT_ROOT=$1 + local SNAP=$2 + local SNAP_NAME=${SNAP%/*} + local CHANNEL=${3:?Snap channel must be specified} + + local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" + local snaps_dir="$seed_dir/snaps" + local seed_yaml="$seed_dir/seed.yaml" + local assertions_dir="$seed_dir/assertions" + + # Download the snap & assertion + local snap_download_failed=0 + + # Preseed a snap only once + if [ -f ${snaps_dir}/${SNAP_NAME}_[0-9]*.snap ]; then + return + fi + + sh -c " + set -x; + cd \"$CHROOT_ROOT/var/lib/snapd/seed\"; + SNAPPY_STORE_NO_CDN=1 snap download \ + --cohort="${COHORT_KEY:-}" \ + --channel=\"$CHANNEL\" \"$SNAP_NAME\"" || snap_download_failed=1 + if [ $snap_download_failed = 1 ] ; then + echo "If the channel ($CHANNEL) includes '*/ubuntu-##.##' track per " + echo "Ubuntu policy (ex. stable/ubuntu-18.04) the publisher will need " + echo "to temporarily create the channel/track to allow fallback during" + echo "download (ex. stable/ubuntu-18.04 falls back to stable if the" + echo "prior had been created in the past)." + exit 1 + fi + + mv -v $seed_dir/*.assert $assertions_dir + mv -v $seed_dir/*.snap $snaps_dir + + # Pre-seed snap's base + case $SNAP_NAME in + snapd) + # snapd is self-contained, ignore base + ;; + core|core[0-9][0-9]) + # core and core## are self-contained, ignore base + ;; + *) + # Determine which core snap is needed + local snap_info + + # snap info doesn't have --channel, so must run agains the downloaded snap + snap_info=$(snap info --verbose ${snaps_dir}/${SNAP_NAME}_[0-9]*.snap) + + if [ $? -ne 0 ]; then + echo "Failed to retrieve base of $SNAP_NAME!" + exit 1 + fi + + local core_snap=$(echo "$snap_info" | grep '^base:' | awk '{print $2}') + + # If snap info does not list a base use 'core' + core_snap=${core_snap:-core} + + _snap_preseed $CHROOT_ROOT $core_snap stable + ;; + esac + + # Add the snap to the seed.yaml + ! [ -e $seed_yaml ] && echo "snaps:" > $seed_yaml + cat <> $seed_yaml + - + name: ${SNAP_NAME} + channel: ${CHANNEL} +EOF + + case ${SNAP} in */classic) echo " classic: true" >> $seed_yaml;; esac + + echo -n " file: " >> $seed_yaml + (cd $snaps_dir; ls -1 ${SNAP_NAME}_*.snap) >> $seed_yaml + + _snap_post_process $CHROOT_ROOT $SNAP_NAME +} + +snap_prepare_assertions() { + # Configure basic snapd assertions + local CHROOT_ROOT=$1 + # A colon-separated string of brand:model to be used for the image's model + # assertion + local CUSTOM_BRAND_MODEL=$2 + + local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" + local snaps_dir="$seed_dir/snaps" + local assertions_dir="$seed_dir/assertions" + local model_assertion="$assertions_dir/model" + local account_key_assertion="$assertions_dir/account-key" + local account_assertion="$assertions_dir/account" + + local brand="$(echo $CUSTOM_BRAND_MODEL | cut -d: -f 1)" + local model="$(echo $CUSTOM_BRAND_MODEL | cut -d: -f 2)" + + # Get existing model and brand assertions to compare with new parameters + # For customized images, snap_prepare_assertions is called several times + # with different brand or model. In this case we want to overwrite + # existing brand and models. + local override_model_branch="false" + if [ -e "$model_assertion" ] ; then + existing_model=$(awk '/^model: / {print $2}' $model_assertion) + existing_brand=$(awk '/^brand-id: / {print $2}' $model_assertion) + + if [ "$existing_model" != "$model" ] || [ "$existing_brand" != "$brand" ]; then + override_model_branch="true" + fi + fi + + # Exit if assertions dir exists and we didn't change model or brand + if [ -d "$assertions_dir" ] && [ "$override_model_branch" = "false" ]; then + return + fi + + mkdir -p "$assertions_dir" + mkdir -p "$snaps_dir" + + # Clear the assertions if they already exist + if [ -e "$model_assertion" ] ; then + echo "snap_prepare_assertions: replacing $existing_brand:$existing_model with $brand:$model" + rm "$model_assertion" + rm "$account_key_assertion" + rm "$account_assertion" + fi + + if ! [ -e "$model_assertion" ] ; then + snap known --remote model series=16 \ + model=$model brand-id=$brand \ + > "$model_assertion" + fi + + if ! [ -e "$account_key_assertion" ] ; then + local account_key=$(sed -n -e's/sign-key-sha3-384: //p' \ + < "$model_assertion") + snap known --remote account-key \ + public-key-sha3-384="$account_key" \ + > "$account_key_assertion" + fi + + if ! [ -e "$account_assertion" ] ; then + local account=$(sed -n -e's/account-id: //p' < "$account_key_assertion") + snap known --remote account account-id=$account \ + > "$account_assertion" + fi +} + +snap_prepare() { + # Configure basic snapd assertions and pre-seeds the 'core' snap + local CHROOT_ROOT=$1 + # Optional. If set, should be a colon-separated string of brand:model to be + # used for the image's model assertion + local CUSTOM_BRAND_MODEL=${2:-generic:generic-classic} + + snap_prepare_assertions "$CHROOT_ROOT" "$CUSTOM_BRAND_MODEL" +} + +snap_preseed() { + # Preseed a snap in the image (snap_prepare must be called once prior) + local CHROOT_ROOT=$1 + local SNAP=$2 + local SNAP_NAME=${SNAP%/*} + # Per Ubuntu policy, all seeded snaps (with the exception of the core + # snap) must pull from stable/ubuntu-$(release_ver) as their channel. + local CHANNEL=${3:-"stable/ubuntu-$(release_ver)"} + + if [ ! -e "$CHROOT_ROOT/var/lib/snapd/seed/assertions/model" ]; then + echo "ERROR: Snap model assertion not present, snap_prepare must be called" + exit 1 + fi + + _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL + + # Mark this image as having snapd installed explicitly. + case $SNAP_NAME in + snapd) + touch "$CHROOT_ROOT/var/lib/snapd/seed/.snapd-explicit-install-stamp" + ;; + esac + + # Do basic validation of generated snapd seed.yaml, doing it here + # means we catch all the places(tm) that snaps are added but the + # downside is that each time a snap is added the seed must be valid, + # i.e. snaps with bases need to add bases first etc + # + # Skip validation by setting SNAP_NO_VALIDATE_SEED=1. + if [ -z "${SNAP_NO_VALIDATE_SEED:-}" ]; then + snap_validate_seed "${CHROOT_ROOT}" + fi +} + +snap_validate_seed() { + local CHROOT_ROOT=$1 + + if [ -e "${CHROOT_ROOT}/var/lib/snapd/seed/seed.yaml" ]; then + snap debug validate-seed "${CHROOT_ROOT}/var/lib/snapd/seed/seed.yaml" + fi +} + +snap_from_seed() { + local base_seed=$1 + local out=$2 + local all_snaps + local seeds_expanded + + seeds_expanded=$(inheritance ${base_seed}) + for seed in ${seeds_expanded}; do + echo "snap: considering ${seed}" + file=config/germinate-output/${seed}.snaps + [ -e "${file}" ] || continue + # extract the first column (snap package name) from germinate's output + # translate the human-readable "foo (classic)" into a + # more machine readable "foo/classic" + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") + for snap in ${seed_snaps}; do + echo "snap: found ${snap}" + all_snaps="${all_snaps:+${all_snaps} }${snap}" + done + done + if [ -n "${all_snaps}" ]; then + echo "${all_snaps}" > $out + fi +} + +seed_from_task () +{ + # Retrieve the name of the seed from a task name + local task=$1 + local seed + local seedfile + local seedfiles + + seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)" + if [ -n "$seedfile" ]; then + basename $seedfile .seedtext + return + fi + + seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)" + if [ -n "$seedfiles" ]; then + for seed in $(echo $seedfiles | xargs basename -s .seedtext); do + if [ ${PROJECT}-${seed} = $task ]; then + echo ${seed} + return + fi + done + fi +} + +list_packages_from_seed () { + # Store all packages for a given seed, including its seed dependency + # $1: Name of the seed to expand to a package list + + local all_seeds="$(inheritance $1)" + + for seed in $all_seeds; do + head -n-2 config/germinate-output/${seed}.seed|tail -n+3|awk '{print $1}' + done|sort -u +} + +subtract_package_lists() { + # Subtract a package list from another + # + # $1 source package list + # $2 Package list to subtract from source package list + local list1=$(mktemp) + local list2=$(mktemp) + + list_packages_from_seed $1 > list1 + list_packages_from_seed $2 > list2 + comm -23 list1 list2 + + rm list1 + rm list2 +} + +clean_debian_chroot() { + # remove crufty files that shouldn't be left in an image + rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old + Chroot chroot apt clean + # For the docker images we remove even more stuff. + if [ "${PROJECT}:${SUBPROJECT:-}" = "ubuntu-base:minimized" ]; then + # Remove apt lists (that are currently removed downstream + # anyway) + rm -rf chroot/var/lib/apt/lists/* + # Having device nodes in the docker image can cause problems + # (https://github.com/tianon/docker-brew-ubuntu-core/issues/62) + # so remove them. We only do this for docker out of an + # abundance of caution. + rm -rf chroot/dev/* + fi +} + +configure_universe() { + if [ -f config/universe-enabled ]; then + # This is cargo-culted almost verbatim (with some syntax changes for + # preinstalled being slightly different in what it doesn't ask) from + # debian-installer's apt-setup: + + cat > chroot/etc/apt/sources.list << EOF +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner +# deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner + +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse +EOF + +fi +} + +configure_network_manager() { + # If the image pre-installs network-manager, let it manage all devices by + # default. Installing NM on an existing system only manages wifi and wwan via + # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting + # the global backend to NM, netplan overrides that file. + if [ -e chroot/usr/sbin/NetworkManager -a ! -f chroot/etc/netplan/01-network-manager-all.yaml ]; then + echo "===== Enabling all devices in NetworkManager ====" + mkdir -p chroot/etc/netplan + cat < chroot/etc/netplan/01-network-manager-all.yaml +# Let NetworkManager manage all devices on this system +network: + version: 2 + renderer: NetworkManager +EOF + else + echo "==== NetworkManager not installed ====" + fi +} + +get_parent_pass () { + # return parent pass + # $1 name of the pass + # return parent pass name or '' if pass is root pass. + local pass="$1" + + parent_pass=${pass%.*} + if [ "${parent_pass}" = "${pass}" ]; then + return + fi + echo ${pass%.*} +} + +setenv_file () { + # Exposes an environment variable in a chroot + # $1 Name of the variable + # $2 Value of the variable + # $3 Path to the environment file of the chroot + local var="$1" + local val="$2" + local file="$3" + + grep -v "^$var" $file || true > $file.new + echo "${var}=${val}" >> $file.new + mv $file.new $file +} + +divert_update_initramfs () { + Chroot chroot "dpkg-divert --quiet --add \ + --divert /usr/sbin/update-initramfs.REAL --rename \ + /usr/sbin/update-initramfs" + cat > chroot/usr/sbin/update-initramfs <<'EOF' +#! /bin/sh +if [ $# != 1 ] || [ "$1" != -u ]; then + exec update-initramfs.REAL "$@" +fi +echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2 + +exit 0 +EOF + chmod +x chroot/usr/sbin/update-initramfs +} + +undivert_update_initramfs () { + rm -f chroot/usr/sbin/update-initramfs + Chroot chroot "dpkg-divert --quiet --remove --rename \ + /usr/sbin/update-initramfs" +} + +is_root_layer () { + local pass=$1 + if [ -z "$(get_parent_pass $pass)" ]; then + return 0 + fi + return 1 +} + +is_live_layer () { + local pass=$1 + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue + return 0 + done + return 1 +} + +replace_kernel () { + mountpoint=$1 + new_kernel=$2 + + # Install custom kernel (N.B. the trailing + retains linux-base during this + # operation) + env DEBIAN_FRONTEND=noninteractive chroot "${mountpoint}" apt-get \ + remove --purge --assume-yes '^linux-.*' 'linux-base+' + env DEBIAN_FRONTEND=noninteractive chroot "${mountpoint}" apt-get \ + update --assume-yes + env DEBIAN_FRONTEND=noninteractive chroot "${mountpoint}" apt-get \ + install --assume-yes "${new_kernel}" + env DEBIAN_FRONTEND=noninteractive chroot "${mountpoint}" apt-get \ + autoremove --purge --assume-yes +} diff --git a/config/lb_binary_layered b/config/lb_binary_layered new file mode 100755 index 00000000..2cf56dbb --- /dev/null +++ b/config/lb_binary_layered @@ -0,0 +1,147 @@ +#!/bin/sh + +## live-build(7) - System Build Scripts +## Copyright (C) 2006-2012 Daniel Baumann +## +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING. +## This is free software, and you are welcome to redistribute it +## under certain conditions; see COPYING for details. + + +set -e + +# Including common functions +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh + + +# Automatically populating config tree +if [ -x auto/config ] && [ ! -e .build/config ] +then + Echo_message "Automatically populating config tree." + lb config +fi + +# Setting static variables +DESCRIPTION="$(Echo 'build binary images')" +HELP="" +USAGE="${PROGRAM} [--force]" + +Arguments "${@}" + +# Reading configuration files +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source +Set_defaults + +# Setup cleanup function +Setup_cleanup + +. config/functions + +lb_binary_includes () { + # Copying includes from pass subdirectory + local pass="$1" + + if [ ! -d config/includes.binary.${pass} ]; then + return + fi + + cd config/includes.binary.${pass} + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot + cd "${OLDPWD}" +} + +build_layered_squashfs () { + local pass=$1 + shift 1 # restore ${*} + + Echo_message "lb_binary_layered: treating pass $pass" + + # Building squashfs filesystem & manifest + local overlay_dir="overlay.${pass}" + base="${PWD}/livecd.${PROJECT}.${pass}" + squashfs_f="${base}.squashfs" + + # We have already treated that pass + if [ -f "${squashfs_f}" ]; then + return + fi + + rm -f .build/binary_chroot + + mkdir -p "$overlay_dir/" + lowerdirs=$(get_lowerdirs_for_pass $pass) + if [ -n "$lowerdirs" ]; then + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ + else + ln -s "$overlay_dir/" chroot + fi + + export PASS=${pass} + setenv_file PASS "${pass}" config/environment.chroot + + # Cleanup root filesystem + lb binary_chroot ${*} + + lb_binary_includes $pass ${*} + lb binary_hooks ${*} + + # Copy initrd and vmlinuz outside of chroot and remove them from the layer squashfs + if $(is_live_layer "$pass"); then + lb binary_linux-image ${*} + rm -f chroot/boot/initrd.img-* chroot/boot/vmlinu{x,z}-* + fi + + # Full manifest until that PASS + squashfs_f_manifest="${base}.manifest" + create_manifest "chroot" "${squashfs_f_manifest}.full" + + # Delta manifest + diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest + + squashfs_f_size="${base}.size" + du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}" + + # We take first live pass for "global" ISO properties (used by installers and checkers): + # Prepare initrd + kernel + # Main manifest and size files + prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" + if [ ! -e "${prefix}.manifest" ] && $(is_live_layer "$pass"); then + totalsize=$(cat ${squashfs_f_size}) + curpass="$pass" + while :; do + curpass=$(get_parent_pass $curpass) + # We climbed up the tree to the root layer, we are done + [ -z "$curpass" ] && break + + totalsize=$(expr $totalsize + $(cat "${PWD}/livecd.${PROJECT}.${curpass}.size")) + done + echo ${totalsize} > "${prefix}.size" + + cp "${squashfs_f_manifest}.full" "${prefix}.manifest" + fi + + (cd "overlay.${pass}/" && + mksquashfs . ${squashfs_f} \ + -no-progress -xattrs -comp xz ) + + if [ -n "$lowerdirs" ]; then + umount chroot + rmdir chroot + else + rm chroot + fi +} + +for _PASS in $PASSES +do + build_layered_squashfs "${_PASS}" ${*} +done + +# Ubiquity-compatible removal manifest for ISO not using a layered-aware installer +if [ -n "$(ls livecd.${PROJECT}.*install.live.manifest.full 2>/dev/null)" ] && \ + [ -n "$(ls livecd.${PROJECT}.*install.manifest.full 2>/dev/null)" ]; then + echo "$(diff livecd.${PROJECT}.*install.live.manifest.full livecd.${PROJECT}.*install.manifest.full | awk '/^< / { print $2 }')" > livecd.${PROJECT}-manifest-remove +fi + +chmod 644 *.squashfs *.manifest* *.size diff --git a/config/lb_chroot_layered b/config/lb_chroot_layered new file mode 100755 index 00000000..44429472 --- /dev/null +++ b/config/lb_chroot_layered @@ -0,0 +1,295 @@ +#!/bin/sh + +## live-build(7) - System Build Scripts +## Copyright (C) 2006-2012 Daniel Baumann +## +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING. +## This is free software, and you are welcome to redistribute it +## under certain conditions; see COPYING for details. + +## This is a fork of lb_chroot for layered live system. +## We don't want leaking host configuration in each layer, and so, +## we clean and setup the chroot each time. +## In addition, we create the squashfs for each layer, but top one (live) +## which still can be configured after lb chroot call. + +set -e + +# Including common functions +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh + +# Automatically populating config tree +if [ -x auto/config ] && [ ! -e .build/config ] +then + Echo_message "Automatically populating config tree." + lb config +fi + +# Setting static variables +DESCRIPTION="$(Echo 'customize the Debian system')" +HELP="" +USAGE="${PROGRAM} [--force]" + +Arguments "${@}" + +# Reading configuration files +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source +Set_defaults + +# Setup cleanup function +Setup_cleanup + +. config/functions + +lb_chroot_remove_packages () { + # Remove packages from the chroot specific to this layer + # + # $1: Name of the pass* + local pass=$1 + + Expand_packagelist "$(basename config/package-lists/*.removal-list.chroot_${pass})" "config/package-lists" \ + >> chroot/root/packages.chroot.removal + Chroot chroot "xargs --arg-file=/root/packages.chroot.removal apt-get ${APT_OPTIONS} autoremove --purge" + rm -f chroot/root/packages.chroot.removal +} + +# Create the snap list specific to this layer +lb_chroot_snap_lists () { + local pass=$1 + + # This assumes that the prefix is unique for a given project + local snap_for_pass=$(ls config/package-lists/*.snaplist.chroot_${pass}.full 2>/dev/null || true) + parent_pass=$(get_parent_pass $pass) + local snap_for_parent_pass=$(ls config/package-lists/*.snaplist.chroot_${parent_pass}.full 2>/dev/null || true) + + if [ -z "${snap_for_pass}" ]; then + return + fi + + if [ -z "${snap_for_parent_pass}" ]; then + cp ${snap_for_pass} ${snap_for_pass%.full} + return + fi + + # Generate a list of snaps added to a layer. + diff -NU0 ${snap_for_parent_pass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full} +} + +lb_chroot_install_snaps () { + # Prepare the snap environment and install snaps into a chroot + # + # $1: Name of the pass + + local snaplist_file=$(ls config/package-lists/*.snaplist.chroot_${1} 2>/dev/null || true) + + if [ -z "${snaplist_file}" ]; then + return + fi + + snap_prepare chroot + + while read snap; do + SNAP_NO_VALIDATE_SEED=1 snap_preseed chroot "${snap}" + done < $snaplist_file + snap_validate_seed chroot +} + +lb_chroot_includes () { + # Copying includes from pass subdirectory + local pass="$1" + + if [ ! -d config/includes.chroot.${pass} ]; then + return + fi + + cd config/includes.chroot.${pass} + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot + cd "${OLDPWD}" +} + +reduce_pass_size () { + # Remove duplicated files between parent and current pass + # Note the empty directories created in a child pass are not removed + local pass=$1 + local parent="$(get_parent_pass $pass)" + + $(is_root_layer $pass) && return + + pass_dir="overlay.${pass}" + parent_pass_dir="overlay.${parent}" + + local list_pass=$(mktemp) + local list_parent=$(mktemp) + + (cd $pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_pass) + (cd $parent_pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_parent) + + # Only iterate on common files with same type, owner, permission and size + comm -12 $list_pass $list_parent|cut -d'|' -f1|while read f; do + # If file contents are different, keep it + if ! diff --brief --no-dereference "$pass_dir/$f" "$parent_pass_dir/$f" >/dev/null; then + continue + fi + # Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta. + Echo_message "reduce_pass_size: '$f' is strictly identical between $parent and $pass. Removing." + rm "$pass_dir/$f" + done + + rm $list_pass + rm $list_parent +} + +create_chroot_pass () { + local pass=$1 + shift 1 # restore ${*} + + Echo_message "lb_chroot_layered: treating pass $pass" + + # We have already treated that pass just return. + local overlay_dir="overlay.${pass}" + if [ -d "$overlay_dir/" ]; then + return + fi + + # Only get some function executed on root passes + # Copy bootstrap on root layers + if $(is_root_layer $pass); then + rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks + cp -a chroot.bootstrap/ "$overlay_dir/" + fi + # Others have to be executed on every pass + rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive + + mkdir -p "$overlay_dir/" + lowerdirs=$(get_lowerdirs_for_pass $pass) + if [ -n "$lowerdirs" ]; then + mkdir -p chroot/ + mount_overlay "$lowerdirs" "$overlay_dir" chroot/ + else + ln -s "$overlay_dir/" chroot + fi + + export PASS=${pass} + setenv_file PASS "${pass}" config/environment.chroot + + # Configuring chroot + lb chroot_devpts install ${*} + lb chroot_proc install ${*} + lb chroot_sysfs install ${*} + + # grub-probe should not be called as part of an image build so divert it + divert_grub chroot + + # We run chroot_hacks only on root layers (update-initramfs diverted) + if $(is_root_layer $pass); then + divert_update_initramfs + fi + lb chroot_debianchroot install ${*} + lb chroot_dpkg install ${*} + lb chroot_tmpfs install ${*} + lb chroot_hosts install ${*} + lb chroot_resolv install ${*} + lb chroot_hostname install ${*} + lb chroot_apt install ${*} + # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more + # diff content than desired. So only running this on root pass. + # Only configure universe on root passes + if $(is_root_layer $pass); then + lb chroot_archives chroot install ${*} + configure_universe + fi + + # Customizing chroot + lb chroot_linux-image ${*} + lb chroot_preseed ${*} + lb chroot_early_hooks ${*} + + lb chroot_package-lists ${pass} ${*} + lb chroot_install-packages ${pass} ${*} + lb_chroot_remove_packages ${pass} ${*} + + # Snap management + lb_chroot_snap_lists ${pass} ${*} + lb_chroot_install_snaps ${pass} ${*} + + configure_network_manager + + # Mark kernel headers as autoremovable + Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \ + | awk '/^i/ {print $2}' > chroot.headers + for i in $(cat chroot.headers); do + Chroot chroot "apt-mark auto $i" + done + + Chroot chroot "apt-get --purge -y autoremove" + + # Add live packages to live layers + for livepass in $LIVE_PASSES; do + [ "$livepass" != "$pass" ] && continue + lb chroot_live-packages ${*} + break + done + + # Run includes by pass + lb_chroot_includes ${pass} ${*} + + lb chroot_hooks ${*} + + # Run chroot_hacks only on root layers. + # chroot_hacks changes the mode of boot/initrd*. The side effect in + # layered mode is to create an initrd on each layer with a significant + # impact on image size (+30MB per layer). This is an issue even with + # update-initramfs disabled. + if $(is_live_layer $pass); then + lb chroot_hacks ${*} + fi + + lb chroot_interactive ${*} + + # Misc ubuntu cleanup and post-layer configuration + /usr/share/livecd-rootfs/minimize-manual chroot + clean_debian_chroot + + Chroot chroot "dpkg-query -W" > chroot.packages.${pass} + + # Deconfiguring chroot + if $(is_root_layer $pass); then + lb chroot_archives chroot remove ${*} + fi + lb chroot_apt remove ${*} + lb chroot_hostname remove ${*} + lb chroot_resolv remove ${*} + lb chroot_hosts remove ${*} + lb chroot_tmpfs remove ${*} + lb chroot_dpkg remove ${*} + lb chroot_debianchroot remove ${*} + + # Restore update-initramfs, if necessary + if $(is_root_layer $pass); then + undivert_update_initramfs + fi + + # Restore grub + undivert_grub chroot + + lb chroot_sysfs remove ${*} + lb chroot_proc remove ${*} + lb chroot_devpts remove ${*} + + if [ -n "$lowerdirs" ]; then + umount chroot + rmdir chroot + else + rm chroot + fi + + reduce_pass_size $pass +} + +if [ ! -d chroot.bootstrap/ ]; then + mv chroot/ chroot.bootstrap/ +fi +for _PASS in $PASSES +do + create_chroot_pass "$_PASS" ${*} +done diff --git a/config/package-lists/livecd-rootfs.list.chroot_live b/config/package-lists/livecd-rootfs.list.chroot_live new file mode 100644 index 00000000..cba93bfe --- /dev/null +++ b/config/package-lists/livecd-rootfs.list.chroot_live @@ -0,0 +1 @@ +lupin-casper diff --git a/config/snap-seed-parse b/config/snap-seed-parse new file mode 100755 index 00000000..0e47231c --- /dev/null +++ b/config/snap-seed-parse @@ -0,0 +1,68 @@ +#!/usr/bin/python3 + +""" +Usage: snap-seed-parse [${chroot_dir}] + +This script looks for a seed.yaml path in the given root directory, parsing +it and appending the parsed lines to the given output file. + +The $chroot_dir argument is optional and will default to the empty string. +""" + +import argparse +import os.path +import re +import yaml + + +def log(msg): + print("snap-seed-parse: {}".format(msg)) + + +log("Parsing seed.yaml") + +parser = argparse.ArgumentParser() +parser.add_argument('chroot', nargs='?', default='', + help='root dir for the chroot from which to generate the ' + 'manifest') +parser.add_argument('file', help='Output manifest to this file') + +ARGS = parser.parse_args() +CHROOT_ROOT = ARGS.chroot +FNAME = ARGS.file + +# Trim any trailing slashes for correct appending +log("CHROOT_ROOT: {}".format(CHROOT_ROOT)) +if len(CHROOT_ROOT) > 0 and CHROOT_ROOT[-1] == '/': + CHROOT_ROOT = CHROOT_ROOT[:-1] + +# This is where we expect to find the seed.yaml file +YAML_PATH = CHROOT_ROOT + '/var/lib/snapd/seed/seed.yaml' + +# Snaps are prepended with this string in the manifest +LINE_PREFIX = 'snap:' + +log("yaml path: {}".format(YAML_PATH)) +if not os.path.isfile(YAML_PATH): + log("WARNING: yaml path not found; no seeded snaps found.") + exit(0) +else: + log("yaml path found.") + +with open(YAML_PATH, 'r') as fh: + yaml_lines = yaml.safe_load(fh)['snaps'] + +log('Writing manifest to {}'.format(FNAME)) + +with open(FNAME, 'a+') as fh: + for item in yaml_lines: + filestring = item['file'] + # Pull the revision number off the file name + revision = filestring[filestring.rindex('_')+1:] + revision = re.sub(r'[^0-9]', '', revision) + fh.write("{}{}\t{}\t{}\n".format(LINE_PREFIX, + item['name'], + item['channel'], + revision, + )) +log('Manifest output finished.') diff --git a/debian/changelog b/debian/changelog index 88e725fd..db343575 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +livecd-rootfs (2.667) groovy; urgency=medium + + * Revert of initramfs package removal in KVM image + (LP: #1875400) + + -- Phil Roche Mon, 18 May 2020 17:03:20 +0100 + livecd-rootfs (2.666) groovy; urgency=medium [ Balint Reczey ] diff --git a/live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary b/live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary index 931010bc..094cb375 100755 --- a/live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/kvm-image.binary @@ -49,10 +49,6 @@ replace_kernel ${mount_d} "linux-kvm" chroot "${mount_d}" update-grub undivert_grub "${mount_d}" -# Remove initramfs for kvm image -env DEBIAN_FRONTEND=noninteractive chroot "${mount_d}" apt-get \ - purge -y initramfs-tools busybox-initramfs - env DEBIAN_FRONTEND=noninteractive chroot "${mount_d}" rm \ -rf /boot/initrd.img-* /boot/initrd.img