# vi: ts=4 expandtab syntax=sh #imagesize=${IMAGE_SIZE:-$((2252*1024**2))} # 2.2G (the current size we ship) imagesize=${IMAGE_SIZE:-2361393152} # 2.2G (the current size we ship) fs_label="${FS_LABEL:-rootfs}" rootfs_dev_mapper= loop_device= loop_raw= backing_img= clean_loops() { local kpartx_ret local kpartx_stdout if [ -n "${backing_img}" ]; then # sync before removing loop to avoid "Device or resource busy" errors sync kpartx_ret="" kpartx_stdout=$(kpartx -v -d "${backing_img}") || kpartx_ret=$? echo "$kpartx_stdout" if [ -n "$kpartx_ret" ]; then if echo "$kpartx_stdout" | grep -q "loop deleted"; then echo "Suppressing kpartx returning error (#860894)" else exit $kpartx_ret fi fi unset backing_img fi if [ -z "${rootfs_dev_mapper}" ]; then return 0 fi unset loop_device unset loop_raw unset rootfs_dev_mapper } create_empty_disk_image() { # Prepare an empty disk image dd if=/dev/zero of="$1" bs=1 count=0 seek="${imagesize}" } create_manifest() { local chroot_root=${1} local target_file=${2} echo "create_manifest chroot_root: ${chroot_root}" dpkg-query --show --admindir="${chroot_root}/var/lib/dpkg" > ${target_file} echo "create_manifest call to dpkg-query finished." ./config/snap-seed-parse "${chroot_root}" "${target_file}" echo "create_manifest call to snap_seed_parse finished." echo "create_manifest finished" } make_ext4_partition() { device="$1" label=${fs_label:+-L "${fs_label}"} mkfs.ext4 -F -b 4096 -i 8192 -m 0 ${label} -E resize=536870912 "$device" } mount_image() { trap clean_loops EXIT backing_img="$1" local rootpart="$2" kpartx_mapping="$(kpartx -s -v -a ${backing_img})" # Find the loop device loop_p1="$(echo -e ${kpartx_mapping} | head -n1 | awk '{print$3}')" loop_device="/dev/${loop_p1%p[0-9]*}" if [ ! -b ${loop_device} ]; then echo "unable to find loop device for ${backing_img}" exit 1 fi # Find the rootfs location rootfs_dev_mapper="/dev/mapper/${loop_p1%%[0-9]}${rootpart}" if [ ! -b "${rootfs_dev_mapper}" ]; then echo "${rootfs_dev_mapper} is not a block device"; exit 1 fi # Add some information to the debug logs echo "Mounted disk image ${backing_img} to ${rootfs_dev_mapper}" blkid ${rootfs_dev_mapper} return 0 } setup_mountpoint() { local mountpoint="$1" mount --rbind --make-rslave /dev "$mountpoint/dev" mount proc-live -t proc "$mountpoint/proc" mount sysfs-live -t sysfs "$mountpoint/sys" mount -t tmpfs none "$mountpoint/tmp" mount -t tmpfs none "$mountpoint/var/lib/apt" mount -t tmpfs none "$mountpoint/var/cache/apt" mv "$mountpoint/etc/resolv.conf" resolv.conf.tmp cp /etc/resolv.conf "$mountpoint/etc/resolv.conf" chroot "$mountpoint" apt-get update } teardown_mountpoint() { # Reverse the operations from setup_mountpoint local mountpoint="$1" # ensure we have exactly one trailing slash, and escape all slashes for awk mountpoint_match=$(echo "$mountpoint" | sed -e's,/$,,; s,/,\\/,g;')'\/' # sort -r ensures that deeper mountpoints are unmounted first for submount in $(awk $mountpoint/usr/sbin/policy-rc.d << EOF #!/bin/sh # ${IMAGE_STR} echo "All runlevel operations denied by policy" >&2 exit 101 EOF chmod 0755 $mountpoint/usr/sbin/policy-rc.d } umount_partition() { local mountpoint=${1} teardown_mountpoint $mountpoint umount -R $mountpoint udevadm settle if [ -n "${rootfs_dev_mapper}" -a -b "${rootfs_dev_mapper}" ]; then # buildd's don't have /etc/mtab symlinked # /etc/mtab is needed in order zerofree space for ext4 filesystems [ -e /etc/mtab ] || ln -s /proc/mounts /etc/mtab # both of these are likely overkill, but it does result in slightly # smaller ext4 filesystem e2fsck -y -E discard ${rootfs_dev_mapper} zerofree ${rootfs_dev_mapper} fi } umount_disk_image() { mountpoint="$1" local uefi_dev="/dev/mapper${loop_device///dev/}p15" if [ -e "$mountpoint/boot/efi" -a -b "$uefi_dev" ]; then # zero fill free space in UEFI partition cat < /dev/zero > "$mountpoint/boot/efi/bloat_file" 2> /dev/null || true rm "$mountpoint/boot/efi/bloat_file" umount --detach-loop "$mountpoint/boot/efi" fi if [ -e $mountpoint/usr/sbin/policy-rc.d ]; then rm $mountpoint/usr/sbin/policy-rc.d fi umount_partition $mountpoint clean_loops } modify_vmdk_header() { # Modify the VMDK headers so that both VirtualBox _and_ VMware can # read the vmdk and import them. vmdk_name="${1}" descriptor=$(mktemp) newdescriptor=$(mktemp) # Extract the vmdk header for manipulation dd if="${vmdk_name}" of="${descriptor}" bs=1 skip=512 count=1024 # The sed lines below is where the magic is. Specifically: # ddb.toolsVersion: sets the open-vm-tools so that VMware shows # the tooling as current # ddb.virtualHWVersion: set the version to 7, which covers most # current versions of VMware # createType: make sure its set to stream Optimized # remove the vmdk-stream-converter comment and replace with # # Disk DescriptorFile. This is needed for Virtualbox # remove the comments from vmdk-stream-converter which causes # VirtualBox and others to fail VMDK validation sed -e 's|# Description file.*|# Disk DescriptorFile|' \ -e '/# Believe this is random*/d' \ -e '/# Indicates no parent/d' \ -e '/# The Disk Data Base/d' \ -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ "${descriptor}" > "${newdescriptor}" # The header is cannot be bigger than 1024 expr $(stat --format=%s ${newdescriptor}) \< 1024 > /dev/null 2>&1 || { echo "descriptor is too large, VMDK will be invalid!"; exit 1; } # Overwrite the vmdk header with our new, modified one dd conv=notrunc,nocreat \ if="${newdescriptor}" of="${vmdk_name}" \ bs=1 seek=512 count=1024 rm ${descriptor} ${newdescriptor} } create_vmdk() { # There is no real good way to create a _compressed_ VMDK using open source # tooling that works across multiple VMDK-capable platforms. This functions # uses vmdk-stream-converter and then calls modify_vmdk_header to produce a # compatible VMDK. src="$1" destination="$2" size="${3:-10240}" streamconverter="VMDKstream" scratch_d=$(mktemp -d) cp ${src} ${scratch_d}/resize.img truncate --size=${size}M ${scratch_d}/resize.img python -m ${streamconverter} ${scratch_d}/resize.img ${destination} modify_vmdk_header ${destination} qemu-img info ${destination} rm -rf ${scratch_d} } create_derivative() { # arg1 is the disk type # arg2 is the new name unset derivative_img case ${1} in uefi) disk_image="binary/boot/disk-uefi.ext4"; dname="${disk_image//-uefi/-$2-uefi}";; *) disk_image="binary/boot/disk.ext4"; dname="${disk_image//.ext4/-$2.ext4}";; esac if [ ! -e ${disk_image} ]; then echo "Did not find ${disk_image}!"; exit 1; fi cp ${disk_image} ${dname} export derivative_img=${dname} } convert_to_qcow2() { src="$1" destination="$2" qemu-img convert -c -O qcow2 -o compat=0.10 "$src" "$destination" qemu-img info "$destination" } replace_grub_root_with_label() { # When update-grub is run, it will detect the disks in the build system. # Instead, we want grub to use the right labelled disk CHROOT_ROOT="$1" # If boot by partuuid has been requested, don't override. if [ -f $CHROOT_ROOT/etc/default/grub.d/40-force-partuuid.cfg ] && \ grep -q ^GRUB_FORCE_PARTUUID= $CHROOT_ROOT/etc/default/grub.d/40-force-partuuid.cfg then return 0 fi sed -i -e "s,root=[^ ]*,root=LABEL=${fs_label}," \ "$CHROOT_ROOT/boot/grub/grub.cfg" } # When running update-grub in a chroot on a build host, we don't want it to # probe for disks or probe for other installed OSes. Extract common # diversion wrappers, so this isn't reinvented differently for each image. divert_grub() { CHROOT_ROOT="$1" # Don't divert all of grub-probe here; just the scripts we don't want # running. Otherwise, you may be missing part-uuids for the search # command, for example. ~cyphermox chroot "$CHROOT_ROOT" dpkg-divert --local \ --divert /etc/grub.d/30_os-prober.dpkg-divert \ --rename /etc/grub.d/30_os-prober # Divert systemd-detect-virt; /etc/kernel/postinst.d/zz-update-grub # no-ops if we are in a container, and the launchpad farm runs builds # in lxd. We therefore pretend that we're never in a container (by # exiting 1). chroot "$CHROOT_ROOT" dpkg-divert --local \ --rename /usr/bin/systemd-detect-virt echo "exit 1" > "$CHROOT_ROOT"/usr/bin/systemd-detect-virt chmod +x "$CHROOT_ROOT"/usr/bin/systemd-detect-virt } undivert_grub() { CHROOT_ROOT="$1" chroot "$CHROOT_ROOT" dpkg-divert --remove --local \ --divert /etc/grub.d/30_os-prober.dpkg-divert \ --rename /etc/grub.d/30_os-prober rm "$CHROOT_ROOT"/usr/bin/systemd-detect-virt chroot "$CHROOT_ROOT" dpkg-divert --remove --local \ --rename /usr/bin/systemd-detect-virt } recreate_initramfs() { # Regenerate the initramfs by running update-initramfs in the # chroot at $1 and copying the generated initramfs # around. Beware that this was written for a single use case # (live-server) and may not work in all cases without # tweaking... # config/common must be sourced before calling this function. CHROOT="$1" # Start by cargo culting bits of lb_chroot_hacks: if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$CHROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf fi chroot "$CHROOT" sh -c "${UPDATE_INITRAMFS_OPTIONS:-} update-initramfs -k all -t -u" rm -rf "$CHROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf # Then bits of lb_binary_linux-image: case "${LB_INITRAMFS}" in casper) DESTDIR="binary/casper" ;; live-boot) DESTDIR="binary/live" ;; *) DESTDIR="binary/boot" ;; esac mv "$CHROOT"/boot/initrd.img-* $DESTDIR } release_ver() { # Return the release version number distro-info --series="$LB_DISTRIBUTION" -r | awk '{ print $1 }' } # cribbed from cdimage, perhaps this should be a small helper script in germinate? add_inheritance () { case " $inherit " in *" $1 "*) ;; *) inherit="${inherit:+$inherit }$1" ;; esac } expand_inheritance () { for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do expand_inheritance "$seed" done add_inheritance "$1" } inheritance () { inherit= expand_inheritance "$1" echo "$inherit" } _snap_preseed() { # Download the snap/assertion and add to the preseed local CHROOT_ROOT=$1 local SNAP=$2 local SNAP_NAME=${SNAP%/*} local CHANNEL=${3:?Snap channel must be specified} local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" local snaps_dir="$seed_dir/snaps" local seed_yaml="$seed_dir/seed.yaml" local assertions_dir="$seed_dir/assertions" # Download the snap & assertion local snap_download_failed=0 # Preseed a snap only once if [ -f ${snaps_dir}/${SNAP_NAME}_[0-9]*.snap ]; then return fi sh -c " set -x; cd \"$CHROOT_ROOT/var/lib/snapd/seed\"; SNAPPY_STORE_NO_CDN=1 /usr/share/livecd-rootfs/snap-tool download \ --cohort-key=\"${COHORT_KEY:-}\" \ --channel=\"$CHANNEL\" \"$SNAP_NAME\"" || snap_download_failed=1 if [ $snap_download_failed = 1 ] ; then echo "If the channel ($CHANNEL) includes '*/ubuntu-##.##' track per " echo "Ubuntu policy (ex. stable/ubuntu-18.04) the publisher will need " echo "to temporarily create the channel/track to allow fallback during" echo "download (ex. stable/ubuntu-18.04 falls back to stable if the" echo "prior had been created in the past)." exit 1 fi mv -v $seed_dir/*.assert $assertions_dir mv -v $seed_dir/*.snap $snaps_dir # Add the snap to the seed.yaml ! [ -e $seed_yaml ] && echo "snaps:" > $seed_yaml cat <> $seed_yaml - name: ${SNAP_NAME} channel: ${CHANNEL} EOF case ${SNAP} in */classic) echo " classic: true" >> $seed_yaml;; esac echo -n " file: " >> $seed_yaml (cd $snaps_dir; ls -1 ${SNAP_NAME}_*.snap) >> $seed_yaml } snap_prepare_assertions() { # Configure basic snapd assertions local CHROOT_ROOT=$1 # A colon-separated string of brand:model to be used for the image's model # assertion local CUSTOM_BRAND_MODEL=$2 local seed_dir="$CHROOT_ROOT/var/lib/snapd/seed" local snaps_dir="$seed_dir/snaps" local assertions_dir="$seed_dir/assertions" local model_assertion="$assertions_dir/model" local account_key_assertion="$assertions_dir/account-key" local account_assertion="$assertions_dir/account" if [ -d "$assertions_dir" ]; then return fi mkdir -p "$assertions_dir" mkdir -p "$snaps_dir" local brand="$(echo $CUSTOM_BRAND_MODEL | cut -d: -f 1)" local model="$(echo $CUSTOM_BRAND_MODEL | cut -d: -f 2)" # Clear the assertions if they already exist if [ -e "$model_assertion" ] ; then existing_model=$(awk '/^model: / {print $2}' $model_assertion) existing_brand=$(awk '/^brand-id: / {print $2}' $model_assertion) echo "snap_prepare_assertions: replacing $existing_brand:$existing_model with $brand:$model" rm "$model_assertion" rm "$account_key_assertion" rm "$account_assertion" fi if ! [ -e "$model_assertion" ] ; then snap known --remote model series=16 \ model=$model brand-id=$brand \ > "$model_assertion" fi if ! [ -e "$account_key_assertion" ] ; then local account_key=$(sed -n -e's/sign-key-sha3-384: //p' \ < "$model_assertion") snap known --remote account-key \ public-key-sha3-384="$account_key" \ > "$account_key_assertion" fi if ! [ -e "$account_assertion" ] ; then local account=$(sed -n -e's/account-id: //p' < "$account_key_assertion") snap known --remote account account-id=$account \ > "$account_assertion" fi } snap_prepare() { # Configure basic snapd assertions and pre-seeds the 'core' snap local CHROOT_ROOT=$1 # Optional. If set, should be a colon-separated string of brand:model to be # used for the image's model assertion local CUSTOM_BRAND_MODEL=${2:-generic:generic-classic} snap_prepare_assertions "$CHROOT_ROOT" "$CUSTOM_BRAND_MODEL" # Download the core snap _snap_preseed $CHROOT_ROOT core stable } snap_preseed() { # Preseed a snap in the image (snap_prepare must be called once prior) local CHROOT_ROOT=$1 local SNAP=$2 # Per Ubuntu policy, all seeded snaps (with the exception of the core # snap) must pull from stable/ubuntu-$(release_ver) as their channel. local CHANNEL=${3:-"stable/ubuntu-$(release_ver)"} if [ ! -e "$CHROOT_ROOT/var/lib/snapd/seed/assertions/model" ]; then echo "ERROR: Snap model assertion not present, snap_prepare must be called" exit 1 fi _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL } snap_from_seed() { local base_seed=$1 local out=$2 local all_snaps local seeds_expanded seeds_expanded=$(inheritance ${base_seed}) for seed in ${seeds_expanded}; do echo "snap: considering ${seed}" file=config/germinate-output/${seed}.snaps [ -e "${file}" ] || continue # extract the first column (snap package name) from germinate's output # translate the human-readable "foo (classic)" into a # more machine readable "foo/classic" seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") for snap in ${seed_snaps}; do echo "snap: found ${snap}" all_snaps="${all_snaps:+${all_snaps} }${snap}" done done if [ -n "${all_snaps}" ]; then echo "${all_snaps}" > $out fi } seed_from_task () { # Retrieve the name of the seed from a task name local task=$1 local seed local seedfile local seedfiles seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)" if [ -n "$seedfile" ]; then basename $seedfile .seedtext return fi seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)" if [ -n "$seedfiles" ]; then for seed in $(echo $seedfiles | xargs basename -s .seedtext); do if [ ${PROJECT}-${seed} = $task ]; then echo ${seed} return fi done fi } list_packages_from_seed () { # Store all packages for a given seed, including its seed dependency # $1: Name of the seed to expand to a package list local all_seeds="$(inheritance $1)" for seed in $all_seeds; do head -n-2 config/germinate-output/${seed}.seed|tail -n+3|awk '{print $1}' done|sort -u } subtract_package_lists() { # Subtract a package list from another # # $1 source package list # $2 Package list to subtract from source package list local list1=$(mktemp) local list2=$(mktemp) list_packages_from_seed $1 > list1 list_packages_from_seed $2 > list2 comm -23 list1 list2 rm list1 rm list2 } clean_debian_chroot() { # remove crufty files that shouldn't be left in an image rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old Chroot chroot apt clean } configure_universe() { if [ -f config/universe-enabled ]; then # This is cargo-culted almost verbatim (with some syntax changes for # preinstalled being slightly different in what it doesn't ask) from # debian-installer's apt-setup: cat > chroot/etc/apt/sources.list << EOF # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted ## Major bug fix updates produced after the final release of the ## distribution. deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted ## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu ## team. Also, please note that software in universe WILL NOT receive any ## review or updates from the Ubuntu security team. deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe ## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu ## team, and may not be under a free licence. Please satisfy yourself as to ## your rights to use the software. Also, please note that software in ## multiverse WILL NOT receive any review or updates from the Ubuntu ## security team. deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse ## N.B. software from this repository may not have been tested as ## extensively as that contained in the main release, although it includes ## newer versions of some applications which may provide useful features. ## Also, please note that software in backports WILL NOT receive any review ## or updates from the Ubuntu security team. deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse # deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse ## Uncomment the following two lines to add software from Canonical's ## 'partner' repository. ## This software is not part of Ubuntu, but is offered by Canonical and the ## respective vendors as a service to Ubuntu users. # deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner # deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted # deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe # deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse # deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse EOF fi } configure_network_manager() { # If the image pre-installs network-manager, let it manage all devices by # default. Installing NM on an existing system only manages wifi and wwan via # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting # the global backend to NM, netplan overrides that file. if [ -e chroot/usr/sbin/NetworkManager -a ! -f chroot/etc/netplan/01-network-manager-all.yaml ]; then echo "===== Enabling all devices in NetworkManager ====" mkdir -p chroot/etc/netplan cat < chroot/etc/netplan/01-network-manager-all.yaml # Let NetworkManager manage all devices on this system network: version: 2 renderer: NetworkManager EOF else echo "==== NetworkManager not installed ====" fi } get_parent_pass () { # return parent pass # $1 name of the pass # return parent pass name or '' if pass is root pass. local pass="$1" parent_pass=${pass%.*} if [ "${parent_pass}" = "${pass}" ]; then return fi echo ${pass%.*} } setenv_file () { # Exposes an environment variable in a chroot # $1 Name of the variable # $2 Value of the variable # $3 Path to the environment file of the chroot local var="$1" local val="$2" local file="$3" grep -v "^$var" $file || true > $file.new echo "${var}=${val}" >> $file.new mv $file.new $file } divert_update_initramfs () { Chroot chroot "dpkg-divert --quiet --add \ --divert /usr/sbin/update-initramfs.REAL --rename \ /usr/sbin/update-initramfs" cat > chroot/usr/sbin/update-initramfs <<'EOF' #! /bin/sh if [ $# != 1 ] || [ "$1" != -u ]; then exec update-initramfs.REAL "$@" fi echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2 exit 0 EOF chmod +x chroot/usr/sbin/update-initramfs cat > config/hooks/999-undivert-update-initramfs.chroot <<'EOF' #! /bin/sh [ ! -f /usr/sbin/update-initramfs.REAL ] && exit 0 rm -f /usr/sbin/update-initramfs dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs EOF } is_root_layer () { local pass=$1 if [ -z "$(get_parent_pass $pass)" ]; then return 0 fi return 1 } is_live_layer () { local pass=$1 for livepass in $LIVE_PASSES; do [ "$livepass" != "$pass" ] && continue return 0 done return 1 }