From 07873ec5285348165fd9f494cd7e09d7b12f4481 Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Thu, 4 Jan 2018 12:11:46 +0000 Subject: [PATCH 1/8] Support seeding of snaps We run germinate during configure, and read from each resulting .snaps file (expanded from STRUCTURE) to find out which snaps to seed. This is written to config/seeded-snaps, which is read during build to download each one. --- live-build/auto/build | 65 ++++++++++++------------- live-build/auto/config | 106 ++++++++++++++++++++++++----------------- 2 files changed, 93 insertions(+), 78 deletions(-) diff --git a/live-build/auto/build b/live-build/auto/build index f9d05e2c..31f4b8a7 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -42,7 +42,7 @@ preinstall_snaps() { chroot chroot sh -c " set -x; cd /var/lib/snapd/seed; - SNAPPY_STORE_NO_CDN=1 snap download \"$snap\"" + SNAPPY_STORE_NO_CDN=1 snap download \"${snap%/*}\"" done lb chroot_resolv remove @@ -55,12 +55,14 @@ snaps: file: ${CORE_SNAP} EOF for snap in "$@"; do + snap_name=${snap%/*} cat <> chroot/var/lib/snapd/seed/seed.yaml - - name: $snap + - name: ${snap_name} channel: stable EOF + case ${snap} in */classic) echo " classic: true" >> chroot/var/lib/snapd/seed/seed.yaml;; esac echo -n " file: " >> chroot/var/lib/snapd/seed/seed.yaml - (cd chroot/var/lib/snapd/seed; ls -1 ${snap}_*.snap) \ + (cd chroot/var/lib/snapd/seed; ls -1 ${snap_name}_*.snap) \ >> chroot/var/lib/snapd/seed/seed.yaml done @@ -367,40 +369,33 @@ deb file:/var/lib/preinstalled-pool/ $LB_DISTRIBUTION $LB_PARENT_ARCHIVE_AREAS rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig fi case $PROJECT:$SUBPROJECT in - ubuntu-server:live|ubuntu-mate:*) - assertions_dir="chroot/var/lib/snapd/seed/assertions" - model_assertion="$assertions_dir/generic-classic.model" - account_key_assertion="$assertions_dir/generic.account-key" - account_assertion="$assertions_dir/generic.account" - - mkdir -p "$assertions_dir" - snap known --remote model series=16 \ - model=generic-classic brand-id=generic \ - > "$model_assertion" - account_key=$(sed -n -e's/sign-key-sha3-384: //p' \ - < "$model_assertion") - - snap known --remote account-key \ - public-key-sha3-384="$account_key" \ - > "$account_key_assertion" - account=$(sed -n -e's/account-id: //p' \ - < "$account_key_assertion") - - snap known --remote account account-id=generic \ - > "$account_assertion" - ;; - esac - - # FIXME: this should not be a hard-coded list in this script; this - # is an interim solution for 17.10. - case $PROJECT:$SUBPROJECT in - ubuntu-mate:*) - SNAP_LIST=pulsemixer + *) + if [ -e "config/seeded-snaps" ]; then + assertions_dir="chroot/var/lib/snapd/seed/assertions" + model_assertion="$assertions_dir/generic-classic.model" + account_key_assertion="$assertions_dir/generic.account-key" + account_assertion="$assertions_dir/generic.account" + + mkdir -p "$assertions_dir" + snap known --remote model series=16 \ + model=generic-classic brand-id=generic \ + > "$model_assertion" + account_key=$(sed -n -e's/sign-key-sha3-384: //p' \ + < "$model_assertion") + + snap known --remote account-key \ + public-key-sha3-384="$account_key" \ + > "$account_key_assertion" + account=$(sed -n -e's/account-id: //p' \ + < "$account_key_assertion") + + snap known --remote account account-id=generic \ + > "$account_assertion" + snap_list=$(cat config/seeded-snaps) + preinstall_snaps $snap_list + fi ;; esac - if [ -n "$SNAP_LIST" ]; then - preinstall_snaps $SNAP_LIST - fi if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then if [ "$ARCH" = "armhf" ]; then diff --git a/live-build/auto/config b/live-build/auto/config index 8c749ec6..f6ceea42 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -254,13 +254,76 @@ if [ "${SUBPROJECT:-}" = minimized ]; then OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal --linux-packages=linux-image" fi +# cribbed from cdimage, perhaps this should be a small helper script in germinate? +add_inheritance () { + case " $inherit " in + *" $1 "*) + ;; + *) + inherit="${inherit:+$inherit }$1" + ;; + esac +} + +expand_inheritance () { + for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do + expand_inheritance "$seed" + done + add_inheritance "$1" +} + +inheritance () { + inherit= + expand_inheritance "$1" + echo "$inherit" +} + +mkdir -p config/germinate-output +case $PROJECT in + kubuntu-active*) + SEED=kubuntu-active.$SUITE + ;; + kubuntu*) + SEED=kubuntu.$SUITE + ;; + xubuntu*) + SEED=xubuntu.$SUITE + ;; + *) + SEED=ubuntu.$SUITE + ;; +esac + +if ! [ -e config/germinate-output/structure ]; then + echo "Running germinate..." + (cd config/germinate-output && germinate --no-rdepends --no-installer \ + -S $SEEDMIRROR -m $MIRROR -d $SUITE -s $SEED \ + ${COMPONENTS:+-c "$COMPONENTS"} -a $ARCH) +fi + case $PROJECT in ubuntu|ubuntu-dvd) add_task install minimal standard ubuntu-desktop LIVE_TASK='ubuntu-live' + BASE_SEED='desktop' + SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) case $ARCH in amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; esac + for seed in ${SEEDS_EXPANDED}; do + echo "snap: considering ${seed}" + file=config/germinate-output/${seed}.snaps + [ -e "${file}" ] || continue + # extract the first column (snap package name) from germinate's output + # translate the human-readable "foo (classic)" into a + # more machine readable "foo/classic" + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") + for snap in ${seed_snaps}; do + echo "snap: found ${snap}" + ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" + done + done + echo "${ALL_SNAPS}" > config/seeded-snaps ;; ubuntu-desktop-next) @@ -933,51 +996,8 @@ case ${SUBPROJECT:-} in ;; esac -# cribbed from cdimage, perhaps this should be a small helper script in germinate? -add_inheritance () { - case " $inherit " in - *" $1 "*) - ;; - *) - inherit="${inherit:+$inherit }$1" - ;; - esac -} - -expand_inheritance () { - for seed in $(grep "^$1:" config/germinate-output/structure | cut -d: -f2); do - expand_inheritance "$seed" - done - add_inheritance "$1" -} - -inheritance () { - inherit= - expand_inheritance "$1" - echo "$inherit" -} - if [ "$PREINSTALLED" = "true" ]; then if [ -n "$PREINSTALL_POOL_SEEDS" ]; then - mkdir -p config/germinate-output - case $PROJECT in - kubuntu-active*) - SEED=kubuntu-active.$SUITE - ;; - kubuntu*) - SEED=kubuntu.$SUITE - ;; - xubuntu*) - SEED=xubuntu.$SUITE - ;; - *) - SEED=ubuntu.$SUITE - ;; - esac - (cd config/germinate-output && germinate --no-rdepends --no-installer \ - -S $SEEDMIRROR -m $MIRROR -d $SUITE -s $SEED \ - ${COMPONENTS:+-c "$COMPONENTS"} -a $ARCH) - UNWANTED_SEEDS="${LIVE_TASK:+$LIVE_TASK }boot installer required" for i in $UNWANTED_SEEDS; do UNWANTED_SEEDS="${UNWANTED_SEEDS:+$UNWANTED_SEEDS }$(inheritance $i)" From d41cc70bf54b84e4c298cd883d3eeeaabfa114b5 Mon Sep 17 00:00:00 2001 From: Robert C Jennings Date: Fri, 19 Jan 2018 12:26:32 -0600 Subject: [PATCH 2/8] ubuntu-cpc: Initramfs compression should match runtime default (LP: #1744114) Cloud images created from the 'ubuntu-cpc' project use lzma initramfs compression for the initial initramfs shipped in the image; this is the default for livecd-rootfs. When a user updates the initramfs (or installs a new kernel) on a running instance created from a cloud image they will get gzip compression, which is the default shipped in initramfs-tools. With this patch the compression method used to create the initramfs in the initial cloud images will match the compression menthod used when the end-user creates/updates an initramfs, namely gzip compression. --- debian/changelog | 7 +++++++ live-build/auto/config | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 843794e2..fb46ac9c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +livecd-rootfs (2.490) bionic; urgency=medium + + * ubuntu-cpc: Initramfs compression should match default for + update-initramfs (LP: #1744114) + + -- Robert C Jennings Thu, 18 Jan 2018 10:16:37 -0600 + livecd-rootfs (2.489) bionic; urgency=medium [ Balint Reczey ] diff --git a/live-build/auto/config b/live-build/auto/config index 9abaaa65..15788881 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -487,7 +487,8 @@ case $PROJECT in add_package install shim-signed ;; esac - OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none --initramfs-compression=none" + OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none" + INITRAMFS_COMPRESSION="none" # contains the framework definition add_package install ubuntu-core-libs @@ -583,6 +584,7 @@ case $PROJECT in OPTS="${OPTS:+$OPTS }--hdd-label=cloudimg-rootfs" OPTS="${OPTS:+$OPTS }--ext-resize-blocks=536870912 --ext-block-size=4096" OPTS="${OPTS:+$OPTS }--ext-fudge-factor=15" + INITRAMFS_COMPRESSION="gzip" ;; *) @@ -696,7 +698,7 @@ lb config noauto \ ${KERNEL_FLAVOURS:+--linux-flavours "$KERNEL_FLAVOURS"} \ --initsystem none \ --bootloader "$BOOTLOADER" \ - --initramfs-compression lzma \ + --initramfs-compression "${INITRAMFS_COMPRESSION:-lzma}" \ --cache false \ ${BOOTAPPEND_LIVE:+--bootappend-live "$BOOTAPPEND_LIVE"} \ $OPTS \ From 019d1668782d3209cd9351de97befa590b835694 Mon Sep 17 00:00:00 2001 From: Mathieu Trudel-Lapierre Date: Mon, 29 Jan 2018 12:15:27 -0500 Subject: [PATCH 3/8] releasing package livecd-rootfs version 2.490 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index dbac59c8..badd9dea 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ -livecd-rootfs (2.490) UNRELEASED; urgency=medium +livecd-rootfs (2.490) bionic; urgency=medium * ubuntu-cpc: Initramfs compression should match default for update-initramfs (LP: #1744114) - -- Robert C Jennings Thu, 18 Jan 2018 10:16:37 -0600 + -- Robert C Jennings Mon, 29 Jan 2018 12:15:17 -0500 livecd-rootfs (2.489) bionic; urgency=medium From 26612d0b266a20efa99abbe0dca5377fd7684d3b Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Tue, 30 Jan 2018 09:44:25 +0000 Subject: [PATCH 4/8] snap: move the seeding block down, so that it is common to all flavours --- live-build/auto/config | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index f6ceea42..053e31fd 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -306,24 +306,9 @@ case $PROJECT in add_task install minimal standard ubuntu-desktop LIVE_TASK='ubuntu-live' BASE_SEED='desktop' - SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) case $ARCH in amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; esac - for seed in ${SEEDS_EXPANDED}; do - echo "snap: considering ${seed}" - file=config/germinate-output/${seed}.snaps - [ -e "${file}" ] || continue - # extract the first column (snap package name) from germinate's output - # translate the human-readable "foo (classic)" into a - # more machine readable "foo/classic" - seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") - for snap in ${seed_snaps}; do - echo "snap: found ${snap}" - ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" - done - done - echo "${ALL_SNAPS}" > config/seeded-snaps ;; ubuntu-desktop-next) @@ -654,6 +639,24 @@ case $PROJECT in ;; esac +if [ -n "${BASE_SEED}" ]; then + SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) + for seed in ${SEEDS_EXPANDED}; do + echo "snap: considering ${seed}" + file=config/germinate-output/${seed}.snaps + [ -e "${file}" ] || continue + # extract the first column (snap package name) from germinate's output + # translate the human-readable "foo (classic)" into a + # more machine readable "foo/classic" + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") + for snap in ${seed_snaps}; do + echo "snap: found ${snap}" + ALL_SNAPS="${ALL_SNAPS:+${ALL_SNAPS} }${snap}" + done + done + echo "${ALL_SNAPS}" > config/seeded-snaps +fi + export APT_OPTIONS if [ "$PREINSTALLED" != "true" ] && [ "$LIVE_TASK" ]; then From 77de6a289f34c9a35b9e5b23a7637a5d46dbe676 Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Tue, 30 Jan 2018 09:51:32 +0000 Subject: [PATCH 5/8] snap: Add BASE_SEED for all desktop flavours --- live-build/auto/config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/live-build/auto/config b/live-build/auto/config index 053e31fd..0767479a 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -305,7 +305,6 @@ case $PROJECT in ubuntu|ubuntu-dvd) add_task install minimal standard ubuntu-desktop LIVE_TASK='ubuntu-live' - BASE_SEED='desktop' case $ARCH in amd64) add_package live $SIGNED_KERNEL_PACKAGE ;; esac @@ -639,6 +638,14 @@ case $PROJECT in ;; esac +# we'll expand the base seed given here according to the STRUCTURE file, and +# then look in all of the seeds found to see which snaps are seeded +case $PROJECT in + ubuntu|kubuntu*|lubuntu*|xubuntu*|ubuntu-mate*|ubuntustudio*|ubuntukylin*|ubuntu-budgie*) + BASE_SEED='desktop' + ;; +esac + if [ -n "${BASE_SEED}" ]; then SEEDS_EXPANDED=$(inheritance ${BASE_SEED}) for seed in ${SEEDS_EXPANDED}; do From 9bd0fd192f401b471232c16b1cf2d52b16cc890d Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Tue, 30 Jan 2018 10:49:38 +0000 Subject: [PATCH 6/8] Specify seeds for all flavours too --- live-build/auto/config | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/live-build/auto/config b/live-build/auto/config index 5e24832f..bbe61a26 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -289,6 +289,21 @@ case $PROJECT in xubuntu*) SEED=xubuntu.$SUITE ;; + ubuntu-mate*) + SEED=ubuntu-mate.$SUITE + ;; + lubuntu*) + SEED=lubuntu.$SUITE + ;; + ubuntu-budgie*) + SEED=ubuntu-budgie.$SUITE + ;; + ubuntukylin*) + SEED=ubuntukylin.$SUITE + ;; + ubuntustudio*) + SEED=ubuntustudio.$SUITE + ;; *) SEED=ubuntu.$SUITE ;; From 284240feb5c0445062db8cf4d956daeba6a0e160 Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Tue, 30 Jan 2018 11:20:34 +0000 Subject: [PATCH 7/8] releasing package livecd-rootfs version 2.491 --- debian/changelog | 4 ++-- live-build/auto/config | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index b8dc1a64..162f1157 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ -livecd-rootfs (2.491) UNRELEASED; urgency=medium +livecd-rootfs (2.491) bionic; urgency=medium * Support seeding of snaps, as emitted by germinate >= 2.27. - -- Iain Lane Tue, 30 Jan 2018 10:05:49 +0000 + -- Iain Lane Tue, 30 Jan 2018 11:20:28 +0000 livecd-rootfs (2.490) bionic; urgency=medium diff --git a/live-build/auto/config b/live-build/auto/config index bbe61a26..89af8489 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -5,7 +5,8 @@ rm -rf config echo "Building on $(hostname --fqdn)" -SEEDMIRROR=http://people.canonical.com/~ubuntu-archive/seeds/ +SEEDMIRROR=http://people.canonical.com/~ubuntu-archive/~laney/seeds/ +MIRROR=http://raleigh/mirrors/ubuntu/ if [ -z "$MIRROR" ]; then case $(hostname --fqdn) in bld-*.mmjgroup.com) ;; From 42a6450584afc1bb0c8240e355e152319799a5da Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Tue, 30 Jan 2018 11:51:23 +0000 Subject: [PATCH 8/8] Oops. I shouldn't reference my private stuff. --- debian/changelog | 6 ++++++ live-build/auto/config | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 162f1157..17617560 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +livecd-rootfs (2.492) bionic; urgency=medium + + * Oops. I shouldn't reference my private stuff. + + -- Iain Lane Tue, 30 Jan 2018 11:51:19 +0000 + livecd-rootfs (2.491) bionic; urgency=medium * Support seeding of snaps, as emitted by germinate >= 2.27. diff --git a/live-build/auto/config b/live-build/auto/config index 89af8489..bbe61a26 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -5,8 +5,7 @@ rm -rf config echo "Building on $(hostname --fqdn)" -SEEDMIRROR=http://people.canonical.com/~ubuntu-archive/~laney/seeds/ -MIRROR=http://raleigh/mirrors/ubuntu/ +SEEDMIRROR=http://people.canonical.com/~ubuntu-archive/seeds/ if [ -z "$MIRROR" ]; then case $(hostname --fqdn) in bld-*.mmjgroup.com) ;;