Initial implementation of canary image builds.

2025-02-23 03:11:12 +00:00 · 2023-03-30 14:09:04 +02:00 · 2023-03-30 14:09:04 +02:00 · 11a583d619
commit 11a583d619
parent 140385c10c
9 changed files with 342 additions and 31 deletions
--- a/debian/control
+++ b/debian/control
@ -34,6 +34,7 @@ Depends: ${misc:Depends},
         python3-launchpadlib [!i386],
         python3-yaml,
         qemu-utils [!i386],
+         rsync,
         snapd (>= 2.39) [!i386],
         squashfs-tools (>= 1:3.3-1),
         sudo,
--- a/live-build/auto/config
+++ b/live-build/auto/config
@ -95,6 +95,13 @@ _register_pass () {
 	PASSES="$PASSES $1"
 }

+add_pass ()
+{
+	local pass="$1"
+	_check_immutable_passes_to_layers
+	_register_pass "$pass"
+}
+
 add_task ()
 {
 	local pass="$1"
@ -724,6 +731,59 @@ case $PROJECT in
 						;;
 				esac
 				;;
+			canary)
+				PASSES_TO_LAYERS="true"
+				# the minimal layer, for minimal installs
+				add_task minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages
+				add_package minimal cloud-init
+				# the standard layer, contains all base common packages for later layers (we're splitting out the snaps)
+				add_task minimal.standard ubuntu-desktop ubuntu-desktop-default-languages
+				# the classic layer, basically only contains snaps from the standard tasks
+				add_pass minimal.standard.classic
+				mv config/package-lists/livecd-rootfs.snaplist.chroot_minimal.standard.full \
+					config/package-lists/livecd-rootfs.snaplist.chroot_minimal.standard.classic.full
+				# the live layer, contains all packages for the live session installer
+				# TODO: we should probably add the kernel per KERNEL_FLAVOURS
+				add_package minimal.standard.live linux-generic casper lvm2 mdadm
+				remove_package minimal.standard.live ubiquity-frontend-gtk
+				# the enhanced-secureboot layer, contains all packages for the enhanced secureboot install
+				add_package minimal.standard.enhanced-secureboot cryptsetup
+				# now let's create the neccessary catalog files
+				cat <<-EOF > config/minimal.catalog-in.yaml
+					name: "Ubuntu Desktop (minimized)"
+					description: >-
+					  A minimal but usable Ubuntu Desktop.
+					id: ubuntu-desktop-minimal
+					type: fsimage-layered
+					variant: desktop
+					locale_support: langpack
+				EOF
+				cat <<-EOF > config/minimal.standard.catalog-in.yaml
+					name: "Ubuntu Desktop"
+					description: >-
+					  A full featured Ubuntu Desktop.
+					id: ubuntu-desktop
+					type: fsimage-layered
+					variant: desktop
+					locale_support: langpack
+					default: yes
+				EOF
+				cat <<-EOF > config/minimal.standard.classic.catalog-in.yaml
+					id: ubuntu-desktop
+					variations:
+					  classic:
+					    path: minimal.standard.squashfs
+				EOF
+				cat <<-EOF > config/minimal.standard.enhanced-secureboot.catalog-in.yaml
+					id: ubuntu-desktop
+					variations:
+					  enhanced-secureboot:
+					    path: minimal.standard.enhanced-secureboot.squashfs
+					    snapd_system_label: enhanced-secureboot-desktop
+				EOF
+				/usr/share/livecd-rootfs/checkout-translations-branch \
+					https://git.launchpad.net/subiquity po config/catalog-translations
+				;;
 			*)
 				touch config/universe-enabled
 				PASSES_TO_LAYERS="true"
@ -1182,7 +1242,7 @@ case "$ARCH${SUBARCH:++$SUBARCH}" in
 esac

 case $PROJECT:${SUBPROJECT:-} in
-	ubuntu-server:*|ubuntu-base:*|ubuntu-oci:*)
+	ubuntu-server:*|ubuntu-base:*|ubuntu-oci:*|ubuntu:canary)
 		OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none"
 		KERNEL_FLAVOURS=none
 		BINARY_REMOVE_LINUX=false
@ -1426,7 +1486,7 @@ EOF
 		fi
 		;;

-	ubuntu-cpc:*|ubuntu-server:live|ubuntu:desktop-preinstalled|ubuntu-wsl:*|ubuntu-mini-iso:*)
+	ubuntu-cpc:*|ubuntu-server:live|ubuntu:desktop-preinstalled|ubuntu-wsl:*|ubuntu-mini-iso:*|ubuntu:canary)
 		# Ensure that most things e.g. includes.chroot are copied as is
 		for entry in /usr/share/livecd-rootfs/live-build/${PROJECT}/*; do
 			case $entry in
--- a/live-build/lb_binary_layered
+++ b/live-build/lb_binary_layered
@ -116,7 +116,9 @@ build_layered_squashfs () {
 		create_manifest "chroot" "${squashfs_f_manifest}.full"

 		# Delta manifest
-		diff -NU0 ${PWD}/livecd.${PROJECT_FULL}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
+		diff -NU0 ${PWD}/livecd.${PROJECT_FULL}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest || true
+		echo "Delta manifest:"
+		cat $squashfs_f_manifest

 		squashfs_f_size="${base}.size"
 		du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}"
--- a/live-build/ubuntu-cpc/hooks.d/base/disk-image-uefi.binary
+++ b/live-build/ubuntu-cpc/hooks.d/base/disk-image-uefi.binary
@ -9,7 +9,11 @@ case $ARCH in
        ;;
 esac

-case ${PROJECT:-} in
+case ${PROJECT:-}:${SUBPROJECT:-} in
+    *:canary)
+        echo "We don't create EFI images for canary."
+        exit 0
+        ;;
    ubuntu)
        IMAGE_STR="# DESKTOP_IMG: This file was created/modified by the Desktop Image build process"
        FS_LABEL="desktop-rootfs"
--- a/live-build/ubuntu/hooks/020-canary-enhanced-sb.binary
+++ b/live-build/ubuntu/hooks/020-canary-enhanced-sb.binary
@ -0,0 +1,111 @@
+#! /bin/sh
+
+set -eux
+
+case $PASS in
+    minimal.standard.enhanced-secureboot)
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+case ${SUBPROJECT:-} in
+    canary)
+        ;;
+    *)
+        echo "We don't run canary hooks for this project."
+        exit 0
+        ;;
+esac
+
+/usr/lib/snapd/snap-preseed --reset $(realpath "chroot")
+rm -rf chroot/var/lib/snapd/seed/*
+
+# env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-classic-2304-amd64 > config/classic-model.model
+cat <<EOF > config/classic-model.model
+type: model
+authority-id: canonical
+series: 16
+brand-id: canonical
+model: ubuntu-classic-2304-amd64
+architecture: amd64
+base: core22
+classic: true
+distribution: ubuntu
+grade: signed
+snaps:
+  -
+    default-channel: classic-23.04/stable
+    id: UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH
+    name: pc
+    type: gadget
+  -
+    default-channel: 24-hwe/stable
+    id: pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza
+    name: pc-kernel
+    type: kernel
+  -
+    default-channel: latest/stable
+    id: amcUKQILKXHHTlmSa7NMdnXSx02dNeeT
+    name: core22
+    type: base
+  -
+    default-channel: latest/stable
+    id: PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4
+    name: snapd
+    type: snapd
+  -
+    default-channel: latest/stable
+    id: DLqre5XGLbDqg9jPtiAhRRjDuPVa5X1q
+    name: core20
+    type: base
+  -
+    default-channel: latest/stable
+    id: EISPgh06mRh1vordZY9OZ34QHdd7OrdR
+    name: bare
+    type: base
+  -
+    default-channel: latest/stable
+    id: 3wdHCAVyZEmYsCMFDE9qt92UV8rC8Wdk
+    name: firefox
+    type: app
+  -
+    default-channel: latest/stable
+    id: lATO8HzwVvrAPrlZRAWpfyrJKlAJrZS3
+    name: gnome-42-2204
+    type: app
+  -
+    default-channel: latest/stable
+    id: jZLfBRzf1cYlYysIjD2bwSzNtngY0qit
+    name: gtk-common-themes
+    type: app
+  -
+    default-channel: latest/stable
+    id: gjf3IPXoRiipCu9K0kVu52f0H56fIksg
+    name: snap-store
+    type: app
+  -
+    default-channel: latest/stable
+    id: IrwRHakqtzhFRHJOOPxKVPU0Kk7Erhcu
+    name: snapd-desktop-integration
+    type: app
+timestamp: 2023-03-19T12:00:00.0Z
+sign-key-sha3-384: 9tydnLa6MTJ-jaQTFUXEwHl1yRx7ZS4K5cyFDhYDcPzhS7uyEkDxdUjg9g08BtNn
+
+AcLBXAQAAQoABgUCZBxqbQAKCRDgT5vottzAEudYD/49FlV7VOhnZ1+c/jxQgu6ucGNjjOsZdmqi
+2tJEz7tGyStMe2yDnBOjEBb6yvghzzXZX+gLIolHReXYWzrdB0UScPdXJvSRhngeAgn70hEC2H5S
+Pb3/r3tXl3QQs0hd+5Szg31qICoW8ASOsvM/ymULTECOWOemCDnFRJ8wpAleNQ0LEACY/RD2814F
+mmkunA6yXto1OhUySpaH5xXWoHjfvZsq/nXd4P1aI42aPxkB1JBHBwIl0ECVbzofCvh4f8NbbvgA
+FuGfvFrNa9ifUh7m0L3XwZML9sVZKvLviG6tANHWl8sCJ02W+Hw4Lq93OC6rN5Kfii+1zhapFE8Q
+tF0KtN6HPp69PzOzhYB52J66i1hrmWRkdoB4lqxOzpANYEXGCKopwTDP+eX3xIjiLft+sMmIHzXn
+ITA2WLD31AxLXk4IKBWBLh3NgJa/A7tRjGnOhZaEfRA1clSsvSA45PXCDY9fuH3ejMvgi1qvO/TW
+m2S+MgKSvJL6n5KRs5amBiiLqVuAs+WFOIr6c1d6PR6DoM4D7WJxR31jOJdAPh/VeT+DbOkFYybg
+PXNB7cgStSAyvustnx6T8UgueAXy1QknO7TMco06R2MXS9DMWSOMMfyCJAbKvx9epkSXWpL487ag
+h88GPKYhsiv85hZNB65Cpnz7MJJVWg0WIhGMwG9clg==
+EOF
+
+env SNAPPY_STORE_NO_CDN=1 snap prepare-image --classic config/classic-model.model chroot
+mv chroot/system-seed/systems/* chroot/system-seed/systems/classic
+rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed
+rm -rf chroot/system-seed/
--- a/live-build/ubuntu/hooks/020-canary-live.binary
+++ b/live-build/ubuntu/hooks/020-canary-live.binary
@ -0,0 +1,96 @@
+#! /bin/sh
+
+set -eux
+
+case $PASS in
+    minimal.standard.live)
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+case ${SUBPROJECT:-} in
+    canary)
+        ;;
+    *)
+        echo "We don't run canary hooks for this project."
+        exit 0
+        ;;
+esac
+
+# env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-server-installer-classic-2304-amd64 > config/classic-model-installer.model
+cat <<EOF > config/classic-model-installer.model
+type: model
+authority-id: canonical
+series: 16
+brand-id: canonical
+model: ubuntu-server-installer-classic-2304-amd64
+architecture: amd64
+base: core22
+classic: true
+distribution: ubuntu
+grade: signed
+snaps:
+  -
+    default-channel: latest/stable
+    id: amcUKQILKXHHTlmSa7NMdnXSx02dNeeT
+    name: core22
+    type: base
+  -
+    default-channel: latest/stable
+    id: PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4
+    name: snapd
+    type: snapd
+  -
+    classic: true
+    default-channel: latest/stable/canary-23.04
+    id: ba2aj8guta0zSRlT3QM5aJNAUXPlBtf9
+    name: subiquity
+    type: app
+timestamp: 2023-03-19T12:00:00.0Z
+sign-key-sha3-384: 9tydnLa6MTJ-jaQTFUXEwHl1yRx7ZS4K5cyFDhYDcPzhS7uyEkDxdUjg9g08BtNn
+
+AcLBXAQAAQoABgUCZCT4iAAKCRDgT5vottzAEkmPD/46nms8edTFkoiiH/SjAdyo8MW2FhR3tW03
+CyH1NQwrNJtHiqb7EcWQHDPexM3Wxvwj2iPN04IWlGugge7Fw12AmhVsYYCnCjYjDL8uBa9ixBj3
+kghOsFlVbDBk+f5tLmNGNncQfPbV6TvywK9a+2zgVEdRdLlmNeorX3b7P0woCIWg9wEikX+vcLi/
+q1GvWtI4Gb/rAIN+/OaD4DFgbJSUdSaayGX90hbNtvsZc0K7KPTeNlF0IGKg72yvqBGDX/0m2IDb
+6HozMxGg7xcq+WwHU3hucBNqw6kWKKDd1gzZJICrJf9kLEZ7GoGa8Oj9eET+bp2KYVHzjNzHvQbd
+v4odCG3Wn7m5ogpPmt/mUoTOhVtY/RNGnk46eHAFHuFcPAjR1BnIfLLmh4EmEnkIboTrlBKFgHMW
+ddFSOFYonboYu7R9Gad3J+9VD5IyyNqJNpCrEqLfw81QaFz1+KM00XD+wns2+ZqnI90a6olXcmUH
+t1Fv0QIDNCG0x8Lf+My/gJycNCEbUj+TA6xOL9NH0EFU6myKyR74sC5tlMZPMN2RKdZglOZUnkyC
+lpuhRVI3fcNZL2sVGwmvoKUDj7QaLrETpCUqDXMzNw73NOS3zA8+gQr/X438KpEFscszS1g+FtJ4
+LVOcgLrDw/S61ciBuOBeOaskqZdZ4xYmm1wKMzecXg==
+EOF
+
+env SNAPPY_STORE_NO_CDN=1 snap prepare-image --classic config/classic-model-installer.model chroot
+mv chroot/system-seed/systems/* chroot/system-seed/systems/classic-installer
+rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed
+rm -rf chroot/system-seed/
+
+cat <<EOF  > chroot/var/lib/snapd/modeenv
+mode=run
+recovery_system=classic-installer
+EOF
+
+# Temporary while we still use the subiquity TUI for the live system
+cat > chroot/usr/lib/systemd/user/subiquity-tui.service  << EOF
+[Unit]
+Description=subiquity tui
+PartOf=graphical-session.target
+After=graphical-session.target
+
+# Never run in GDM
+Conflicts=gnome-session@gnome-login.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/gnome-terminal --wait -- sudo sh -c 'echo waiting for snapd; snap wait system seed.loaded; sleep 5; snap run subiquity'
+Restart=no
+EOF
+mkdir chroot/etc/systemd/user/graphical-session.target.wants/
+ln -vs /usr/lib/systemd/user/subiquity-tui.service chroot/etc/systemd/user/graphical-session.target.wants/
+
+mv chroot/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-generic
+mv chroot/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-generic
+chmod a+r ${PWD}/livecd.${PROJECT}.initrd-generic ${PWD}/livecd.${PROJECT}.kernel-generic
--- a/live-build/ubuntu/hooks/020-canary-live.chroot_early
+++ b/live-build/ubuntu/hooks/020-canary-live.chroot_early
@ -0,0 +1,26 @@
+#! /bin/sh
+
+case $PASS in
+    minimal.standard.live)
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+case ${SUBPROJECT:-} in
+    canary)
+        ;;
+    *)
+        echo "We don't run canary hooks for this project."
+        exit 0
+        ;;
+esac
+
+cat <<EOF > /etc/initramfs-tools/conf.d/casperize.conf
+export CASPER_GENERATE_UUID=1
+EOF
+
+cat <<EOF > /etc/initramfs-tools/conf.d/default-layer.conf
+LAYERFS_PATH=${PASS}.squashfs
+EOF
--- a/live-build/ubuntu/hooks/040-hyperv-desktop-images.binary
+++ b/live-build/ubuntu/hooks/040-hyperv-desktop-images.binary
@ -3,7 +3,7 @@
 echo "Creating Hyper-V image with Desktop..."

 case ${SUBPROJECT:-} in
-    minimized)
+    minimized|canary)
        echo "We don't create minimized images for $0."
        exit 0
        ;;
--- a/63
+++ b/63
@ -28,45 +28,56 @@ with open(opts.template) as fp:
    template = yaml.safe_load(fp)


-template['size'] = int(opts.size)
-template['path'] = opts.squashfs
+id = template['id']
+for entry in output:
+    # First, look if this source catalogue template id is already present.
+    # If so, use the template to extend the existing entry with additional
+    # variations.
+    if entry['id'] == id:
+        if 'variations' not in template:
+            print("Non unique id in source catalog but no variations!")
+            sys.exit(1)
+        if 'variations' not in entry:
+            entry['variations'] = {}
+        for k, variation in template['variations'].items():
+            variation['size'] = int(opts.size)
+        entry['variations'].update(template['variations'])
+        break
+else:
+    # No entry with this id found, so add a new one.
+    template['size'] = int(opts.size)
+    template['path'] = opts.squashfs

-en_name = template['name']
-en_description = template['description']
+    en_name = template['name']
+    en_description = template['description']

-template['name'] = {'en': en_name}
-template['description'] = {'en': en_description}
+    template['name'] = {'en': en_name}
+    template['description'] = {'en': en_description}

-for mo in glob.glob(os.path.join(opts.translations, '*.mo')):
-    with open(mo, 'rb') as fp:
-        t = gettext.GNUTranslations(fp=fp)
-    t_name = t.gettext(en_name)
-    if t_name != en_name:
-        lang = os.path.splitext(os.path.basename(mo))[0]
-        template['name'][lang] = t_name
-    t_description = t.gettext(en_description)
-    if t_description != en_description:
-        lang = os.path.splitext(os.path.basedescription(mo))[0]
-        template['description'][lang] = t_description
-    if opts.langs is not None:
-        template['preinstalled_langs'] = opts.langs.split(',')
+    for mo in glob.glob(os.path.join(opts.translations, '*.mo')):
+        with open(mo, 'rb') as fp:
+            t = gettext.GNUTranslations(fp=fp)
+        t_name = t.gettext(en_name)
+        if t_name != en_name:
+            lang = os.path.splitext(os.path.basename(mo))[0]
+            template['name'][lang] = t_name
+        t_description = t.gettext(en_description)
+        if t_description != en_description:
+            lang = os.path.splitext(os.path.basedescription(mo))[0]
+            template['description'][lang] = t_description
+        if opts.langs is not None:
+            template['preinstalled_langs'] = opts.langs.split(',')

-output.append(template)
+    output.append(template)

 default_count = 0
-ids = set()
 for entry in output:
    if entry.get('default', False):
        default_count += 1
-    ids.add(entry['id'])
-

 if default_count > 1:
    print("Too many defaults in source catalog!")
    sys.exit(1)

-if len(ids) != len(output):
-    print("Non unique ids in %s!" % output)
-
 with open(opts.output, 'w') as fp:
    yaml.dump(output, fp)