Initial implementation of canary image builds.

debian/control

@@ -34,6 +34,7 @@ Depends: ${misc:Depends},
          python3-launchpadlib [!i386],
          python3-yaml,
          qemu-utils [!i386],
+         rsync,
          snapd (>= 2.39) [!i386],
          squashfs-tools (>= 1:3.3-1),
          sudo,

live-build/auto/config

@@ -95,6 +95,13 @@ _register_pass () {
 	PASSES="$PASSES $1"
 }
 
+add_pass ()
+{
+	local pass="$1"
+	_check_immutable_passes_to_layers
+	_register_pass "$pass"
+}
+
 add_task ()
 {
 	local pass="$1"
@@ -724,6 +731,59 @@ case $PROJECT in
 						;;
 				esac
 				;;
+			canary)
+				PASSES_TO_LAYERS="true"
+				# the minimal layer, for minimal installs
+				add_task minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages
+				add_package minimal cloud-init
+				# the standard layer, contains all base common packages for later layers (we're splitting out the snaps)
+				add_task minimal.standard ubuntu-desktop ubuntu-desktop-default-languages
+				# the classic layer, basically only contains snaps from the standard tasks
+				add_pass minimal.standard.classic
+				mv config/package-lists/livecd-rootfs.snaplist.chroot_minimal.standard.full \
+					config/package-lists/livecd-rootfs.snaplist.chroot_minimal.standard.classic.full
+				# the live layer, contains all packages for the live session installer
+				# TODO: we should probably add the kernel per KERNEL_FLAVOURS
+				add_package minimal.standard.live linux-generic casper lvm2 mdadm
+				remove_package minimal.standard.live ubiquity-frontend-gtk
+				# the enhanced-secureboot layer, contains all packages for the enhanced secureboot install
+				add_package minimal.standard.enhanced-secureboot cryptsetup
+				# now let's create the neccessary catalog files
+				cat <<-EOF > config/minimal.catalog-in.yaml
+					name: "Ubuntu Desktop (minimized)"
+					description: >-
+					  A minimal but usable Ubuntu Desktop.
+					id: ubuntu-desktop-minimal
+					type: fsimage-layered
+					variant: desktop
+					locale_support: langpack
+				EOF
+				cat <<-EOF > config/minimal.standard.catalog-in.yaml
+					name: "Ubuntu Desktop"
+					description: >-
+					  A full featured Ubuntu Desktop.
+					id: ubuntu-desktop
+					type: fsimage-layered
+					variant: desktop
+					locale_support: langpack
+					default: yes
+				EOF
+				cat <<-EOF > config/minimal.standard.classic.catalog-in.yaml
+					id: ubuntu-desktop
+					variations:
+					  classic:
+					    path: minimal.standard.squashfs
+				EOF
+				cat <<-EOF > config/minimal.standard.enhanced-secureboot.catalog-in.yaml
+					id: ubuntu-desktop
+					variations:
+					  enhanced-secureboot:
+					    path: minimal.standard.enhanced-secureboot.squashfs
+					    snapd_system_label: enhanced-secureboot-desktop
+				EOF
+				/usr/share/livecd-rootfs/checkout-translations-branch \
+					https://git.launchpad.net/subiquity po config/catalog-translations
+				;;
 			*)
 				touch config/universe-enabled
 				PASSES_TO_LAYERS="true"
@@ -1182,7 +1242,7 @@ case "$ARCH${SUBARCH:++$SUBARCH}" in
 esac
 
 case $PROJECT:${SUBPROJECT:-} in
-	ubuntu-server:*|ubuntu-base:*|ubuntu-oci:*)
+	ubuntu-server:*|ubuntu-base:*|ubuntu-oci:*|ubuntu:canary)
 		OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none"
 		KERNEL_FLAVOURS=none
 		BINARY_REMOVE_LINUX=false
@@ -1426,7 +1486,7 @@ EOF
 		fi
 		;;
 
-	ubuntu-cpc:*|ubuntu-server:live|ubuntu:desktop-preinstalled|ubuntu-wsl:*|ubuntu-mini-iso:*)
+	ubuntu-cpc:*|ubuntu-server:live|ubuntu:desktop-preinstalled|ubuntu-wsl:*|ubuntu-mini-iso:*|ubuntu:canary)
 		# Ensure that most things e.g. includes.chroot are copied as is
 		for entry in /usr/share/livecd-rootfs/live-build/${PROJECT}/*; do
 			case $entry in

live-build/lb_binary_layered

@@ -116,7 +116,9 @@ build_layered_squashfs () {
 		create_manifest "chroot" "${squashfs_f_manifest}.full"
 
 		# Delta manifest
-		diff -NU0 ${PWD}/livecd.${PROJECT_FULL}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
+		diff -NU0 ${PWD}/livecd.${PROJECT_FULL}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest || true
+		echo "Delta manifest:"
+		cat $squashfs_f_manifest
 
 		squashfs_f_size="${base}.size"
 		du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}"

live-build/ubuntu-cpc/hooks.d/base/disk-image-uefi.binary

@@ -9,7 +9,11 @@ case $ARCH in
         ;;
 esac
 
-case ${PROJECT:-} in
+case ${PROJECT:-}:${SUBPROJECT:-} in
+    *:canary)
+        echo "We don't create EFI images for canary."
+        exit 0
+        ;;
     ubuntu)
         IMAGE_STR="# DESKTOP_IMG: This file was created/modified by the Desktop Image build process"
         FS_LABEL="desktop-rootfs"

live-build/ubuntu/hooks/020-canary-enhanced-sb.binary

@@ -0,0 +1,111 @@
+#! /bin/sh
+
+set -eux
+
+case $PASS in
+    minimal.standard.enhanced-secureboot)
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+case ${SUBPROJECT:-} in
+    canary)
+        ;;
+    *)
+        echo "We don't run canary hooks for this project."
+        exit 0
+        ;;
+esac
+
+/usr/lib/snapd/snap-preseed --reset $(realpath "chroot")
+rm -rf chroot/var/lib/snapd/seed/*
+
+# env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-classic-2304-amd64 > config/classic-model.model
+cat <<EOF > config/classic-model.model
+type: model
+authority-id: canonical
+series: 16
+brand-id: canonical
+model: ubuntu-classic-2304-amd64
+architecture: amd64
+base: core22
+classic: true
+distribution: ubuntu
+grade: signed
+snaps:
+  -
+    default-channel: classic-23.04/stable
+    id: UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH
+    name: pc
+    type: gadget
+  -
+    default-channel: 24-hwe/stable
+    id: pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza
+    name: pc-kernel
+    type: kernel
+  -
+    default-channel: latest/stable
+    id: amcUKQILKXHHTlmSa7NMdnXSx02dNeeT
+    name: core22
+    type: base
+  -
+    default-channel: latest/stable
+    id: PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4
+    name: snapd
+    type: snapd
+  -
+    default-channel: latest/stable
+    id: DLqre5XGLbDqg9jPtiAhRRjDuPVa5X1q
+    name: core20
+    type: base
+  -
+    default-channel: latest/stable
+    id: EISPgh06mRh1vordZY9OZ34QHdd7OrdR
+    name: bare
+    type: base
+  -
+    default-channel: latest/stable
+    id: 3wdHCAVyZEmYsCMFDE9qt92UV8rC8Wdk
+    name: firefox
+    type: app
+  -
+    default-channel: latest/stable
+    id: lATO8HzwVvrAPrlZRAWpfyrJKlAJrZS3
+    name: gnome-42-2204
+    type: app
+  -
+    default-channel: latest/stable
+    id: jZLfBRzf1cYlYysIjD2bwSzNtngY0qit
+    name: gtk-common-themes
+    type: app
+  -
+    default-channel: latest/stable
+    id: gjf3IPXoRiipCu9K0kVu52f0H56fIksg
+    name: snap-store
+    type: app
+  -
+    default-channel: latest/stable
+    id: IrwRHakqtzhFRHJOOPxKVPU0Kk7Erhcu
+    name: snapd-desktop-integration
+    type: app
+timestamp: 2023-03-19T12:00:00.0Z
+sign-key-sha3-384: 9tydnLa6MTJ-jaQTFUXEwHl1yRx7ZS4K5cyFDhYDcPzhS7uyEkDxdUjg9g08BtNn
+
+AcLBXAQAAQoABgUCZBxqbQAKCRDgT5vottzAEudYD/49FlV7VOhnZ1+c/jxQgu6ucGNjjOsZdmqi
+2tJEz7tGyStMe2yDnBOjEBb6yvghzzXZX+gLIolHReXYWzrdB0UScPdXJvSRhngeAgn70hEC2H5S
+Pb3/r3tXl3QQs0hd+5Szg31qICoW8ASOsvM/ymULTECOWOemCDnFRJ8wpAleNQ0LEACY/RD2814F
+mmkunA6yXto1OhUySpaH5xXWoHjfvZsq/nXd4P1aI42aPxkB1JBHBwIl0ECVbzofCvh4f8NbbvgA
+FuGfvFrNa9ifUh7m0L3XwZML9sVZKvLviG6tANHWl8sCJ02W+Hw4Lq93OC6rN5Kfii+1zhapFE8Q
+tF0KtN6HPp69PzOzhYB52J66i1hrmWRkdoB4lqxOzpANYEXGCKopwTDP+eX3xIjiLft+sMmIHzXn
+ITA2WLD31AxLXk4IKBWBLh3NgJa/A7tRjGnOhZaEfRA1clSsvSA45PXCDY9fuH3ejMvgi1qvO/TW
+m2S+MgKSvJL6n5KRs5amBiiLqVuAs+WFOIr6c1d6PR6DoM4D7WJxR31jOJdAPh/VeT+DbOkFYybg
+PXNB7cgStSAyvustnx6T8UgueAXy1QknO7TMco06R2MXS9DMWSOMMfyCJAbKvx9epkSXWpL487ag
+h88GPKYhsiv85hZNB65Cpnz7MJJVWg0WIhGMwG9clg==
+EOF
+
+env SNAPPY_STORE_NO_CDN=1 snap prepare-image --classic config/classic-model.model chroot
+mv chroot/system-seed/systems/* chroot/system-seed/systems/classic
+rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed
+rm -rf chroot/system-seed/

live-build/ubuntu/hooks/020-canary-live.binary

@@ -0,0 +1,96 @@
+#! /bin/sh
+
+set -eux
+
+case $PASS in
+    minimal.standard.live)
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+case ${SUBPROJECT:-} in
+    canary)
+        ;;
+    *)
+        echo "We don't run canary hooks for this project."
+        exit 0
+        ;;
+esac
+
+# env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 brand-id=canonical model=ubuntu-server-installer-classic-2304-amd64 > config/classic-model-installer.model
+cat <<EOF > config/classic-model-installer.model
+type: model
+authority-id: canonical
+series: 16
+brand-id: canonical
+model: ubuntu-server-installer-classic-2304-amd64
+architecture: amd64
+base: core22
+classic: true
+distribution: ubuntu
+grade: signed
+snaps:
+  -
+    default-channel: latest/stable
+    id: amcUKQILKXHHTlmSa7NMdnXSx02dNeeT
+    name: core22
+    type: base
+  -
+    default-channel: latest/stable
+    id: PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4
+    name: snapd
+    type: snapd
+  -
+    classic: true
+    default-channel: latest/stable/canary-23.04
+    id: ba2aj8guta0zSRlT3QM5aJNAUXPlBtf9
+    name: subiquity
+    type: app
+timestamp: 2023-03-19T12:00:00.0Z
+sign-key-sha3-384: 9tydnLa6MTJ-jaQTFUXEwHl1yRx7ZS4K5cyFDhYDcPzhS7uyEkDxdUjg9g08BtNn
+
+AcLBXAQAAQoABgUCZCT4iAAKCRDgT5vottzAEkmPD/46nms8edTFkoiiH/SjAdyo8MW2FhR3tW03
+CyH1NQwrNJtHiqb7EcWQHDPexM3Wxvwj2iPN04IWlGugge7Fw12AmhVsYYCnCjYjDL8uBa9ixBj3
+kghOsFlVbDBk+f5tLmNGNncQfPbV6TvywK9a+2zgVEdRdLlmNeorX3b7P0woCIWg9wEikX+vcLi/
+q1GvWtI4Gb/rAIN+/OaD4DFgbJSUdSaayGX90hbNtvsZc0K7KPTeNlF0IGKg72yvqBGDX/0m2IDb
+6HozMxGg7xcq+WwHU3hucBNqw6kWKKDd1gzZJICrJf9kLEZ7GoGa8Oj9eET+bp2KYVHzjNzHvQbd
+v4odCG3Wn7m5ogpPmt/mUoTOhVtY/RNGnk46eHAFHuFcPAjR1BnIfLLmh4EmEnkIboTrlBKFgHMW
+ddFSOFYonboYu7R9Gad3J+9VD5IyyNqJNpCrEqLfw81QaFz1+KM00XD+wns2+ZqnI90a6olXcmUH
+t1Fv0QIDNCG0x8Lf+My/gJycNCEbUj+TA6xOL9NH0EFU6myKyR74sC5tlMZPMN2RKdZglOZUnkyC
+lpuhRVI3fcNZL2sVGwmvoKUDj7QaLrETpCUqDXMzNw73NOS3zA8+gQr/X438KpEFscszS1g+FtJ4
+LVOcgLrDw/S61ciBuOBeOaskqZdZ4xYmm1wKMzecXg==
+EOF
+
+env SNAPPY_STORE_NO_CDN=1 snap prepare-image --classic config/classic-model-installer.model chroot
+mv chroot/system-seed/systems/* chroot/system-seed/systems/classic-installer
+rsync -a chroot/system-seed/ chroot/var/lib/snapd/seed
+rm -rf chroot/system-seed/
+
+cat <<EOF  > chroot/var/lib/snapd/modeenv
+mode=run
+recovery_system=classic-installer
+EOF
+
+# Temporary while we still use the subiquity TUI for the live system
+cat > chroot/usr/lib/systemd/user/subiquity-tui.service  << EOF
+[Unit]
+Description=subiquity tui
+PartOf=graphical-session.target
+After=graphical-session.target
+
+# Never run in GDM
+Conflicts=gnome-session@gnome-login.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/gnome-terminal --wait -- sudo sh -c 'echo waiting for snapd; snap wait system seed.loaded; sleep 5; snap run subiquity'
+Restart=no
+EOF
+mkdir chroot/etc/systemd/user/graphical-session.target.wants/
+ln -vs /usr/lib/systemd/user/subiquity-tui.service chroot/etc/systemd/user/graphical-session.target.wants/
+
+mv chroot/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-generic
+mv chroot/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-generic
+chmod a+r ${PWD}/livecd.${PROJECT}.initrd-generic ${PWD}/livecd.${PROJECT}.kernel-generic

live-build/ubuntu/hooks/020-canary-live.chroot_early

@@ -0,0 +1,26 @@
+#! /bin/sh
+
+case $PASS in
+    minimal.standard.live)
+        ;;
+    *)
+        exit 0
+        ;;
+esac
+
+case ${SUBPROJECT:-} in
+    canary)
+        ;;
+    *)
+        echo "We don't run canary hooks for this project."
+        exit 0
+        ;;
+esac
+
+cat <<EOF > /etc/initramfs-tools/conf.d/casperize.conf
+export CASPER_GENERATE_UUID=1
+EOF
+
+cat <<EOF > /etc/initramfs-tools/conf.d/default-layer.conf
+LAYERFS_PATH=${PASS}.squashfs
+EOF

live-build/ubuntu/hooks/040-hyperv-desktop-images.binary

@@ -3,7 +3,7 @@
 echo "Creating Hyper-V image with Desktop..."
 
 case ${SUBPROJECT:-} in
-    minimized)
+    minimized|canary)
         echo "We don't create minimized images for $0."
         exit 0
         ;;

update-source-catalog

@@ -28,6 +28,23 @@ with open(opts.template) as fp:
     template = yaml.safe_load(fp)
 
 
+id = template['id']
+for entry in output:
+    # First, look if this source catalogue template id is already present.
+    # If so, use the template to extend the existing entry with additional
+    # variations.
+    if entry['id'] == id:
+        if 'variations' not in template:
+            print("Non unique id in source catalog but no variations!")
+            sys.exit(1)
+        if 'variations' not in entry:
+            entry['variations'] = {}
+        for k, variation in template['variations'].items():
+            variation['size'] = int(opts.size)
+        entry['variations'].update(template['variations'])
+        break
+else:
+    # No entry with this id found, so add a new one.
     template['size'] = int(opts.size)
     template['path'] = opts.squashfs
 
@@ -54,19 +71,13 @@ for mo in glob.glob(os.path.join(opts.translations, '*.mo')):
     output.append(template)
 
 default_count = 0
-ids = set()
 for entry in output:
     if entry.get('default', False):
         default_count += 1
-    ids.add(entry['id'])
-
 
 if default_count > 1:
     print("Too many defaults in source catalog!")
     sys.exit(1)
 
-if len(ids) != len(output):
-    print("Non unique ids in %s!" % output)
-
 with open(opts.output, 'w') as fp:
     yaml.dump(output, fp)