Imported 2.525.12

No reason for CPC update specified.

debian/changelog

@@ -1,3 +1,33 @@
+livecd-rootfs (2.525.12) bionic; urgency=medium
+
+  * Key netplan delegation to NetworkManager on presence of
+    /usr/sbin/NetworkManager, not on /usr/lib/NetworkManager which may have
+    hooks from other packages (i.e., wpasupplicant).
+
+  [ Cody Shepherd ]
+  * Include snaps in image manifests (LP: #1805497)
+  * Change call to add grub efi packages using new create_manifests()
+    function.
+
+ -- Steve Langasek <steve.langasek@ubuntu.com>  Mon, 10 Dec 2018 12:46:46 -0800
+
+livecd-rootfs (2.525.11) bionic; urgency=medium
+
+  * Backport all the required changes to enable Raspberry Pi 3 armhf and arm64
+    preinstalled image builds. (LP: #1805668)
+    - Add support for raspi3 rootfs builds (based on Ryan Finnie's changes).
+    - For ubuntu-image consumption, export the kernel and initrd to
+      image/boot/uboot for raspi*.
+    - Avoid issues of hard-linking to a symbolic vmlinuz as this can lead to a
+      dangling symlink.
+    - Add raspi3 arm64 rootfs build support.
+    - Add git to the build dependencies for the gadget tree pull.
+    - Minor fixes to raspi3 builds: add arm64+raspi3 to the supported model
+      list, pass SUITE on to the build stage and use the git:// url for the
+      gadget tree.
+
+ -- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com>  Thu, 29 Nov 2018 16:24:23 +0100
+
 livecd-rootfs (2.525.10) bionic; urgency=medium
 
   [ Cody Shepherd ]

debian/control

@@ -17,12 +17,14 @@ Depends: ${misc:Depends},
          gdisk,
          genisoimage,
          germinate (>= 1.25.1),
+         git,
          gnupg,
          grep-dctrl,
          kpartx,
          live-build (>= 3.0~a57-1ubuntu31~),
          lsb-release,
          lzma,
+         make,
          parted,
          procps,
          python-minimal | python,
@@ -32,6 +34,7 @@ Depends: ${misc:Depends},
          snapd,
          squashfs-tools (>= 1:3.3-1),
          sudo,
+         u-boot-tools [armhf arm64],
          ubuntu-image,
          vmdk-stream-converter [amd64 i386],
          xz-utils,

live-build/auto/build

@@ -23,12 +23,31 @@ PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
 if [ "${IMAGEFORMAT:-}" = "ubuntu-image" ]; then
 	# Use ubuntu-image instead of live-build
 
+	if [ "$PROJECT" = "ubuntu-core"]; then
 		CHANNEL="${CHANNEL:-edge}"
 		env SNAPPY_STORE_NO_CDN=1 \
-	ubuntu-image -c "$CHANNEL" $UBUNTU_IMAGE_ARGS \
+		ubuntu-image snap -c "$CHANNEL" $UBUNTU_IMAGE_ARGS \
-		-o "$PREFIX".img "$PREFIX".model-assertion
+			-O output "$PREFIX".model-assertion
+		# XXX: currently we only have one image generated, but really
+		#  we should be supporting more than one for models that
+		#  define those.
+		mv output/*.img "$PREFIX".img
 		xz -0 -T4 "$PREFIX".img
-	mv seed.manifest "$PREFIX".manifest
+		mv output/seed.manifest "$PREFIX".manifest
+	else
+		# First we need to build the gadget tree
+		make -C "config/$PREFIX-gadget" ARCH=$ARCH SERIES=$SUITE
+		ubuntu-image classic $UBUNTU_IMAGE_ARGS \
+			-s $SUITE -p $PROJECT -a $ARCH --subarch $SUBARCH \
+			-O output config/$PREFIX-gadget/install
+		# XXX: currently we only have one image generated, but really
+		#  we should be supporting more than one for models that
+		#  define those.
+		mv output/*.img "$PREFIX".img
+		xz -0 -T4 "$PREFIX".img
+		mv output/filesystem.manifest "$PREFIX".manifest
+	fi
+
 	exit 0
 fi
 
@@ -396,7 +415,7 @@ EOF
 	# default. Installing NM on an existing system only manages wifi and wwan via
 	# /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting
 	# the global backend to NM, netplan overrides that file.
-	if [ -d chroot/usr/lib/NetworkManager ]; then
+	if [ -e chroot/usr/sbin/NetworkManager ]; then
 		echo "===== Enabling all devices in NetworkManager ===="
 		mkdir -p chroot/etc/netplan
 		cat <<EOF > chroot/etc/netplan/01-network-manager-all.yaml
@@ -503,6 +522,7 @@ fi
 
 # '--initramfs none' produces different manifest names.
 if [ -e "binary/$INITFS/filesystem.packages" ]; then
+	./config/snap-seed-parse "chroot/" "binary/${INITFS}/filesystem.packages"
 	ln "binary/$INITFS/filesystem.packages" "$PREFIX.manifest"
 	chmod 644 "$PREFIX.manifest"
 fi
@@ -800,13 +820,13 @@ done
 NUMFLAVOURS="$(set -- $LB_LINUX_FLAVOURS; echo $#)"
 if [ "$NUMFLAVOURS" = 1 ] && [ "$LB_LINUX_FLAVOURS" != "none" ]; then
 	# only one kernel flavour
-	if [ -e "binary/$INITFS/vmlinuz" ]; then
+	if [ -f "binary/$INITFS/vmlinuz" ] && ! [ -h "binary/$INITFS/vmlinuz" ]; then
 		ln "binary/$INITFS/vmlinuz" "$PREFIX.kernel"
 		chmod 644 "$PREFIX.kernel"
 	else
 		ln -sf "$PREFIX.kernel-$LB_LINUX_FLAVOURS" "$PREFIX.kernel"
 	fi
-	if [ -e "binary/$INITFS/initrd.lz" ]; then
+	if [ -f "binary/$INITFS/initrd.lz" ] && ! [ -h "binary/$INITFS/initrd.lz" ]; then
 		ln "binary/$INITFS/initrd.lz" "$PREFIX.initrd"
 		chmod 644 "$PREFIX.initrd"
 	else
@@ -814,7 +834,8 @@ if [ "$NUMFLAVOURS" = 1 ] && [ "$LB_LINUX_FLAVOURS" != "none" ]; then
 	fi
 fi
 
-if [ "$SUBARCH" = "ac100" ]; then
+case $SUBARCH in
+	ac100)
 		# create the md5sum and size files for which we are actually doing all this
 		md5sum $PREFIX.rootfs.tar.gz >chroot/installer.md5
 		wc -c $PREFIX.rootfs.tar.gz >chroot/installer.size
@@ -842,8 +863,23 @@ if [ "$SUBARCH" = "ac100" ]; then
 	
 		cp "chroot/boot/installer-${KVERS}.img" "$PREFIX.bootimg-$FLAVOUR"
 		ln -sf "$PREFIX.bootimg-$FLAVOUR" "$PREFIX.bootimg"
+		;;
 
-fi
+	raspi2|raspi3)
+		# copy the kernel and initrd to a predictable directory for
+		# ubuntu-image consumption.  In some cases, like in pi2/3
+		# u-boot, the bootloader needs to contain the kernel and initrd,
+		# so during rootfs build we copy it over to a directory that
+		# ubuntu-image looks for and shoves into the bootloader
+		# partition.
+		UBOOT_BOOT="image/boot/uboot"
+
+		mkdir -p $UBOOT_BOOT
+
+		cp $PREFIX.initrd $UBOOT_BOOT/initrd.img || true
+		cp $PREFIX.kernel $UBOOT_BOOT/vmlinuz || true
+		;;
+esac
 
 if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then
 	sourceslist="chroot/etc/apt/sources.list"

live-build/auto/config

@@ -33,6 +33,7 @@ fi
 
 mkdir -p config
 cp -af /usr/share/livecd-rootfs/live-build/functions config/functions
+cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse
 
 mkdir -p config/package-lists
 
@@ -175,7 +176,7 @@ case $IMAGEFORMAT in
 				MODEL=dragonboard ;;
 			armhf+raspi2)
 				MODEL=pi2 ;;
-			armhf+raspi3)
+			armhf+raspi3|arm64+raspi3)
 				MODEL=pi3 ;;
 			armhf+cm3)
 				MODEL=cm3 ;;
@@ -184,6 +185,10 @@ case $IMAGEFORMAT in
 				exit 1
 				;;
 		esac
+
+		if [ $PROJECT = "ubuntu-core" ]; then
+			# snap-based core images
+
 			case $MODEL in
 				pc-amd64|pc-i386)
 					UBUNTU_IMAGE_ARGS="--image-size 3700M" ;;
@@ -198,6 +203,21 @@ case $IMAGEFORMAT in
 			echo "Configured ubuntu-image for the following model assertion:"
 			cat "$PREFIX".model-assertion
 			echo "----------------------------------------------------------"
+		else
+			# classic images
+
+			UBUNTU_IMAGE_ARGS=""
+			UBUNTU_IMAGE_ARGS="$UBUNTU_IMAGE_ARGS${PROPOSED:+ --with-proposed}"
+			UBUNTU_IMAGE_ARGS="$UBUNTU_IMAGE_ARGS${EXTRA_PPAS:+ --extra-ppas \"$EXTRA_PPAS\"}"
+
+			git clone git://git.launchpad.net/~canonical-foundations/snap-$MODEL/+git/github-mirror -b classic config/$PREFIX-gadget
+
+			echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common
+			echo "SUITE=$SUITE" >> config/common
+			echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common
+
+			echo "Configured ubuntu-image for the following gadget model: $MODEL"
+		fi
 		# Fake finished configuration for lb build
 		mkdir -p .build
 		touch .build/config
@@ -751,7 +771,7 @@ case $PROJECT in
 esac
 
 case $ARCH in
-	armel|armhf)
+	armel|armhf|arm64)
 		KERNEL_FLAVOURS="${SUBARCH:-$KERNEL_FLAVOURS}"
 		case $SUBARCH in
 			dove)
@@ -789,6 +809,13 @@ case $ARCH in
 				add_package install linux-firmware-raspi2 u-boot-rpi flash-kernel u-boot-tools
 				BINARY_REMOVE_LINUX=false
 				;;
+			raspi3)
+				COMPONENTS='main restricted universe multiverse'
+				KERNEL_FLAVOURS=raspi2
+				add_package install raspi3-firmware u-boot-rpi flash-kernel u-boot-tools
+				BINARY_REMOVE_LINUX=false
+				;;
+
 		esac
 		;;
 esac
@@ -854,7 +881,7 @@ echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary
 echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/binary
 
 case $ARCH+$SUBARCH in
-	armhf+raspi2)
+	armhf+raspi2|armhf+raspi3|arm64+raspi3)
 		cat > config/hooks/01-firmware-directory.chroot_early <<EOF
 #!/bin/sh -ex
 mkdir -p /boot/firmware

live-build/functions

@@ -43,6 +43,17 @@ create_empty_disk_image() {
     dd if=/dev/zero of="$1" bs=1 count=0 seek="${imagesize}"
 }
 
+create_manifest() {
+    local chroot_root=${1}
+    local target_file=${2}
+    echo "create_manifest chroot_root: ${chroot_root}"
+    dpkg-query --show --admindir="${chroot_root}/var/lib/dpkg" > ${target_file}
+    echo "create_manifest call to dpkg-query finished."
+    ./config/snap-seed-parse "${chroot_root}" "${target_file}"
+    echo "create_manifest call to snap_seed_parse finished."
+    echo "create_manifest finished"
+}
+
 make_ext4_partition() {
     device="$1"
     label=${fs_label:+-L "${fs_label}"}

live-build/snap-seed-parse.py

@@ -0,0 +1,68 @@
+#!/usr/bin/python3
+ 
+"""
+Usage: snap-seed-parse [${chroot_dir}] <output file>
+ 
+This script looks for a seed.yaml path in the given root directory, parsing
+it and appending the parsed lines to the given output file.
+ 
+The $chroot_dir argument is optional and will default to the empty string.
+"""
+ 
+import argparse
+import os.path
+import re
+import yaml
+ 
+ 
+def log(msg):
+    print("snap-seed-parse: {}".format(msg))
+ 
+ 
+log("Parsing seed.yaml")
+ 
+parser = argparse.ArgumentParser()
+parser.add_argument('chroot', nargs='?', default='',
+                    help='root dir for the chroot from which to generate the '
+                         'manifest')
+parser.add_argument('file', help='Output manifest to this file')
+ 
+ARGS = parser.parse_args()
+CHROOT_ROOT = ARGS.chroot
+FNAME = ARGS.file
+ 
+# Trim any trailing slashes for correct appending
+log("CHROOT_ROOT: {}".format(CHROOT_ROOT))
+if len(CHROOT_ROOT) > 0 and CHROOT_ROOT[-1] == '/':
+    CHROOT_ROOT = CHROOT_ROOT[:-1]
+ 
+# This is where we expect to find the seed.yaml file
+YAML_PATH = CHROOT_ROOT + '/var/lib/snapd/seed/seed.yaml'
+ 
+# Snaps are prepended with this string in the manifest
+LINE_PREFIX = 'snap:'
+ 
+log("yaml path: {}".format(YAML_PATH))
+if not os.path.isfile(YAML_PATH):
+    log("WARNING: yaml path not found; no seeded snaps found.")
+    exit(0)
+else:
+    log("yaml path found.")
+ 
+with open(YAML_PATH, 'r') as fh:
+    yaml_lines = yaml.safe_load(fh)['snaps']
+ 
+log('Writing manifest to {}'.format(FNAME))
+ 
+with open(FNAME, 'a+') as fh:
+    for item in yaml_lines:
+        filestring = item['file']
+        # Pull the revision number off the file name
+        revision = filestring[filestring.rindex('_')+1:]
+        revision = re.sub(r'[^0-9]', '', revision)
+        fh.write("{}{}\t{}\t{}\n".format(LINE_PREFIX,
+                                         item['name'],
+                                         item['channel'],
+                                         revision,
+                                         ))
+log('Manifest output finished.')

live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary

@@ -26,4 +26,4 @@ mkdir -p $rootfs_dir/lib/modules
 
 teardown_mountpoint $rootfs_dir
 
-dpkg-query --admindir=$rootfs_dir/var/lib/dpkg -W > $rootfs_dir.manifest
+create_manifest "${rootfs_dir}" "${rootfs_dir}.manifest"

live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary

@@ -97,7 +97,7 @@ install_grub() {
     # grub-efi packages that otherwise would not make it into the base
     # manifest. filesystem.packages is moved into place via symlinking to
     # livecd.ubuntu-cpc.manifest by live-build/auto/build after lb_binary runs
-    dpkg-query --show --admindir="mountpoint/var/lib/dpkg" > "binary/boot/filesystem.packages"
+    create_manifest "mountpoint" "binary/boot/filesystem.packages"
 
     chroot mountpoint grub-install "${loop_device}" \
         --boot-directory=/boot \

live-build/ubuntu-server/hooks/030-root-squashfs.binary

@@ -28,7 +28,7 @@ cp -a chroot/* binary/boot/squashfs.dir
 squashfs_f="${PWD}/livecd.${PROJECT}.squashfs"
 squashfs_f_manifest="${squashfs_f}.manifest"
 
-dpkg-query --admindir=binary/boot/squashfs.dir/var/lib/dpkg -W > ${squashfs_f_manifest}
+create_manifest "binary/boot/squashfs.dir" "${squashfs_f_manifest}"
 
 (cd "binary/boot/squashfs.dir/" &&
       mksquashfs . ${squashfs_f} \