Imported 2.525.18

No reason for CPC update specified.

debian/changelog

@@ -1,3 +1,34 @@
+livecd-rootfs (2.525.18) bionic; urgency=medium
+
+  [ Michael Hudson-Doyle ]
+  * Changes to kernel handling for live-server: do not include kernel/initrd
+    in filesystem.squashfs but rather install it in a throwaway layer on top
+    of installer.squashfs and fish kernel, initrd and modules out of that with
+    an initrd hook that records kernel metapackage name in /run and mounts
+    /lib/modules from a squashfs on the ISO.
+  * Do no install openssh-server in the base filsystem for the live server
+    installer.
+  * A few simple tweaks to reduce size of live servers installer.squashfs:
+    - Do not run apt-get update (which can bring in package lists if we are
+      unlucky wrt publisher schedules).
+    - Run apt-get clean to clear out downloaded debs of curtin/casper and
+      dependencies.
+    - Do not install user-setup.
+    - Use the core snap from the base filesystem if present.
+  * Do not include curtin in the live-server installer.squashfs as the
+    version of subiquity that includes it in the snap has now been released to
+    stable.
+
+ -- Adam Conrad <adconrad@ubuntu.com>  Tue, 12 Feb 2019 11:35:48 -0700
+
+livecd-rootfs (2.525.17) bionic; urgency=medium
+
+  * Add a buildd subproject.
+  * Add a LXD image to builds for the buildd subproject.
+  * Move buildd image building to binary hooks.
+
+ -- Colin Watson <cjwatson@ubuntu.com>  Tue, 05 Feb 2019 14:55:06 +0000
+
 livecd-rootfs (2.525.16) bionic; urgency=medium
 
   * lubuntu: Select lubuntu-gtk-core task as well, so we get the HWE filter

debian/tests/default-bootstraps

@@ -25,6 +25,7 @@ ALL_TRIPLETS="
  mythbuntu::
  ubuntu::
  ubuntu-base::
+ ubuntu-base:buildd:
  ubuntu-budgie::
  ubuntu-budgie-desktop::
  ubuntu-budgie-live::

live-build/auto/build

@@ -17,7 +17,7 @@ fi
 
 . config/functions
 
-# Link output files somewhere BuildLiveCD will be able to find them.
+# Link output files somewhere launchpad-buildd will be able to find them.
 PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
 
 if [ "${IMAGEFORMAT:-}" = "ubuntu-image" ]; then

live-build/auto/config

@@ -162,20 +162,15 @@ case $IMAGEFORMAT in
 		;;
 
 	plain)
-		INITRAMFS_TYPE=none
 		case $PROJECT:${SUBPROJECT:-} in
 			ubuntu-server:live)
-				# Stop lb installing casper into filesystem.squashfs
-				#  by skipping lb_chroot_live-packages.
-				skip_lb_stage chroot_live-packages
-				INITRAMFS_TYPE=auto
 	                        touch config/universe-enabled
 				;;
 			*)
 				PREINSTALLED=true
 				;;
 		esac
-		OPTS="${OPTS:+$OPTS }--initramfs $INITRAMFS_TYPE --chroot-filesystem $IMAGEFORMAT"
+		OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT"
 		;;
 
 	ubuntu-image)
@@ -557,10 +552,7 @@ case $PROJECT in
 			live)
 				add_task install standard
 				add_task install server
-				LIVE_TASK='cloud-image'
-				case $ARCH in
-					amd64)	add_package live linux-signed-generic ;;
-				esac
+				add_package install cloud-init
 				;;
 		esac
 		COMPONENTS='main'
@@ -716,6 +708,37 @@ case $PROJECT in
 		;;
 esac
 
+case $SUBPROJECT in
+	buildd)
+		OPTS="${OPTS:+$OPTS }--archive-areas main"
+		COMPONENTS='main restricted universe multiverse'
+		OPTS="${OPTS:+$OPTS }--apt-recommends false"
+		OPTS="${OPTS:+$OPTS }--apt-secure false"
+		OPTS="${OPTS:+$OPTS }--parent-mirror-binary ${MIRROR}"
+		# XXX cjwatson 2018-04-27: We need to work out how to make
+		# this conditional so that we can do things like building
+		# buildd chroots with -updates.  This probably involves
+		# either extending the PROPOSED hack or fixing the strange
+		# way that SUITE is in fact a series; in either case it's
+		# likely to involve work both here and in launchpad-buildd.
+		OPTS="${OPTS:+$OPTS }--security false --volatile false"
+
+		add_package install adduser
+		add_package install policyrcd-script-zg2
+		add_package install pkgbinarymangler
+		add_package install ca-certificates
+		add_package install gpg
+		add_package install gpg-agent
+		add_package install tzdata
+		add_package install fakeroot
+		add_package install build-essential
+		# Needed for LXD-based builds.
+		add_package install init
+
+		cp -af /usr/share/livecd-rootfs/live-build/make-lxd-metadata.py config/make-lxd-metadata
+		;;
+esac
+
 # we'll expand the base seed given here according to the STRUCTURE file, and
 # then look in all of the seeds found to see which snaps are seeded
 case $PROJECT:${SUBPROJECT:-} in
@@ -843,8 +866,6 @@ case $ARCH in
 esac
 
 case $PROJECT:${SUBPROJECT:-} in
-	ubuntu-server:live)
-		;;
 	ubuntu-server:*|ubuntu-base:*|ubuntu-touch:*|ubuntu-touch-custom:*)
 		OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none"
 		KERNEL_FLAVOURS=none
@@ -1038,6 +1059,12 @@ EOF
 		;;
 esac
 
+case $SUBPROJECT in
+	buildd)
+		cp -af /usr/share/livecd-rootfs/live-build/buildd/* config/
+		;;
+esac
+
 if [ "$EXTRA_PPAS" ]; then
 	rm -f config/archives/extra-ppas.list.chroot \
 	      config/archives/extra-ppas.pref.chroot \

live-build/buildd/hooks/00-kernel-img.chroot

@@ -0,0 +1,5 @@
+#! /bin/sh
+set -e
+
+# At one point, kernel builds needed this.
+echo do_initrd = Yes >>/etc/kernel-img.conf

live-build/buildd/hooks/00-mirror.binary

@@ -0,0 +1,12 @@
+#! /bin/sh
+set -e
+
+. config/bootstrap
+
+# Use a public-facing mirror URL, for the benefit of
+# sbuild-launchpad-chroot.  We deliberately do this only after live-build
+# has run "apt-get update" for the last time, in order that
+# /var/lib/apt/lists/ has suitable cached Packages files; this speeds up
+# builds on buildds.
+sed -i "s,${LB_PARENT_MIRROR_BINARY},${LB_MIRROR_BINARY},g" \
+	binary/etc/apt/sources.list

live-build/buildd/hooks/01-pkgbinarymangler.chroot

@@ -0,0 +1,10 @@
+#! /bin/sh
+set -e
+
+# Configure pkgbinarymangler.
+sed -i /^enable/s/false/true/ \
+	/etc/pkgbinarymangler/maintainermangler.conf \
+	/etc/pkgbinarymangler/striptranslations.conf || true
+sed -i /^invalid_current/s/ignore/fail/ \
+	/etc/pkgbinarymangler/maintainermangler.conf \
+	/etc/pkgbinarymangler/striptranslations.conf || true

live-build/buildd/hooks/02-user.chroot

@@ -0,0 +1,9 @@
+#! /bin/sh
+set -e
+
+# Create the buildd user and group.
+addgroup --gid 2501 buildd
+adduser --system --disabled-password --gecos 'Build Daemon user' \
+	--ingroup buildd --uid 2001 --shell /bin/bash buildd
+mkdir -p /build/buildd
+chown buildd:buildd /build/buildd

live-build/buildd/hooks/50-buildd-tar.binary

@@ -0,0 +1,10 @@
+#! /bin/sh
+# A few things (launchpad-buildd, sbuild-launchpad-chroot) rely on the
+# top-level directory being "chroot-autobuild", so we have to do this
+# ourselves.
+set -e
+
+# gzip was chosen for fastest decompression speed: it decompresses buildd
+# chroots about twice as fast as xz and about five times as fast as bzip2.
+tar --transform='s,^binary,chroot-autobuild,' --sort=name --numeric-owner \
+	-czf "livecd.$PROJECT.rootfs.tar.gz" binary

live-build/buildd/hooks/51-buildd-lxd.binary

@@ -0,0 +1,16 @@
+#! /bin/sh
+# Some build types prefer a LXD image over a traditional chroot tarball.
+set -e
+
+. config/bootstrap
+
+TMPDIR="$(mktemp -d)"
+config/make-lxd-metadata "${LB_DISTRIBUTION%-*}" "$ARCH" \
+	>"$TMPDIR/metadata.yaml"
+tar --numeric-owner -cf "livecd.$PROJECT.lxd.tar" -C "$TMPDIR" metadata.yaml
+rm -rf "$TMPDIR"
+# When using the combined metadata/rootfs form, the rootfs must be under
+# rootfs/ rather than under chroot-autobuild/.
+tar --transform='s,^binary,rootfs,' --sort=name --numeric-owner \
+	-rf "livecd.$PROJECT.lxd.tar" binary
+gzip -9 "livecd.$PROJECT.lxd.tar"

live-build/buildd/includes.chroot/etc/apt/apt.conf.d/99buildd

@@ -0,0 +1,2 @@
+DPkg::Options {"--force-unsafe-io";};
+DPkg::Use-Pty "false";

live-build/buildd/includes.chroot/etc/apt/preferences.d/backports

@@ -0,0 +1,3 @@
+Package: *
+Pin: release a=*-backports
+Pin-Priority: 500

live-build/buildd/includes.chroot/etc/fstab

@@ -0,0 +1 @@
+/dev/root / ext2 noatime,errors=remount-ro 0 1

live-build/buildd/includes.chroot/etc/hostname

@@ -0,0 +1 @@
+INVALID

live-build/buildd/includes.chroot/etc/hosts

@@ -0,0 +1,9 @@
+127.0.0.1	localhost.localdomain	localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts

live-build/buildd/includes.chroot/usr/local/sbin/policy-rc.d

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# policy-rc.d script for chroots.
+# Copyright (c) 2007 Peter Palfrader <peter@palfrader.org>
+# License: <weasel> MIT, if you want one.
+
+while true; do
+  case "$1" in
+    -*)         shift ;;
+    makedev)    exit 0;;
+    *) echo "Not running services in chroot."; exit 101 ;;
+  esac
+done

live-build/buildd/preseed/debconf.preseed

@@ -0,0 +1,2 @@
+# We never want debconf interaction.
+debconf debconf/frontend select Noninteractive

live-build/buildd/preseed/man-db.preseed

@@ -0,0 +1,3 @@
+# Avoid unnecessary manual page database builds (see
+# https://bugs.debian.org/554914).
+man-db man-db/auto-update boolean false

live-build/buildd/preseed/sun-java6.preseed

@@ -0,0 +1,3 @@
+# Pre-accept interactive EULA prompts.
+sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true
+sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true

live-build/make-lxd-metadata.py

@@ -0,0 +1,49 @@
+#! /usr/bin/python3
+
+"""Make a metadata.yaml file for a LXD image."""
+
+import argparse
+import json
+import sys
+import time
+
+
+# Map dpkg architecture names to LXD architecture names.
+lxd_arches = {
+    "amd64": "x86_64",
+    "arm64": "aarch64",
+    "armhf": "armv7l",
+    "i386": "i686",
+    "powerpc": "ppc",
+    "ppc64el": "ppc64le",
+    "s390x": "s390x",
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("series", help="Ubuntu series name")
+    parser.add_argument("architecture", help="Ubuntu architecture name")
+    args = parser.parse_args()
+
+    metadata = {
+        "architecture": lxd_arches[args.architecture],
+        "creation_date": int(time.time()),
+        "properties": {
+            "os": "Ubuntu",
+            "series": args.series,
+            "architecture": args.architecture,
+            "description": "Ubuntu buildd %s %s" % (
+                args.series, args.architecture),
+            },
+        }
+
+    # Encoding this as JSON is good enough, and saves pulling in a YAML
+    # library dependency.
+    json.dump(
+        metadata, sys.stdout, sort_keys=True, indent=4, separators=(",", ": "),
+        ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

live-build/ubuntu-server/hooks/032-installer-squashfs.binary

@@ -24,13 +24,14 @@ fi
 . config/functions
 . config/common
 
-SQUASH_ROOT=binary/boot/squashfs.dir
+FILESYSTEM_ROOT=binary/boot/squashfs.dir
+INSTALLER_ROOT=binary/boot/installer.squashfs.dir
 OVERLAY_ROOT=binary/overlay
 
-mkdir -p "$OVERLAY_ROOT"
+mkdir -p "$INSTALLER_ROOT" "$OVERLAY_ROOT"
 
 # Create an installer squashfs layer
-mount_overlay "$SQUASH_ROOT/" "$OVERLAY_ROOT/" "$SQUASH_ROOT/"
+mount_overlay "$FILESYSTEM_ROOT/" "$OVERLAY_ROOT/" "$INSTALLER_ROOT/"
 
 setup_mountpoint binary/boot/squashfs.dir
 
@@ -40,42 +41,40 @@ setup_mountpoint binary/boot/squashfs.dir
 # It would be better to have this in ../includes.binary/overlay but
 # you can't have backslashes in filenames in bzr branches!
 DEVICE_UNIT_NAME='dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device'
-mkdir -p "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d"
-cat > "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" <<EOF
+mkdir -p "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d"
+cat > "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" <<EOF
 [Unit]
 JobRunningTimeoutSec=0s
 EOF
 
 # Prepare installer layer.
 
-# Install any requirements for the installer, for things we don't want
-# to see on the installed system
-chroot $SQUASH_ROOT apt-get update
-chroot $SQUASH_ROOT apt-get -y install user-setup curtin lupin-casper
+# Install casper for live session magic.
+chroot $INSTALLER_ROOT apt-get -y install lupin-casper
+chroot $INSTALLER_ROOT apt-get clean
 
 # For bug #1743643 "Install to dirty disk with swap fails" remove the
 # "helpful" casper script that mounts any swap partitions it finds.
-rm -f $SQUASH_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
-
-# Installing casper means we need a new initramfs
-UPDATE_INITRAMFS_OPTIONS=CASPER_GENERATE_UUID=1 recreate_initramfs $SQUASH_ROOT
+rm -f $INSTALLER_ROOT/usr/share/initramfs-tools/scripts/casper-bottom/*swap
 
 # Don't let cloud-init run in the live session.
-touch $SQUASH_ROOT/etc/cloud/cloud-init.disabled
+touch $INSTALLER_ROOT/etc/cloud/cloud-init.disabled
 
 # Do the snap seeding dance.
-chroot $SQUASH_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions
-chroot $SQUASH_ROOT sh -c '
+chroot $INSTALLER_ROOT mkdir -p /var/lib/snapd/seed/snaps /var/lib/snapd/seed/assertions
+chroot $INSTALLER_ROOT sh -c '
 set -x;
-cd /var/lib/snapd/seed;
-sudo SNAPPY_STORE_NO_CDN=1 snap download core;
-sudo SNAPPY_STORE_NO_CDN=1 snap download subiquity;
+mkdir -p /var/lib/snapd/seed/snaps/
+cd /var/lib/snapd/seed/snaps/;
+if [ ! -e core_*.snap ]; then
+	SNAPPY_STORE_NO_CDN=1 snap download core
+fi
+SNAPPY_STORE_NO_CDN=1 snap download subiquity;
 
-CORE_SNAP=$(ls -1 core*.snap);
+CORE_SNAP=$(ls -1 core_*.snap);
 SUBIQUITY_SNAP=$(ls -1 subiquity*.snap);
 
 mv *.assert /var/lib/snapd/seed/assertions/;
-mv *.snap /var/lib/snapd/seed/snaps/;
 
 cat <<EOF > /var/lib/snapd/seed/seed.yaml
 snaps:
@@ -89,10 +88,7 @@ snaps:
 EOF
 '
 
-teardown_mountpoint "$SQUASH_ROOT"
-
-# Then unmount the overlay
-umount "$SQUASH_ROOT"
+teardown_mountpoint "$INSTALLER_ROOT"
 
 squashfs_f="${PWD}/livecd.${PROJECT}.installer.squashfs"
 

live-build/ubuntu-server/hooks/033-kernel-bits.binary

@@ -0,0 +1,90 @@
+#!/bin/bash -eux
+# vi: ts=4 noexpandtab
+#
+# Generate a squashfs root and manifest
+
+echo "033-kernel-bits.binary"
+
+case ${IMAGE_TARGETS-} in
+	""|*squashfs*)
+		;;
+	*)
+		echo "Skipping squashfs build"
+		exit 0
+		;;
+esac
+
+if [ -n "${SUBARCH-}" ]; then
+	echo "Skipping rootfs build for subarch flavor build"
+	exit 0
+fi
+
+. config/functions
+. config/common
+
+INSTALLER_ROOT=binary/boot/installer.squashfs.dir
+
+KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir
+KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits
+
+variants='ga hwe'
+
+for variant in $variants; do
+	if [ "$variant" = "ga" ]; then
+		kernel_metapkg=linux-generic
+		flavor=generic
+	elif [ "$variant" = "hwe" ]; then
+		kernel_metapkg=linux-generic-hwe-$(lsb_release -sr)
+		flavor=generic-hwe
+	else
+		echo "bogus variant: $variant"
+		exit 1
+	fi
+
+	# Make preparations
+	mkdir -p $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
+	mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/"
+	setup_mountpoint $KERNEL_BITS_ROOT
+
+	# Our initramfs hook implements a kind of extreme version of
+	# cloud-initramfs-copymods, so remove that and prevent duelling hooks
+	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y remove cloud-initramfs-copymods ubuntu-server || true
+
+	# Configure initramfs creation
+	mkdir -p "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/
+	if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then
+		echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf
+	fi
+	echo "CASPER_GENERATE_UUID=1" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/casper.conf
+
+	# Add a hook to record which kernel was booted and mount the
+	# modules.squashfs created below.
+	cat <<EOF > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
+#!/bin/sh
+case \$1 in
+prereqs) exit 0;;
+esac
+
+echo ${kernel_metapkg} > /run/kernel-meta-package
+mkdir -p \$rootmnt/lib/modules
+mount \$rootmnt/cdrom/casper/extras/modules.squashfs-$flavor \$rootmnt/lib/modules
+EOF
+	chmod +x "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server
+
+	# Install the kernel!
+	env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg}
+
+	# Fish out generated kernel image and initrd
+	mv "$KERNEL_BITS_ROOT"/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-$flavor
+	mv "$KERNEL_BITS_ROOT"/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-$flavor
+
+	# Create squashfs containing all the modules
+	modules_squashfs_path="${PWD}/livecd.${PROJECT}.modules.squashfs-$flavor"
+	(cd "$KERNEL_BITS_ROOT/lib/modules" &&
+		mksquashfs . $modules_squashfs_path -no-progress -xattrs -comp xz)
+
+	# And clean up
+	teardown_mountpoint $KERNEL_BITS_ROOT
+	umount $KERNEL_BITS_ROOT
+	rm -rf $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY
+done