diff --git a/debian/changelog b/debian/changelog index 465afe9e..783c0747 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,34 @@ +livecd-rootfs (2.525.18) bionic; urgency=medium + + [ Michael Hudson-Doyle ] + * Changes to kernel handling for live-server: do not include kernel/initrd + in filesystem.squashfs but rather install it in a throwaway layer on top + of installer.squashfs and fish kernel, initrd and modules out of that with + an initrd hook that records kernel metapackage name in /run and mounts + /lib/modules from a squashfs on the ISO. + * Do no install openssh-server in the base filsystem for the live server + installer. + * A few simple tweaks to reduce size of live servers installer.squashfs: + - Do not run apt-get update (which can bring in package lists if we are + unlucky wrt publisher schedules). + - Run apt-get clean to clear out downloaded debs of curtin/casper and + dependencies. + - Do not install user-setup. + - Use the core snap from the base filesystem if present. + * Do not include curtin in the live-server installer.squashfs as the + version of subiquity that includes it in the snap has now been released to + stable. + + -- Adam Conrad Tue, 12 Feb 2019 11:35:48 -0700 + +livecd-rootfs (2.525.17) bionic; urgency=medium + + * Add a buildd subproject. + * Add a LXD image to builds for the buildd subproject. + * Move buildd image building to binary hooks. + + -- Colin Watson Tue, 05 Feb 2019 14:55:06 +0000 + livecd-rootfs (2.525.16) bionic; urgency=medium * lubuntu: Select lubuntu-gtk-core task as well, so we get the HWE filter diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps index 433cd327..a96d0b67 100755 --- a/debian/tests/default-bootstraps +++ b/debian/tests/default-bootstraps @@ -25,6 +25,7 @@ ALL_TRIPLETS=" mythbuntu:: ubuntu:: ubuntu-base:: + ubuntu-base:buildd: ubuntu-budgie:: ubuntu-budgie-desktop:: ubuntu-budgie-live:: diff --git a/live-build/auto/build b/live-build/auto/build index 7840524b..1912ab5b 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -17,7 +17,7 @@ fi . config/functions -# Link output files somewhere BuildLiveCD will be able to find them. +# Link output files somewhere launchpad-buildd will be able to find them. PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" if [ "${IMAGEFORMAT:-}" = "ubuntu-image" ]; then diff --git a/live-build/auto/config b/live-build/auto/config index 2b8fa8fc..f29a63e0 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -162,20 +162,15 @@ case $IMAGEFORMAT in ;; plain) - INITRAMFS_TYPE=none case $PROJECT:${SUBPROJECT:-} in ubuntu-server:live) - # Stop lb installing casper into filesystem.squashfs - # by skipping lb_chroot_live-packages. - skip_lb_stage chroot_live-packages - INITRAMFS_TYPE=auto touch config/universe-enabled ;; *) PREINSTALLED=true ;; esac - OPTS="${OPTS:+$OPTS }--initramfs $INITRAMFS_TYPE --chroot-filesystem $IMAGEFORMAT" + OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT" ;; ubuntu-image) @@ -557,10 +552,7 @@ case $PROJECT in live) add_task install standard add_task install server - LIVE_TASK='cloud-image' - case $ARCH in - amd64) add_package live linux-signed-generic ;; - esac + add_package install cloud-init ;; esac COMPONENTS='main' @@ -716,6 +708,37 @@ case $PROJECT in ;; esac +case $SUBPROJECT in + buildd) + OPTS="${OPTS:+$OPTS }--archive-areas main" + COMPONENTS='main restricted universe multiverse' + OPTS="${OPTS:+$OPTS }--apt-recommends false" + OPTS="${OPTS:+$OPTS }--apt-secure false" + OPTS="${OPTS:+$OPTS }--parent-mirror-binary ${MIRROR}" + # XXX cjwatson 2018-04-27: We need to work out how to make + # this conditional so that we can do things like building + # buildd chroots with -updates. This probably involves + # either extending the PROPOSED hack or fixing the strange + # way that SUITE is in fact a series; in either case it's + # likely to involve work both here and in launchpad-buildd. + OPTS="${OPTS:+$OPTS }--security false --volatile false" + + add_package install adduser + add_package install policyrcd-script-zg2 + add_package install pkgbinarymangler + add_package install ca-certificates + add_package install gpg + add_package install gpg-agent + add_package install tzdata + add_package install fakeroot + add_package install build-essential + # Needed for LXD-based builds. + add_package install init + + cp -af /usr/share/livecd-rootfs/live-build/make-lxd-metadata.py config/make-lxd-metadata + ;; +esac + # we'll expand the base seed given here according to the STRUCTURE file, and # then look in all of the seeds found to see which snaps are seeded case $PROJECT:${SUBPROJECT:-} in @@ -843,8 +866,6 @@ case $ARCH in esac case $PROJECT:${SUBPROJECT:-} in - ubuntu-server:live) - ;; ubuntu-server:*|ubuntu-base:*|ubuntu-touch:*|ubuntu-touch-custom:*) OPTS="${OPTS:+$OPTS }--linux-packages=none --initramfs=none" KERNEL_FLAVOURS=none @@ -1038,6 +1059,12 @@ EOF ;; esac +case $SUBPROJECT in + buildd) + cp -af /usr/share/livecd-rootfs/live-build/buildd/* config/ + ;; +esac + if [ "$EXTRA_PPAS" ]; then rm -f config/archives/extra-ppas.list.chroot \ config/archives/extra-ppas.pref.chroot \ diff --git a/live-build/buildd/hooks/00-kernel-img.chroot b/live-build/buildd/hooks/00-kernel-img.chroot new file mode 100755 index 00000000..3517b443 --- /dev/null +++ b/live-build/buildd/hooks/00-kernel-img.chroot @@ -0,0 +1,5 @@ +#! /bin/sh +set -e + +# At one point, kernel builds needed this. +echo do_initrd = Yes >>/etc/kernel-img.conf diff --git a/live-build/buildd/hooks/00-mirror.binary b/live-build/buildd/hooks/00-mirror.binary new file mode 100755 index 00000000..7f3cdba4 --- /dev/null +++ b/live-build/buildd/hooks/00-mirror.binary @@ -0,0 +1,12 @@ +#! /bin/sh +set -e + +. config/bootstrap + +# Use a public-facing mirror URL, for the benefit of +# sbuild-launchpad-chroot. We deliberately do this only after live-build +# has run "apt-get update" for the last time, in order that +# /var/lib/apt/lists/ has suitable cached Packages files; this speeds up +# builds on buildds. +sed -i "s,${LB_PARENT_MIRROR_BINARY},${LB_MIRROR_BINARY},g" \ + binary/etc/apt/sources.list diff --git a/live-build/buildd/hooks/01-pkgbinarymangler.chroot b/live-build/buildd/hooks/01-pkgbinarymangler.chroot new file mode 100755 index 00000000..ab901957 --- /dev/null +++ b/live-build/buildd/hooks/01-pkgbinarymangler.chroot @@ -0,0 +1,10 @@ +#! /bin/sh +set -e + +# Configure pkgbinarymangler. +sed -i /^enable/s/false/true/ \ + /etc/pkgbinarymangler/maintainermangler.conf \ + /etc/pkgbinarymangler/striptranslations.conf || true +sed -i /^invalid_current/s/ignore/fail/ \ + /etc/pkgbinarymangler/maintainermangler.conf \ + /etc/pkgbinarymangler/striptranslations.conf || true diff --git a/live-build/buildd/hooks/02-user.chroot b/live-build/buildd/hooks/02-user.chroot new file mode 100755 index 00000000..86000bf2 --- /dev/null +++ b/live-build/buildd/hooks/02-user.chroot @@ -0,0 +1,9 @@ +#! /bin/sh +set -e + +# Create the buildd user and group. +addgroup --gid 2501 buildd +adduser --system --disabled-password --gecos 'Build Daemon user' \ + --ingroup buildd --uid 2001 --shell /bin/bash buildd +mkdir -p /build/buildd +chown buildd:buildd /build/buildd diff --git a/live-build/buildd/hooks/50-buildd-tar.binary b/live-build/buildd/hooks/50-buildd-tar.binary new file mode 100755 index 00000000..8be7b618 --- /dev/null +++ b/live-build/buildd/hooks/50-buildd-tar.binary @@ -0,0 +1,10 @@ +#! /bin/sh +# A few things (launchpad-buildd, sbuild-launchpad-chroot) rely on the +# top-level directory being "chroot-autobuild", so we have to do this +# ourselves. +set -e + +# gzip was chosen for fastest decompression speed: it decompresses buildd +# chroots about twice as fast as xz and about five times as fast as bzip2. +tar --transform='s,^binary,chroot-autobuild,' --sort=name --numeric-owner \ + -czf "livecd.$PROJECT.rootfs.tar.gz" binary diff --git a/live-build/buildd/hooks/51-buildd-lxd.binary b/live-build/buildd/hooks/51-buildd-lxd.binary new file mode 100755 index 00000000..b4b782d9 --- /dev/null +++ b/live-build/buildd/hooks/51-buildd-lxd.binary @@ -0,0 +1,16 @@ +#! /bin/sh +# Some build types prefer a LXD image over a traditional chroot tarball. +set -e + +. config/bootstrap + +TMPDIR="$(mktemp -d)" +config/make-lxd-metadata "${LB_DISTRIBUTION%-*}" "$ARCH" \ + >"$TMPDIR/metadata.yaml" +tar --numeric-owner -cf "livecd.$PROJECT.lxd.tar" -C "$TMPDIR" metadata.yaml +rm -rf "$TMPDIR" +# When using the combined metadata/rootfs form, the rootfs must be under +# rootfs/ rather than under chroot-autobuild/. +tar --transform='s,^binary,rootfs,' --sort=name --numeric-owner \ + -rf "livecd.$PROJECT.lxd.tar" binary +gzip -9 "livecd.$PROJECT.lxd.tar" diff --git a/live-build/buildd/includes.chroot/etc/apt/apt.conf.d/99buildd b/live-build/buildd/includes.chroot/etc/apt/apt.conf.d/99buildd new file mode 100644 index 00000000..0fd77978 --- /dev/null +++ b/live-build/buildd/includes.chroot/etc/apt/apt.conf.d/99buildd @@ -0,0 +1,2 @@ +DPkg::Options {"--force-unsafe-io";}; +DPkg::Use-Pty "false"; diff --git a/live-build/buildd/includes.chroot/etc/apt/preferences.d/backports b/live-build/buildd/includes.chroot/etc/apt/preferences.d/backports new file mode 100644 index 00000000..605907ec --- /dev/null +++ b/live-build/buildd/includes.chroot/etc/apt/preferences.d/backports @@ -0,0 +1,3 @@ +Package: * +Pin: release a=*-backports +Pin-Priority: 500 diff --git a/live-build/buildd/includes.chroot/etc/fstab b/live-build/buildd/includes.chroot/etc/fstab new file mode 100644 index 00000000..b67dd265 --- /dev/null +++ b/live-build/buildd/includes.chroot/etc/fstab @@ -0,0 +1 @@ +/dev/root / ext2 noatime,errors=remount-ro 0 1 diff --git a/live-build/buildd/includes.chroot/etc/hostname b/live-build/buildd/includes.chroot/etc/hostname new file mode 100644 index 00000000..e420fe4d --- /dev/null +++ b/live-build/buildd/includes.chroot/etc/hostname @@ -0,0 +1 @@ +INVALID diff --git a/live-build/buildd/includes.chroot/etc/hosts b/live-build/buildd/includes.chroot/etc/hosts new file mode 100644 index 00000000..7a0cb5d4 --- /dev/null +++ b/live-build/buildd/includes.chroot/etc/hosts @@ -0,0 +1,9 @@ +127.0.0.1 localhost.localdomain localhost + +# The following lines are desirable for IPv6 capable hosts +::1 ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +ff02::3 ip6-allhosts diff --git a/live-build/buildd/includes.chroot/etc/resolv.conf b/live-build/buildd/includes.chroot/etc/resolv.conf new file mode 100644 index 00000000..e69de29b diff --git a/live-build/buildd/includes.chroot/usr/local/sbin/policy-rc.d b/live-build/buildd/includes.chroot/usr/local/sbin/policy-rc.d new file mode 100755 index 00000000..cf2290b6 --- /dev/null +++ b/live-build/buildd/includes.chroot/usr/local/sbin/policy-rc.d @@ -0,0 +1,13 @@ +#!/bin/sh + +# policy-rc.d script for chroots. +# Copyright (c) 2007 Peter Palfrader +# License: MIT, if you want one. + +while true; do + case "$1" in + -*) shift ;; + makedev) exit 0;; + *) echo "Not running services in chroot."; exit 101 ;; + esac +done diff --git a/live-build/buildd/preseed/debconf.preseed b/live-build/buildd/preseed/debconf.preseed new file mode 100644 index 00000000..4e8c607b --- /dev/null +++ b/live-build/buildd/preseed/debconf.preseed @@ -0,0 +1,2 @@ +# We never want debconf interaction. +debconf debconf/frontend select Noninteractive diff --git a/live-build/buildd/preseed/man-db.preseed b/live-build/buildd/preseed/man-db.preseed new file mode 100644 index 00000000..7281ae4d --- /dev/null +++ b/live-build/buildd/preseed/man-db.preseed @@ -0,0 +1,3 @@ +# Avoid unnecessary manual page database builds (see +# https://bugs.debian.org/554914). +man-db man-db/auto-update boolean false diff --git a/live-build/buildd/preseed/sun-java6.preseed b/live-build/buildd/preseed/sun-java6.preseed new file mode 100644 index 00000000..53011956 --- /dev/null +++ b/live-build/buildd/preseed/sun-java6.preseed @@ -0,0 +1,3 @@ +# Pre-accept interactive EULA prompts. +sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true +sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true diff --git a/live-build/make-lxd-metadata.py b/live-build/make-lxd-metadata.py new file mode 100755 index 00000000..896f811a --- /dev/null +++ b/live-build/make-lxd-metadata.py @@ -0,0 +1,49 @@ +#! /usr/bin/python3 + +"""Make a metadata.yaml file for a LXD image.""" + +import argparse +import json +import sys +import time + + +# Map dpkg architecture names to LXD architecture names. +lxd_arches = { + "amd64": "x86_64", + "arm64": "aarch64", + "armhf": "armv7l", + "i386": "i686", + "powerpc": "ppc", + "ppc64el": "ppc64le", + "s390x": "s390x", + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("series", help="Ubuntu series name") + parser.add_argument("architecture", help="Ubuntu architecture name") + args = parser.parse_args() + + metadata = { + "architecture": lxd_arches[args.architecture], + "creation_date": int(time.time()), + "properties": { + "os": "Ubuntu", + "series": args.series, + "architecture": args.architecture, + "description": "Ubuntu buildd %s %s" % ( + args.series, args.architecture), + }, + } + + # Encoding this as JSON is good enough, and saves pulling in a YAML + # library dependency. + json.dump( + metadata, sys.stdout, sort_keys=True, indent=4, separators=(",", ": "), + ensure_ascii=False) + + +if __name__ == "__main__": + main() diff --git a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary index bab6ab76..518fc724 100755 --- a/live-build/ubuntu-server/hooks/032-installer-squashfs.binary +++ b/live-build/ubuntu-server/hooks/032-installer-squashfs.binary @@ -24,13 +24,14 @@ fi . config/functions . config/common -SQUASH_ROOT=binary/boot/squashfs.dir +FILESYSTEM_ROOT=binary/boot/squashfs.dir +INSTALLER_ROOT=binary/boot/installer.squashfs.dir OVERLAY_ROOT=binary/overlay -mkdir -p "$OVERLAY_ROOT" +mkdir -p "$INSTALLER_ROOT" "$OVERLAY_ROOT" # Create an installer squashfs layer -mount_overlay "$SQUASH_ROOT/" "$OVERLAY_ROOT/" "$SQUASH_ROOT/" +mount_overlay "$FILESYSTEM_ROOT/" "$OVERLAY_ROOT/" "$INSTALLER_ROOT/" setup_mountpoint binary/boot/squashfs.dir @@ -40,42 +41,40 @@ setup_mountpoint binary/boot/squashfs.dir # It would be better to have this in ../includes.binary/overlay but # you can't have backslashes in filenames in bzr branches! DEVICE_UNIT_NAME='dev-disk-by\x2duuid-00c629d6\x2d06ab\x2d4dfd\x2db21e\x2dc3186f34105d.device' -mkdir -p "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d" -cat > "$SQUASH_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" < "$INSTALLER_ROOT/etc/systemd/system/$DEVICE_UNIT_NAME.d/override.conf" < /var/lib/snapd/seed/seed.yaml snaps: @@ -89,10 +88,7 @@ snaps: EOF ' -teardown_mountpoint "$SQUASH_ROOT" - -# Then unmount the overlay -umount "$SQUASH_ROOT" +teardown_mountpoint "$INSTALLER_ROOT" squashfs_f="${PWD}/livecd.${PROJECT}.installer.squashfs" diff --git a/live-build/ubuntu-server/hooks/033-kernel-bits.binary b/live-build/ubuntu-server/hooks/033-kernel-bits.binary new file mode 100644 index 00000000..0d27344a --- /dev/null +++ b/live-build/ubuntu-server/hooks/033-kernel-bits.binary @@ -0,0 +1,90 @@ +#!/bin/bash -eux +# vi: ts=4 noexpandtab +# +# Generate a squashfs root and manifest + +echo "033-kernel-bits.binary" + +case ${IMAGE_TARGETS-} in + ""|*squashfs*) + ;; + *) + echo "Skipping squashfs build" + exit 0 + ;; +esac + +if [ -n "${SUBARCH-}" ]; then + echo "Skipping rootfs build for subarch flavor build" + exit 0 +fi + +. config/functions +. config/common + +INSTALLER_ROOT=binary/boot/installer.squashfs.dir + +KERNEL_BITS_ROOT=binary/boot/kernel-bits.dir +KERNEL_BITS_OVERLAY=binary/boot/overlay-kernel-bits + +variants='ga hwe' + +for variant in $variants; do + if [ "$variant" = "ga" ]; then + kernel_metapkg=linux-generic + flavor=generic + elif [ "$variant" = "hwe" ]; then + kernel_metapkg=linux-generic-hwe-$(lsb_release -sr) + flavor=generic-hwe + else + echo "bogus variant: $variant" + exit 1 + fi + + # Make preparations + mkdir -p $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY + mount_overlay "$INSTALLER_ROOT/" "$KERNEL_BITS_OVERLAY/" "$KERNEL_BITS_ROOT/" + setup_mountpoint $KERNEL_BITS_ROOT + + # Our initramfs hook implements a kind of extreme version of + # cloud-initramfs-copymods, so remove that and prevent duelling hooks + env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y remove cloud-initramfs-copymods ubuntu-server || true + + # Configure initramfs creation + mkdir -p "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/ + if [ -n "$LB_INITRAMFS_COMPRESSION" ]; then + echo "COMPRESS=$LB_INITRAMFS_COMPRESSION" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/livecd-rootfs.conf + fi + echo "CASPER_GENERATE_UUID=1" > "$KERNEL_BITS_ROOT"/etc/initramfs-tools/conf.d/casper.conf + + # Add a hook to record which kernel was booted and mount the + # modules.squashfs created below. + cat < "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server +#!/bin/sh +case \$1 in +prereqs) exit 0;; +esac + +echo ${kernel_metapkg} > /run/kernel-meta-package +mkdir -p \$rootmnt/lib/modules +mount \$rootmnt/cdrom/casper/extras/modules.squashfs-$flavor \$rootmnt/lib/modules +EOF + chmod +x "$KERNEL_BITS_ROOT"/etc/initramfs-tools/scripts/init-bottom/live-server + + # Install the kernel! + env DEBIAN_FRONTEND=noninteractive chroot $KERNEL_BITS_ROOT apt-get -y install ${kernel_metapkg} + + # Fish out generated kernel image and initrd + mv "$KERNEL_BITS_ROOT"/boot/initrd.img-* ${PWD}/livecd.${PROJECT}.initrd-$flavor + mv "$KERNEL_BITS_ROOT"/boot/vmlinu?-* ${PWD}/livecd.${PROJECT}.kernel-$flavor + + # Create squashfs containing all the modules + modules_squashfs_path="${PWD}/livecd.${PROJECT}.modules.squashfs-$flavor" + (cd "$KERNEL_BITS_ROOT/lib/modules" && + mksquashfs . $modules_squashfs_path -no-progress -xattrs -comp xz) + + # And clean up + teardown_mountpoint $KERNEL_BITS_ROOT + umount $KERNEL_BITS_ROOT + rm -rf $KERNEL_BITS_ROOT $KERNEL_BITS_OVERLAY +done