Merge latest prerequisite branch

debian/changelog

@@ -1,12 +1,65 @@
-livecd-rootfs (2.537) UNRELEASED; urgency=medium
+livecd-rootfs (2.543) UNRELEASED; urgency=medium
 
+  [ Cody Shepherd ]
+  * Ensure pre-seeded snaps are now published in the image manifests.
+  * Include grub efi packages in manifests for uefi images.
+
+  [ Aleksandr Bogdanov ]
+  * vagrant: disabling automatic console log file
+
+  [ Łukasz 'sil2100' Zemczak ]
   * Avoid issues of hard-linking to a symbolic vmlinuz as this can lead to a
     dangling symlink.
   * Add support for raspi3 rootfs builds (based on Ryan Finnie).
   * For ubuntu-image consumption, export the kernel and initrd to
     image/boot/uboot for raspi*.
 
- -- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com>  Tue, 18 Sep 2018 10:45:58 +0200
+ -- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com>  Tue, 18 Sep 2018 10:21:30 +0200
+
+livecd-rootfs (2.542) cosmic; urgency=medium
+
+  * Decide what model assertion series to fetch depending on the suite. Use 16
+    for xenial and 18 for other series (bionic+). This enables core18 image
+    builds.
+
+ -- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com>  Wed, 03 Oct 2018 11:51:59 +0200
+
+livecd-rootfs (2.541) cosmic; urgency=medium
+
+  [ Michael Hudson-Doyle ]
+  * Remove device nodes from Docker images. (LP: #1645468) 
+
+  [ Robert C Jennings ]
+  * Add the server snap seed to the ubuntu-cpc project
+  * Allow hooks to replace generic snap assertion
+
+ -- Steve Langasek <steve.langasek@ubuntu.com>  Thu, 27 Sep 2018 15:21:49 -0700
+
+livecd-rootfs (2.540) cosmic; urgency=medium
+
+  * Ensure /lib/modules exists in root tarballs and sqashfs.
+    (LP: #1792905)
+
+ -- Tobias Koch <tobias.koch@canonical.com>  Thu, 20 Sep 2018 09:38:34 +0200
+
+livecd-rootfs (2.539) cosmic; urgency=medium
+
+  * Stop building buildd chroots with --xattrs until we move to python3.
+
+ -- Adam Conrad <adconrad@ubuntu.com>  Thu, 20 Sep 2018 02:52:10 -0600
+
+livecd-rootfs (2.538) cosmic; urgency=medium
+
+  * Add a buildd subproject.
+
+ -- Colin Watson <cjwatson@ubuntu.com>  Wed, 19 Sep 2018 16:35:54 +0200
+
+livecd-rootfs (2.537) cosmic; urgency=medium
+
+  * Minimize the number of manually installed packages in images by marking
+    dependencies of metapackages as automatically installed.
+
+ -- Julian Andres Klode <juliank@ubuntu.com>  Tue, 18 Sep 2018 08:55:04 +0200
 
 livecd-rootfs (2.536) cosmic; urgency=medium
 

debian/control

@@ -26,6 +26,7 @@ Depends: ${misc:Depends},
          parted,
          procps,
          python-minimal | python,
+         python3-apt,
          python3-software-properties,
          qemu-utils,
          rsync,

debian/install

@@ -1,2 +1,3 @@
 live-build usr/share/livecd-rootfs
 get-ppa-fingerprint usr/share/livecd-rootfs
+minimize-manual usr/share/livecd-rootfs

live-build/auto/build

@@ -266,6 +266,11 @@ EOF
 		# Save even more size by removing apt lists (that are currently removed
 		# downstream anyway)
 		rm -rf chroot/var/lib/apt/lists/*
+		# Having device notes in the docker image can cause problems
+		# (https://github.com/tianon/docker-brew-ubuntu-core/issues/62)
+		# so remove them.  We only do this for docker out of an
+		# abundance of caution.
+		rm -rf chroot/dev/*
 	fi
 
 	if [ -f config/universe-enabled ]; then
@@ -445,6 +450,8 @@ EOF
 	(cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr)
 	echo END docdirs
 
+	/usr/share/livecd-rootfs/minimize-manual chroot
+
 	lb binary "$@"
 	touch binary.success
 ) 2>&1 | tee binary.log
@@ -495,6 +502,16 @@ if [ -e "binary/$INITFS/filesystem.dir" ]; then
 	chmod 644 "$PREFIX.rootfs.tar.gz"
 elif [ -e binary-tar.tar.gz ]; then
 	cp -a binary-tar.tar.gz "$PREFIX.rootfs.tar.gz"
+elif [ "$SUBPROJECT" = buildd ]; then
+	# A few things (launchpad-buildd, sbuild-launchpad-chroot) rely on
+	# the top-level directory being "chroot-autobuild", so we have to do
+	# this ourselves.
+	# gzip was chosen for fastest decompression speed: it decompresses
+	# buildd chroots about twice as fast as xz and about five times as
+	# fast as bzip2.
+	tar --transform='s,^binary,chroot-autobuild,' \
+		--sort=name --numeric-owner \
+		-czf "$PREFIX.rootfs.tar.gz" binary
 fi
 
 if [ "$PROJECT:${SUBPROJECT:-}" = "ubuntu-core:system-image" ]; then
@@ -534,6 +551,7 @@ fi
 
 # '--initramfs none' produces different manifest names.
 if [ -e "binary/$INITFS/filesystem.packages" ]; then
+	./config/snap-seed-parse "chroot/" >> "binary/${INITFS}/filesystem.packages"
 	ln "binary/$INITFS/filesystem.packages" "$PREFIX.manifest"
 	chmod 644 "$PREFIX.manifest"
 fi

live-build/auto/config

@@ -33,6 +33,7 @@ fi
 
 mkdir -p config
 cp -af /usr/share/livecd-rootfs/live-build/functions config/functions
+cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse
 
 mkdir -p config/package-lists
 
@@ -185,6 +186,14 @@ case $IMAGEFORMAT in
 			*)
 				UBUNTU_IMAGE_ARGS="" ;;
 		esac
+		case $SUITE in
+			xenial)
+				# Ubuntu Core 16
+				;;
+			*)
+				# Ubuntu Core 18
+				MODEL="ubuntu-core-18-${MODEL#pc-}" ;;
+		esac
 
 		echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common
 		echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common
@@ -642,12 +651,63 @@ case $PROJECT in
 		;;
 esac
 
+case $SUBPROJECT in
+	buildd)
+		OPTS="${OPTS:+$OPTS }--archive-areas main"
+		COMPONENTS='main restricted universe multiverse'
+		OPTS="${OPTS:+$OPTS }--apt-recommends false"
+		OPTS="${OPTS:+$OPTS }--apt-secure false"
+		OPTS="${OPTS:+$OPTS }--parent-mirror-binary ${MIRROR}"
+		# XXX cjwatson 2018-04-27: We need to work out how to make
+		# this conditional so that we can do things like building
+		# buildd chroots with -updates.  This probably involves
+		# either extending the PROPOSED hack or fixing the strange
+		# way that SUITE is in fact a series; in either case it's
+		# likely to involve work both here and in launchpad-buildd.
+		OPTS="${OPTS:+$OPTS }--security false --volatile false"
+
+		add_package install adduser
+		add_package install policyrcd-script-zg2
+		add_package install pkgbinarymangler
+		add_package install ca-certificates
+		add_package install gpg
+		add_package install gpg-agent
+		case $SUITE in
+			precise|trusty|xenial)
+				# no longer needed in >= artful
+				add_package install pkg-create-dbgsym
+				# no longer needed in >= bionic
+				add_package install apt-transport-https
+				# no longer needed in >= cosmic
+				add_package install tzdata
+				;;
+			artful)
+				# no longer needed in >= bionic
+				add_package install apt-transport-https
+				# no longer needed in >= cosmic
+				add_package install tzdata
+				;;
+			bionic)
+				# no longer needed in >= cosmic
+				add_package install tzdata
+				;;
+		esac
+		add_package install fakeroot
+		add_package install build-essential
+		# Needed for LXD-based builds.
+		add_package install init
+		;;
+esac
+
 # we'll expand the base seed given here according to the STRUCTURE file, and
 # then look in all of the seeds found to see which snaps are seeded
 case $PROJECT:${SUBPROJECT:-} in
 	ubuntu:*|kubuntu*:*|lubuntu*:*|xubuntu*:*|ubuntu-mate*:*|ubuntustudio*:*|ubuntukylin*:*|ubuntu-budgie*:*)
 		BASE_SEED='desktop'
 		;;
+	ubuntu-cpc:*)
+		BASE_SEED='server'
+		;;
 	ubuntu-server:live)
 		BASE_SEED='server'
 		# subiquity is seeded but in a separate squashfs via hooks; set HOOK_SNAPS and ALL_SNAPS.
@@ -952,6 +1012,12 @@ EOF
 		;;
 esac
 
+case $SUBPROJECT in
+	buildd)
+		cp -af /usr/share/livecd-rootfs/live-build/buildd/* config/
+		;;
+esac
+
 if [ "$EXTRA_PPAS" ]; then
 	rm -f config/archives/extra-ppas.list.chroot \
 	      config/archives/extra-ppas.pref.chroot \

live-build/buildd/hooks/00-kernel-img.chroot

@@ -0,0 +1,5 @@
+#! /bin/sh
+set -e
+
+# At one point, kernel builds needed this.
+echo do_initrd = Yes >>/etc/kernel-img.conf

live-build/buildd/hooks/00-mirror.binary

@@ -0,0 +1,12 @@
+#! /bin/sh
+set -e
+
+. config/bootstrap
+
+# Use a public-facing mirror URL, for the benefit of
+# sbuild-launchpad-chroot.  We deliberately do this only after live-build
+# has run "apt-get update" for the last time, in order that
+# /var/lib/apt/lists/ has suitable cached Packages files; this speeds up
+# builds on buildds.
+sed -i "s,${LB_PARENT_MIRROR_BINARY},${LB_MIRROR_BINARY},g" \
+	binary/etc/apt/sources.list

live-build/buildd/hooks/01-pkgbinarymangler.chroot

@@ -0,0 +1,10 @@
+#! /bin/sh
+set -e
+
+# Configure pkgbinarymangler.
+sed -i /^enable/s/false/true/ \
+	/etc/pkgbinarymangler/maintainermangler.conf \
+	/etc/pkgbinarymangler/striptranslations.conf || true
+sed -i /^invalid_current/s/ignore/fail/ \
+	/etc/pkgbinarymangler/maintainermangler.conf \
+	/etc/pkgbinarymangler/striptranslations.conf || true

live-build/buildd/hooks/02-user.chroot

@@ -0,0 +1,9 @@
+#! /bin/sh
+set -e
+
+# Create the buildd user and group.
+addgroup --gid 2501 buildd
+adduser --system --disabled-password --gecos 'Build Daemon user' \
+	--ingroup buildd --uid 2001 --shell /bin/bash buildd
+mkdir -p /build/buildd
+chown buildd:buildd /build/buildd

live-build/buildd/includes.chroot/etc/apt/apt.conf.d/99buildd

@@ -0,0 +1,2 @@
+DPkg::Options {"--force-unsafe-io";};
+DPkg::Use-Pty "false";

live-build/buildd/includes.chroot/etc/apt/preferences.d/backports

@@ -0,0 +1,3 @@
+Package: *
+Pin: release a=*-backports
+Pin-Priority: 500

live-build/buildd/includes.chroot/etc/fstab

@@ -0,0 +1 @@
+/dev/root / ext2 noatime,errors=remount-ro 0 1

live-build/buildd/includes.chroot/etc/hostname

@@ -0,0 +1 @@
+INVALID

live-build/buildd/includes.chroot/etc/hosts

@@ -0,0 +1,9 @@
+127.0.0.1	localhost.localdomain	localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts

live-build/buildd/includes.chroot/usr/local/sbin/policy-rc.d

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# policy-rc.d script for chroots.
+# Copyright (c) 2007 Peter Palfrader <peter@palfrader.org>
+# License: <weasel> MIT, if you want one.
+
+while true; do
+  case "$1" in
+    -*)         shift ;;
+    makedev)    exit 0;;
+    *) echo "Not running services in chroot."; exit 101 ;;
+  esac
+done

live-build/buildd/preseed/debconf.preseed

@@ -0,0 +1,2 @@
+# We never want debconf interaction.
+debconf debconf/frontend select Noninteractive

live-build/buildd/preseed/man-db.preseed

@@ -0,0 +1,3 @@
+# Avoid unnecessary manual page database builds (see
+# https://bugs.debian.org/554914).
+man-db man-db/auto-update boolean false

live-build/buildd/preseed/sun-java6.preseed

@@ -0,0 +1,3 @@
+# Pre-accept interactive EULA prompts.
+sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true
+sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true

live-build/functions

@@ -43,6 +43,17 @@ create_empty_disk_image() {
     dd if=/dev/zero of="$1" bs=1 count=0 seek="${imagesize}"
 }
 
+create_manifest() {
+    local chroot_root=${1}
+    local target_file=${2}
+    echo "create_manifest chroot_root: ${chroot_root}"
+    dpkg-query --show --admindir="${chroot_root}/var/lib/dpkg" > ${target_file}
+    echo "create_manifest call to dpkg-query finished."
+    ./config/snap-seed-parse "${chroot_root}" >> ${target_file}
+    echo "create_manifest call to snap_seed_parse finished."
+    echo "create_manifest finished"
+}
+
 make_ext4_partition() {
     device="$1"
     label=${fs_label:+-L "${fs_label}"}
@@ -427,6 +438,16 @@ snap_prepare_assertions() {
     local brand="$(echo $CUSTOM_BRAND_MODEL | cut -d: -f 1)"
     local model="$(echo $CUSTOM_BRAND_MODEL | cut -d: -f 2)"
 
+    # Clear the assertions if they already exist
+    if [ -e "$model_assertion" ] ; then
+        existing_model=$(awk '/^model: / {print $2}' $model_assertion)
+        existing_brand=$(awk '/^brand-id: / {print $2}' $model_assertion)
+        echo "snap_prepare_assertions: replacing $existing_brand:$existing_model with $brand:$model"
+        rm "$model_assertion"
+        rm "$account_key_assertion"
+        rm "$account_assertion"
+    fi
+
     if ! [ -e "$model_assertion" ] ; then
         snap known --remote model series=16 \
             model=$model brand-id=$brand \
@@ -441,7 +462,6 @@ snap_prepare_assertions() {
             > "$account_key_assertion"
     fi
 
-
     if ! [ -e "$account_assertion" ] ; then
         local account=$(sed -n -e's/account-id: //p' < "$account_key_assertion")
         snap known --remote account account-id=$account \
@@ -468,13 +488,16 @@ snap_prepare() {
 }
 
 snap_preseed() {
-    # Preseed a snap in the image
+    # Preseed a snap in the image (snap_prepare must be called once prior)
     local CHROOT_ROOT=$1
     local SNAP=$2
     # Per Ubuntu policy, all seeded snaps (with the exception of the core
     # snap) must pull from stable/ubuntu-$(release_ver) as their channel.
     local CHANNEL=${3:-"stable/ubuntu-$(release_ver)"}
 
-    snap_prepare $CHROOT_ROOT
+    if [ ! -e "$CHROOT_ROOT/var/lib/snapd/seed/assertions/model" ]; then
+        echo "ERROR: Snap model assertion not present, snap_prepare must be called"
+        exit 1
+    fi
     _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL
 }

live-build/snap-seed-parse.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+
+"""
+Usage: snap-seed-parse ${chroot_dir} > somefile.manifest
+
+This script looks for a seed.yaml path in the given root directory, parsing
+it and printing generated manifest lines to stdout for easy redirection.
+"""
+
+import re
+import sys
+import yaml
+import os.path
+
+
+def log(msg):
+    sys.stderr.write("snap-seed-parse: {}\n".format(msg))
+
+
+log("Parsing seed.yaml")
+
+CHROOT_ROOT = sys.argv[1] if len(sys.argv) > 1 and len(sys.argv[1]) > 0 \
+                          else ''
+
+# Trim any trailing slashes for correct appending
+log("CHROOT_ROOT: {}".format(CHROOT_ROOT))
+if len(CHROOT_ROOT) > 0 and CHROOT_ROOT[-1] == '/':
+    CHROOT_ROOT = CHROOT_ROOT[:-1]
+
+# This is where we expect to find the seed.yaml file
+YAML_PATH = CHROOT_ROOT + '/var/lib/snapd/seed/seed.yaml'
+
+# Snaps are prepended with this string in the manifest
+LINE_PREFIX = 'snap:'
+
+log("yaml path: {}".format(YAML_PATH))
+if not os.path.isfile(YAML_PATH):
+    log("WARNING: yaml path not found; no seeded snaps found.")
+    exit(0)
+else:
+    log("yaml path found.")
+
+with open(YAML_PATH, 'r') as fh:
+    yaml_lines = yaml.safe_load(fh)['snaps']
+
+# Loop over dict items, outputting one manifest line from each triplet
+for item in yaml_lines:
+    filestring = item['file']
+    # Pull the revision number off the file name
+    revision = filestring[filestring.rindex('_')+1:]
+    revision = re.sub(r'[^0-9]', '', revision)
+    print("{}{}\t{}\t{}".format(LINE_PREFIX,
+                                        item['name'],
+                                        item['channel'],
+                                        revision,
+                                        ))

live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary

@@ -21,6 +21,9 @@ env DEBIAN_FRONTEND=noninteractive chroot $rootfs_dir apt-get --purge remove --a
 env DEBIAN_FRONTEND=noninteractive chroot $rootfs_dir apt-get autoremove --purge --assume-yes
 rm -rf $rootfs_dir/boot/grub
 
+# Keep this as some derivatives mount a tempfs here
+mkdir -p $rootfs_dir/lib/modules
+
 teardown_mountpoint $rootfs_dir
 
-dpkg-query --admindir=$rootfs_dir/var/lib/dpkg -W > $rootfs_dir.manifest
+create_manifest "${rootfs_dir}" "${rootfs_dir}.manifest"

live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary

@@ -93,6 +93,12 @@ install_grub() {
             ;;
     esac
 
+    # This call to generate the package manifest is added here to capture 
+    # grub-efi packages that otherwise would not make it into the base
+    # manifest. filesystem.packages is moved into place via symlinking to
+    # livecd.ubuntu-cpc.manifest by live-build/auto/build after lb_binary runs
+    create_manifest "mountpoint" "binary/boot/filesystem.packages"
+
     chroot mountpoint grub-install "${loop_device}" \
         --boot-directory=/boot \
         --efi-directory=/boot/efi \

live-build/ubuntu-cpc/hooks/042-vagrant.binary

@@ -163,7 +163,8 @@ Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |vb|
      vb.customize [ "modifyvm", :id, "--uart1", "0x3F8", "4" ]
-     vb.customize [ "modifyvm", :id, "--uartmode1", "file", File.join(Dir.pwd, "${prefix}-console.log") ]
+# Creating a console log file is not an expected behavior for vagrant boxes. LP #1777827
+#     vb.customize [ "modifyvm", :id, "--uartmode1", "file", File.join(Dir.pwd, "${prefix}-console.log") ]
   end
 end
 EOF

live-build/ubuntu-server/hooks/030-root-squashfs.binary

@@ -28,7 +28,7 @@ cp -a chroot/* binary/boot/squashfs.dir
 squashfs_f="${PWD}/livecd.${PROJECT}.squashfs"
 squashfs_f_manifest="${squashfs_f}.manifest"
 
-dpkg-query --admindir=binary/boot/squashfs.dir/var/lib/dpkg -W > ${squashfs_f_manifest}
+create_manifest "binary/boot/squashfs.dir" "${squashfs_f_manifest}"
 
 (cd "binary/boot/squashfs.dir/" &&
       mksquashfs . ${squashfs_f} \

minimize-manual

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+"""Minimize the number of manually installed packages in the image.
+
+Finds all manually installed meta packages, and marks their dependencies
+as automatically installed.
+"""
+import sys
+
+import apt
+
+
+def is_root(pkg):
+    """Check if the package is a root package (manually inst. meta)"""
+    return (pkg.is_installed and
+            not pkg.is_auto_installed and
+            (pkg.section == "metapackages" or
+             pkg.section.endswith("/metapackages")))
+
+
+def main():
+    """Main function"""
+    cache = apt.Cache(rootdir=sys.argv[1] if len(sys.argv) > 1 else None)
+    roots = set(pkg for pkg in cache if is_root(pkg))
+    workset = set(roots)
+    seen = set()
+
+    with cache.actiongroup():
+        while True:
+            print("Iteration", file=sys.stderr)
+            to_proc = workset - seen
+            if not to_proc:
+                break
+            for pkg in sorted(to_proc):
+                print("    Visiting", pkg, file=sys.stderr)
+
+                if pkg not in roots:
+                    pkg.mark_auto()
+
+                for dep in (pkg.installed.dependencies +
+                            pkg.installed.recommends):
+                    for bdep in dep.or_dependencies:
+                        for ver in bdep.target_versions:
+                            if ver.package.is_installed:
+                                workset.add(ver.package)
+
+                seen.add(pkg)
+
+        cache.commit()
+
+
+if __name__ == '__main__':
+    main()