merge trunk

debian/changelog

@@ -1,7 +1,60 @@
-livecd-rootfs (2.533) UNRELEASED; urgency=medium
+livecd-rootfs (2.540) cosmic; urgency=medium
+
+  * Ensure /lib/modules exists in root tarballs and sqashfs.
+    (LP: #1792905)
+
+ -- Tobias Koch <tobias.koch@canonical.com>  Thu, 20 Sep 2018 09:38:34 +0200
+
+livecd-rootfs (2.539) cosmic; urgency=medium
+
+  * Stop building buildd chroots with --xattrs until we move to python3.
+
+ -- Adam Conrad <adconrad@ubuntu.com>  Thu, 20 Sep 2018 02:52:10 -0600
+
+livecd-rootfs (2.538) cosmic; urgency=medium
+
+  * Add a buildd subproject.
+
+ -- Colin Watson <cjwatson@ubuntu.com>  Wed, 19 Sep 2018 16:35:54 +0200
+
+livecd-rootfs (2.537) cosmic; urgency=medium
+
+  * Minimize the number of manually installed packages in images by marking
+    dependencies of metapackages as automatically installed.
+
+ -- Julian Andres Klode <juliank@ubuntu.com>  Tue, 18 Sep 2018 08:55:04 +0200
+
+livecd-rootfs (2.536) cosmic; urgency=medium
+
+  * Fix live-server journald config snippet to actually disable journald rate
+    limiting.
+
+ -- Michael Hudson-Doyle <michael.hudson@ubuntu.com>  Tue, 28 Aug 2018 11:00:39 +1200
+
+livecd-rootfs (2.535) cosmic; urgency=medium
+
+  * Disentangle enabling universe in the final image a little from having
+    PREINSTALLED=true set and enable it for a live-server build.
+    (LP: #1783129)
+
+ -- Michael Hudson-Doyle <michael.hudson@ubuntu.com>  Fri, 24 Aug 2018 13:04:51 +1200
+
+livecd-rootfs (2.534) cosmic; urgency=medium
+
+  [ Michael Hudson-Doyle ]
+  * Disable journald rate limiting in the live-server live session.
+    (LP: #1776891)
+
+  [ Steve Langasek ]
+  * generate all tar files with --xattrs.  LP: #1302192.
+
+ -- Steve Langasek <steve.langasek@ubuntu.com>  Mon, 06 Aug 2018 13:12:02 -0700
+
+livecd-rootfs (2.533) cosmic; urgency=medium
 
   [ Robert C Jennings ]
   * ubuntu-cpc: Remove git-man for minimized images 
+  * Disable checksum generation
 
   [ Colin Watson ]
   * live-build/auto/config: Drop "lp-*" hostname filter for Launchpad
@@ -14,7 +67,7 @@ livecd-rootfs (2.533) UNRELEASED; urgency=medium
   [ Michael Hudson-Doyle ]
   * Remove device nodes from Docker images. (LP: #1645468) 
 
- -- Robert C Jennings <robert.jennings@canonical.com>  Fri, 08 Jun 2018 10:10:53 -0700
+ -- Adam Conrad <adconrad@ubuntu.com>  Wed, 18 Jul 2018 11:06:54 -0600
 
 livecd-rootfs (2.532) cosmic; urgency=medium
 

debian/control

@@ -26,6 +26,7 @@ Depends: ${misc:Depends},
          parted,
          procps,
          python-minimal | python,
+         python3-apt,
          python3-software-properties,
          qemu-utils,
          rsync,

debian/install

@@ -1,2 +1,3 @@
 live-build usr/share/livecd-rootfs
 get-ppa-fingerprint usr/share/livecd-rootfs
+minimize-manual usr/share/livecd-rootfs

live-build/auto/build

@@ -273,7 +273,7 @@ EOF
 		rm -rf chroot/dev/*
 	fi
 
-	if [ -f config/oem-config-preinstalled ]; then
+	if [ -f config/universe-enabled ]; then
 
 # This is cargo-culted almost verbatim (with some syntax changes for
 # preinstalled being slightly different in what it doesn't ask) from
@@ -450,6 +450,8 @@ EOF
 	(cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr)
 	echo END docdirs
 
+	/usr/share/livecd-rootfs/minimize-manual chroot
+
 	lb binary "$@"
 	touch binary.success
 ) 2>&1 | tee binary.log
@@ -495,11 +497,21 @@ for ISO in binary.iso binary.hybrid.iso; do
 done
 
 if [ -e "binary/$INITFS/filesystem.dir" ]; then
-	(cd "binary/$INITFS/filesystem.dir/" && tar -c *) | \
+	(cd "binary/$INITFS/filesystem.dir/" && tar -c --xattrs *) | \
 		gzip -9 --rsyncable > "$PREFIX.rootfs.tar.gz"
 	chmod 644 "$PREFIX.rootfs.tar.gz"
 elif [ -e binary-tar.tar.gz ]; then
 	cp -a binary-tar.tar.gz "$PREFIX.rootfs.tar.gz"
+elif [ "$SUBPROJECT" = buildd ]; then
+	# A few things (launchpad-buildd, sbuild-launchpad-chroot) rely on
+	# the top-level directory being "chroot-autobuild", so we have to do
+	# this ourselves.
+	# gzip was chosen for fastest decompression speed: it decompresses
+	# buildd chroots about twice as fast as xz and about five times as
+	# fast as bzip2.
+	tar --transform='s,^binary,chroot-autobuild,' \
+		--sort=name --numeric-owner \
+		-czf "$PREFIX.rootfs.tar.gz" binary
 fi
 
 if [ "$PROJECT:${SUBPROJECT:-}" = "ubuntu-core:system-image" ]; then
@@ -532,7 +544,7 @@ EOF
 fi
 
 if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then
-	(cd "binary/$INITFS/custom.dir/" && tar -c *) | \
+	(cd "binary/$INITFS/custom.dir/" && tar -c --xattrs *) | \
 		gzip -9 --rsyncable > "$PREFIX.custom.tar.gz"
 	chmod 644 "$PREFIX.custom.tar.gz"
 fi

live-build/auto/config

@@ -151,6 +151,7 @@ case $IMAGEFORMAT in
 				#  by skipping lb_chroot_live-packages.
 				skip_lb_stage chroot_live-packages
 				INITRAMFS_TYPE=auto
+	                        touch config/universe-enabled
 				;;
 			*)
 				PREINSTALLED=true
@@ -213,9 +214,9 @@ case $IMAGEFORMAT in
 esac
 
 if [ "$PREINSTALLED" = "true" ]; then
-	# This is an oem-config preinstalled image, touch a random file that
+	# Touch a random file that we can refer back to during build,
-	# we can refer back to during build, cause that's wildly hackish
+	# cause that's wildly hackish
-	touch config/oem-config-preinstalled
+	touch config/universe-enabled
 	case $PROJECT in
 		kubuntu*)
 			add_package live oem-config-kde ubiquity-frontend-kde
@@ -641,6 +642,54 @@ case $PROJECT in
 		;;
 esac
 
+case $SUBPROJECT in
+	buildd)
+		OPTS="${OPTS:+$OPTS }--archive-areas main"
+		COMPONENTS='main restricted universe multiverse'
+		OPTS="${OPTS:+$OPTS }--apt-recommends false"
+		OPTS="${OPTS:+$OPTS }--apt-secure false"
+		OPTS="${OPTS:+$OPTS }--parent-mirror-binary ${MIRROR}"
+		# XXX cjwatson 2018-04-27: We need to work out how to make
+		# this conditional so that we can do things like building
+		# buildd chroots with -updates.  This probably involves
+		# either extending the PROPOSED hack or fixing the strange
+		# way that SUITE is in fact a series; in either case it's
+		# likely to involve work both here and in launchpad-buildd.
+		OPTS="${OPTS:+$OPTS }--security false --volatile false"
+
+		add_package install adduser
+		add_package install policyrcd-script-zg2
+		add_package install pkgbinarymangler
+		add_package install ca-certificates
+		add_package install gpg
+		add_package install gpg-agent
+		case $SUITE in
+			precise|trusty|xenial)
+				# no longer needed in >= artful
+				add_package install pkg-create-dbgsym
+				# no longer needed in >= bionic
+				add_package install apt-transport-https
+				# no longer needed in >= cosmic
+				add_package install tzdata
+				;;
+			artful)
+				# no longer needed in >= bionic
+				add_package install apt-transport-https
+				# no longer needed in >= cosmic
+				add_package install tzdata
+				;;
+			bionic)
+				# no longer needed in >= cosmic
+				add_package install tzdata
+				;;
+		esac
+		add_package install fakeroot
+		add_package install build-essential
+		# Needed for LXD-based builds.
+		add_package install init
+		;;
+esac
+
 # we'll expand the base seed given here according to the STRUCTURE file, and
 # then look in all of the seeds found to see which snaps are seeded
 case $PROJECT:${SUBPROJECT:-} in
@@ -795,6 +844,7 @@ lb config noauto \
 	--initsystem none \
 	--bootloader "$BOOTLOADER" \
 	${INITRAMFS_COMPRESSION:+--initramfs-compression "$INITRAMFS_COMPRESSION"} \
+	--checksums none \
 	--cache false \
 	${BOOTAPPEND_LIVE:+--bootappend-live "$BOOTAPPEND_LIVE"} \
 	$OPTS \
@@ -944,6 +994,12 @@ EOF
 		;;
 esac
 
+case $SUBPROJECT in
+	buildd)
+		cp -af /usr/share/livecd-rootfs/live-build/buildd/* config/
+		;;
+esac
+
 if [ "$EXTRA_PPAS" ]; then
 	rm -f config/archives/extra-ppas.list.chroot \
 	      config/archives/extra-ppas.pref.chroot \

live-build/buildd/hooks/00-kernel-img.chroot

@@ -0,0 +1,5 @@
+#! /bin/sh
+set -e
+
+# At one point, kernel builds needed this.
+echo do_initrd = Yes >>/etc/kernel-img.conf

live-build/buildd/hooks/00-mirror.binary

@@ -0,0 +1,12 @@
+#! /bin/sh
+set -e
+
+. config/bootstrap
+
+# Use a public-facing mirror URL, for the benefit of
+# sbuild-launchpad-chroot.  We deliberately do this only after live-build
+# has run "apt-get update" for the last time, in order that
+# /var/lib/apt/lists/ has suitable cached Packages files; this speeds up
+# builds on buildds.
+sed -i "s,${LB_PARENT_MIRROR_BINARY},${LB_MIRROR_BINARY},g" \
+	binary/etc/apt/sources.list

live-build/buildd/hooks/01-pkgbinarymangler.chroot

@@ -0,0 +1,10 @@
+#! /bin/sh
+set -e
+
+# Configure pkgbinarymangler.
+sed -i /^enable/s/false/true/ \
+	/etc/pkgbinarymangler/maintainermangler.conf \
+	/etc/pkgbinarymangler/striptranslations.conf || true
+sed -i /^invalid_current/s/ignore/fail/ \
+	/etc/pkgbinarymangler/maintainermangler.conf \
+	/etc/pkgbinarymangler/striptranslations.conf || true

live-build/buildd/hooks/02-user.chroot

@@ -0,0 +1,9 @@
+#! /bin/sh
+set -e
+
+# Create the buildd user and group.
+addgroup --gid 2501 buildd
+adduser --system --disabled-password --gecos 'Build Daemon user' \
+	--ingroup buildd --uid 2001 --shell /bin/bash buildd
+mkdir -p /build/buildd
+chown buildd:buildd /build/buildd

live-build/buildd/includes.chroot/etc/apt/apt.conf.d/99buildd

@@ -0,0 +1,2 @@
+DPkg::Options {"--force-unsafe-io";};
+DPkg::Use-Pty "false";

live-build/buildd/includes.chroot/etc/apt/preferences.d/backports

@@ -0,0 +1,3 @@
+Package: *
+Pin: release a=*-backports
+Pin-Priority: 500

live-build/buildd/includes.chroot/etc/fstab

@@ -0,0 +1 @@
+/dev/root / ext2 noatime,errors=remount-ro 0 1

live-build/buildd/includes.chroot/etc/hostname

@@ -0,0 +1 @@
+INVALID

live-build/buildd/includes.chroot/etc/hosts

@@ -0,0 +1,9 @@
+127.0.0.1	localhost.localdomain	localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts

live-build/buildd/includes.chroot/usr/local/sbin/policy-rc.d

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# policy-rc.d script for chroots.
+# Copyright (c) 2007 Peter Palfrader <peter@palfrader.org>
+# License: <weasel> MIT, if you want one.
+
+while true; do
+  case "$1" in
+    -*)         shift ;;
+    makedev)    exit 0;;
+    *) echo "Not running services in chroot."; exit 101 ;;
+  esac
+done

live-build/buildd/preseed/debconf.preseed

@@ -0,0 +1,2 @@
+# We never want debconf interaction.
+debconf debconf/frontend select Noninteractive

live-build/buildd/preseed/man-db.preseed

@@ -0,0 +1,3 @@
+# Avoid unnecessary manual page database builds (see
+# https://bugs.debian.org/554914).
+man-db man-db/auto-update boolean false

live-build/buildd/preseed/sun-java6.preseed

@@ -0,0 +1,3 @@
+# Pre-accept interactive EULA prompts.
+sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true
+sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true

live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary

@@ -21,6 +21,9 @@ env DEBIAN_FRONTEND=noninteractive chroot $rootfs_dir apt-get --purge remove --a
 env DEBIAN_FRONTEND=noninteractive chroot $rootfs_dir apt-get autoremove --purge --assume-yes
 rm -rf $rootfs_dir/boot/grub
 
+# Keep this as some derivatives mount a tempfs here
+mkdir -p $rootfs_dir/lib/modules
+
 teardown_mountpoint $rootfs_dir
 
 dpkg-query --admindir=$rootfs_dir/var/lib/dpkg -W > $rootfs_dir.manifest

live-build/ubuntu-cpc/hooks/031-1-root-xz.binary

@@ -12,4 +12,4 @@ fi
 rootfs_dir=rootfs.dir
 
 cp $rootfs_dir.manifest livecd.ubuntu-cpc.rootfs.manifest
-(cd $rootfs_dir/ && tar -c *) | xz > livecd.ubuntu-cpc.rootfs.tar.xz
+(cd $rootfs_dir/ && tar -c --xattrs *) | xz > livecd.ubuntu-cpc.rootfs.tar.xz

live-build/ubuntu-server/includes.binary/overlay/etc/systemd/journald.conf.d/no-rate-limit.conf

@@ -0,0 +1,2 @@
+[Journal]
+RateLimitIntervalSec=0

minimize-manual

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+"""Minimize the number of manually installed packages in the image.
+
+Finds all manually installed meta packages, and marks their dependencies
+as automatically installed.
+"""
+import sys
+
+import apt
+
+
+def is_root(pkg):
+    """Check if the package is a root package (manually inst. meta)"""
+    return (pkg.is_installed and
+            not pkg.is_auto_installed and
+            (pkg.section == "metapackages" or
+             pkg.section.endswith("/metapackages")))
+
+
+def main():
+    """Main function"""
+    cache = apt.Cache(rootdir=sys.argv[1] if len(sys.argv) > 1 else None)
+    roots = set(pkg for pkg in cache if is_root(pkg))
+    workset = set(roots)
+    seen = set()
+
+    with cache.actiongroup():
+        while True:
+            print("Iteration", file=sys.stderr)
+            to_proc = workset - seen
+            if not to_proc:
+                break
+            for pkg in sorted(to_proc):
+                print("    Visiting", pkg, file=sys.stderr)
+
+                if pkg not in roots:
+                    pkg.mark_auto()
+
+                for dep in (pkg.installed.dependencies +
+                            pkg.installed.recommends):
+                    for bdep in dep.or_dependencies:
+                        for ver in bdep.target_versions:
+                            if ver.package.is_installed:
+                                workset.add(ver.package)
+
+                seen.add(pkg)
+
+        cache.commit()
+
+
+if __name__ == '__main__':
+    main()