Merge trunk.

debian/changelog

@@ -1,12 +1,57 @@
-livecd-rootfs (2.533) UNRELEASED; urgency=medium
+livecd-rootfs (2.538) UNRELEASED; urgency=medium
+
+  * Add a buildd subproject.
+
+ -- Colin Watson <cjwatson@ubuntu.com>  Wed, 19 Sep 2018 16:19:58 +0200
+
+livecd-rootfs (2.537) cosmic; urgency=medium
+
+  * Minimize the number of manually installed packages in images by marking
+    dependencies of metapackages as automatically installed.
+
+ -- Julian Andres Klode <juliank@ubuntu.com>  Tue, 18 Sep 2018 08:55:04 +0200
+
+livecd-rootfs (2.536) cosmic; urgency=medium
+
+  * Fix live-server journald config snippet to actually disable journald rate
+    limiting.
+
+ -- Michael Hudson-Doyle <michael.hudson@ubuntu.com>  Tue, 28 Aug 2018 11:00:39 +1200
+
+livecd-rootfs (2.535) cosmic; urgency=medium
+
+  * Disentangle enabling universe in the final image a little from having
+    PREINSTALLED=true set and enable it for a live-server build.
+    (LP: #1783129)
+
+ -- Michael Hudson-Doyle <michael.hudson@ubuntu.com>  Fri, 24 Aug 2018 13:04:51 +1200
+
+livecd-rootfs (2.534) cosmic; urgency=medium
+
+  [ Michael Hudson-Doyle ]
+  * Disable journald rate limiting in the live-server live session.
+    (LP: #1776891)
+
+  [ Steve Langasek ]
+  * generate all tar files with --xattrs.  LP: #1302192.
+
+ -- Steve Langasek <steve.langasek@ubuntu.com>  Mon, 06 Aug 2018 13:12:02 -0700
+
+livecd-rootfs (2.533) cosmic; urgency=medium
 
   [ Robert C Jennings ]
   * ubuntu-cpc: Remove git-man for minimized images 
+  * Disable checksum generation
 
   [ Colin Watson ]
-  * Add a buildd subproject.
+  * live-build/auto/config: Drop "lp-*" hostname filter for Launchpad
+    builders, since LP #1747015 is now fixed; "*.buildd" should be enough.
 
- -- Robert C Jennings <robert.jennings@canonical.com>  Fri, 08 Jun 2018 10:10:53 -0700
+  [ Francis Ginther ]
+  * Update unminimize script text and install ubuntu-standard when
+    unminimizing a minimal image (LP: #1778777)
+
+ -- Adam Conrad <adconrad@ubuntu.com>  Wed, 18 Jul 2018 11:06:54 -0600
 
 livecd-rootfs (2.532) cosmic; urgency=medium
 

debian/control

@@ -26,6 +26,7 @@ Depends: ${misc:Depends},
          parted,
          procps,
          python-minimal | python,
+         python3-apt,
          python3-software-properties,
          qemu-utils,
          rsync,

debian/install

@@ -1,2 +1,3 @@
 live-build usr/share/livecd-rootfs
 get-ppa-fingerprint usr/share/livecd-rootfs
+minimize-manual usr/share/livecd-rootfs

live-build/auto/build

@@ -109,12 +109,16 @@ set -e
 echo "This system has been minimized by removing packages and content that are"
 echo "not required on a system that users do not log into."
 echo ""
-echo "This script restores the content and packages that are found on a default"
-echo "Ubuntu server system."
+echo "This script restores content and packages that are found on a default"
+echo "Ubuntu server system in order to make this system more suitable for"
+echo "interactive use."
 echo ""
 echo "Reinstallation of packages may fail due to changes to the system"
 echo "configuration, the presence of third-party packages, or for other"
 echo "reasons."
+echo ""
+echo "This operation may take some time."
+echo ""
 read -p "Would you like to continue? [y/N] " REPLY
 echo    # (optional) move to a new line
 if [ "$REPLY" != "y" ] && [ "$REPLY" != "Y" ]
@@ -167,7 +171,7 @@ EOF
 
 if ! dpkg-query --show --showformat='${db:Status-Status}\n' ubuntu-minimal 2> /dev/null | grep -q '^installed$'; then
     echo "Installing ubuntu-minimal package to provide the familiar Ubuntu minimal system..."
-    DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-minimal
+    DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-minimal ubuntu-standard
 fi
 
 if dpkg-query --show --showformat='${db:Status-Status}\n' ubuntu-server 2> /dev/null | grep -q '^installed$' \
@@ -264,7 +268,7 @@ EOF
 		rm -rf chroot/var/lib/apt/lists/*
 	fi
 
-	if [ -f config/oem-config-preinstalled ]; then
+	if [ -f config/universe-enabled ]; then
 
 # This is cargo-culted almost verbatim (with some syntax changes for
 # preinstalled being slightly different in what it doesn't ask) from
@@ -441,6 +445,8 @@ EOF
 	(cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr)
 	echo END docdirs
 
+	/usr/share/livecd-rootfs/minimize-manual chroot
+
 	lb binary "$@"
 	touch binary.success
 ) 2>&1 | tee binary.log
@@ -486,7 +492,7 @@ for ISO in binary.iso binary.hybrid.iso; do
 done
 
 if [ -e "binary/$INITFS/filesystem.dir" ]; then
-	(cd "binary/$INITFS/filesystem.dir/" && tar -c *) | \
+	(cd "binary/$INITFS/filesystem.dir/" && tar -c --xattrs *) | \
 		gzip -9 --rsyncable > "$PREFIX.rootfs.tar.gz"
 	chmod 644 "$PREFIX.rootfs.tar.gz"
 elif [ -e binary-tar.tar.gz ]; then
@@ -533,7 +539,7 @@ EOF
 fi
 
 if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then
-	(cd "binary/$INITFS/custom.dir/" && tar -c *) | \
+	(cd "binary/$INITFS/custom.dir/" && tar -c --xattrs *) | \
 		gzip -9 --rsyncable > "$PREFIX.custom.tar.gz"
 	chmod 644 "$PREFIX.custom.tar.gz"
 fi

live-build/auto/config

@@ -17,7 +17,7 @@ if [ -z "$MIRROR" ]; then
 		*.ubuntu.com)		MIRROR=http://ftpmaster.internal/ubuntu/
 					SEEDMIRROR=http://archive-team.internal/seeds/
 					;;
-		*.buildd|lp-*)		MIRROR=http://ftpmaster.internal/ubuntu/
+		*.buildd)		MIRROR=http://ftpmaster.internal/ubuntu/
 					SEEDMIRROR=http://archive-team.internal/seeds/
 					;;
 		*.ppa|*.scalingstack)	MIRROR=http://ftpmaster.internal/ubuntu/
@@ -151,6 +151,7 @@ case $IMAGEFORMAT in
 				#  by skipping lb_chroot_live-packages.
 				skip_lb_stage chroot_live-packages
 				INITRAMFS_TYPE=auto
+	                        touch config/universe-enabled
 				;;
 			*)
 				PREINSTALLED=true
@@ -213,9 +214,9 @@ case $IMAGEFORMAT in
 esac
 
 if [ "$PREINSTALLED" = "true" ]; then
-	# This is an oem-config preinstalled image, touch a random file that
-	# we can refer back to during build, cause that's wildly hackish
-	touch config/oem-config-preinstalled
+	# Touch a random file that we can refer back to during build,
+	# cause that's wildly hackish
+	touch config/universe-enabled
 	case $PROJECT in
 		kubuntu*)
 			add_package live oem-config-kde ubiquity-frontend-kde
@@ -843,6 +844,7 @@ lb config noauto \
 	--initsystem none \
 	--bootloader "$BOOTLOADER" \
 	${INITRAMFS_COMPRESSION:+--initramfs-compression "$INITRAMFS_COMPRESSION"} \
+	--checksums none \
 	--cache false \
 	${BOOTAPPEND_LIVE:+--bootappend-live "$BOOTAPPEND_LIVE"} \
 	$OPTS \

live-build/ubuntu-cpc/hooks/031-1-root-xz.binary

@@ -12,4 +12,4 @@ fi
 rootfs_dir=rootfs.dir
 
 cp $rootfs_dir.manifest livecd.ubuntu-cpc.rootfs.manifest
-(cd $rootfs_dir/ && tar -c *) | xz > livecd.ubuntu-cpc.rootfs.tar.xz
+(cd $rootfs_dir/ && tar -c --xattrs *) | xz > livecd.ubuntu-cpc.rootfs.tar.xz

live-build/ubuntu-server/includes.binary/overlay/etc/systemd/journald.conf.d/no-rate-limit.conf

@@ -0,0 +1,2 @@
+[Journal]
+RateLimitIntervalSec=0

minimize-manual

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+"""Minimize the number of manually installed packages in the image.
+
+Finds all manually installed meta packages, and marks their dependencies
+as automatically installed.
+"""
+import sys
+
+import apt
+
+
+def is_root(pkg):
+    """Check if the package is a root package (manually inst. meta)"""
+    return (pkg.is_installed and
+            not pkg.is_auto_installed and
+            (pkg.section == "metapackages" or
+             pkg.section.endswith("/metapackages")))
+
+
+def main():
+    """Main function"""
+    cache = apt.Cache(rootdir=sys.argv[1] if len(sys.argv) > 1 else None)
+    roots = set(pkg for pkg in cache if is_root(pkg))
+    workset = set(roots)
+    seen = set()
+
+    with cache.actiongroup():
+        while True:
+            print("Iteration", file=sys.stderr)
+            to_proc = workset - seen
+            if not to_proc:
+                break
+            for pkg in sorted(to_proc):
+                print("    Visiting", pkg, file=sys.stderr)
+
+                if pkg not in roots:
+                    pkg.mark_auto()
+
+                for dep in (pkg.installed.dependencies +
+                            pkg.installed.recommends):
+                    for bdep in dep.or_dependencies:
+                        for ver in bdep.target_versions:
+                            if ver.package.is_installed:
+                                workset.add(ver.package)
+
+                seen.add(pkg)
+
+        cache.commit()
+
+
+if __name__ == '__main__':
+    main()