Merge lp:~rbalint/livecd-rootfs/minimize-unminimize

debian/changelog

@@ -1,10 +1,30 @@
 livecd-rootfs (2.460) UNRELEASED; urgency=medium
 
+  [ Steve Langasek ]
+  * Begin adding support for a project-independent 'minimize' subproject, which
+    (ironically) omits ubuntu-minimal in favor of using only the minbase
+    package set.
+  * Export the subproject into config/chroot and config/binary, so that this
+    information is available to per-project hooks that need to be
+    subproject-aware (e.g., to skip steps when SUBPROJECT=minimize)
+  * Make the 999-cpc-fixes.chroot subproject-aware, so we don't try to
+    locale-gen
+  * In a cloud environment, we can rely on the kernel being able to boot the
+    root filesystem directly, without an initramfs; enable this when building
+    minimized.
+  * If we're using SUBPROJECT=minimize, and tzdata is not installed, remove
+    files that have been left behind.  This is a workaround for a bug that
+    should be fixed in tzdata.
+
   [ Balint Reczey ]
   * Mount using --make-rslave to ensure safe unmounts for rbind mounts
   * Don't ask for password and GECOS while creating vagrant user
+  * Drop man pages and most of the documentation from minimized images
+    /usr/share/doc/*/copyright and changelog.Debian.gz files are still kept
+  * Add unminimize script for reverting minimization on running system
+  * Install ubuntu-minimal while unminimizing the system
 
- -- Steve Langasek <steve.langasek@ubuntu.com>  Tue, 26 Sep 2017 00:59:08 -0400
+ -- Balint Reczey <rbalint@ubuntu.com>  Wed, 27 Sep 2017 12:29:02 -0400
 
 livecd-rootfs (2.459) artful; urgency=medium
 

live-build/auto/build

@@ -45,6 +45,79 @@ Expire-Date: 0
 
 	lb bootstrap "$@"
 
+	if [ "$SUBPROJECT" = minimize ] \
+	   && ! Chroot chroot dpkg -l tzdata 2>&1 |grep -q ^ii; then
+		# workaround for tzdata purge not removing these files
+		rm -f chroot/etc/localtime chroot/etc/timezone
+	fi
+
+	if [ "$SUBPROJECT" = minimize ]; then
+		# set up dpkg filters to skip installing docs on minimized system
+		mkdir -p chroot/etc/dpkg/dpkg.cfg.d
+		cat > chroot/etc/dpkg/dpkg.cfg.d/excludes <<EOF
+# Drop all man pages
+path-exclude=/usr/share/man/*
+
+# Drop all documentation ...
+path-exclude=/usr/share/doc/*
+
+# ... except copyright files ...
+path-include=/usr/share/doc/*/copyright
+
+# ... and Debian changelogs
+path-include=/usr/share/doc/*/changelog.Debian.*
+EOF
+
+		# Remove docs installed by bootstrap
+		Chroot chroot dpkg-query -f '${binary:Package}\n' -W | Chroot chroot xargs apt-get install --reinstall
+
+                # Add unminimizer script which restores default image behavior
+                mkdir -p chroot/usr/local/sbin
+                cat > chroot/usr/local/sbin/unminimize <<'EOF'
+#!/bin/sh
+
+set -e
+
+if [ -f /etc/dpkg/dpkg.cfg.d/excludes ] || [ -f /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp ]; then
+    echo "Re-enabling installation of all documentation in dpkg..."
+    if [ -f /etc/dpkg/dpkg.cfg.d/excludes ]; then
+        mv /etc/dpkg/dpkg.cfg.d/excludes /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp
+    fi
+    echo "Updating package list and upgrading packages..."
+    apt-get update
+    # apt-get upgrade asks for confirmation before upgrading packages to let the user stop here
+    apt-get upgrade
+    echo "Restoring system documentation..."
+    echo "Reinstalling packages with files in /usr/share/man/ ..."
+    # Reinstallation takes place in two steps because a single dpkg --verified
+    # command generates very long parameter list for "xargs dpkg -S" and may go
+    # over ARG_MAX. Since many packages have man pages the second download
+    # handles a much smaller amount of packages.
+    dpkg -S /usr/share/man/ |sed 's|, |\n|g;s|: [^:]*$||' | DEBIAN_FRONTEND=noninteractive xargs apt-get install --reinstall -y
+    echo "Reinstalling packages with system documentation in /usr/share/doc/ .."
+    # This step processes the packages which still have missing documentation
+    dpkg --verify --verify-format rpm | awk '/..5......   \/usr\/share\/doc/ {print $2}' | sed 's|/[^/]*$||' | sort |uniq \
+         | xargs dpkg -S | sed 's|, |\n|g;s|: [^:]*$||' | uniq | DEBIAN_FRONTEND=noninteractive xargs apt-get install --reinstall -y
+    if dpkg --verify --verify-format rpm | awk '/..5......   \/usr\/share\/doc/ {exit 1}'; then
+        echo "Documentation has been restored successfully."
+        rm /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp
+    else
+        echo "There are still files missing from /usr/share/doc/:"
+        dpkg --verify --verify-format rpm | awk '/..5......   \/usr\/share\/doc/ {print " " $2}'
+        echo "You may want to try running this script again or you can remove"
+        echo "/etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp and restore the files manually."
+    fi
+fi
+
+if ! dpkg-query --show --showformat='${db:Status-Status}\n' ubuntu-minimal 2> /dev/null | grep -q '^installed$'; then
+    echo "Installing ubuntu-minimal package to provide the familiar Ubuntu minimal system..."
+    DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-minimal
+fi
+
+EOF
+                chmod +x chroot/usr/local/sbin/unminimize
+        fi
+
 	Chroot chroot "dpkg-divert --quiet --add \
 		--divert /usr/sbin/update-initramfs.REAL --rename \
 		/usr/sbin/update-initramfs"

live-build/auto/config

@@ -238,6 +238,10 @@ esac
 
 SIGNED_KERNEL_PACKAGE="linux-signed-generic"
 
+if [ "$SUBPROJECT" = minimize ]; then
+	OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal"
+fi
+
 case $PROJECT in
 	ubuntu|ubuntu-dvd)
 		add_task install minimal standard ubuntu-desktop
@@ -524,8 +528,13 @@ case $PROJECT in
 		;;
 
 	ubuntu-cpc)
-		add_task install minimal standard cloud-image
-		add_package install ubuntu-minimal
+		if [ "$SUBPROJECT" = minimize ]; then
+			add_task install cloud-image
+			add_package install sudo
+		else
+			add_task install minimal standard cloud-image
+			add_package install ubuntu-minimal
+		fi
 
 		BINARY_REMOVE_LINUX=false
 		OPTS="${OPTS:+$OPTS }--initramfs=none"
@@ -679,8 +688,10 @@ lb config noauto \
 	"$@"
 
 echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot
+echo "SUBPROJECT=\"$SUBPROJECT\"" >> config/chroot
 echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary
 echo "BUILDSTAMP=\"$NOW\"" >> config/binary
+echo "SUBPROJECT=\"$SUBPROJECT\"" >> config/binary
 
 case $ARCH+$SUBARCH in
 	armhf+raspi2)

live-build/functions

@@ -275,6 +275,11 @@ replace_grub_root_with_label() {
     # Instead, we want grub to use the right labelled disk
     CHROOT_ROOT="$1"
 
+    # If boot by partuuid has been requested, don't override.
+    if [ -f $CHROOT_ROOT/etc/default/grub.d/40-partuuid ] && \
+           grep -q ^GRUB_FORCE_PARTUUID= $CHROOT_ROOT/etc/default/grub.d/40-partuuid; then
+        return 0
+    fi
     sed -i -e "s,root=[^ ]\+,root=LABEL=${fs_label}," \
         "$CHROOT_ROOT/boot/grub/grub.cfg"
 }

live-build/ubuntu-cpc/hooks/032-disk-image.binary

@@ -5,6 +5,8 @@ FS_LABEL="cloudimg-rootfs"
 
 . config/functions
 
+. config/binary
+
 BOOTPART_START=
 BOOTPART_END=
 BOOT_MOUNTPOINT=
@@ -69,6 +71,8 @@ create_empty_partition "${disk_image}" "$ROOTPART" "$ROOTPART_START" -1 ext2 "$R
 
 mount_image "${disk_image}" "$ROOTPART"
 
+partuuid=$(blkid -s PARTUUID -o value "$rootfs_dev_mapper")
+
 # Copy the chroot in to the disk
 make_ext4_partition "${rootfs_dev_mapper}"
 mkdir mountpoint
@@ -116,6 +120,13 @@ if [ "${should_install_grub}" -eq 1 ]; then
         ${loop_device}
 
     rm mountpoint/tmp/device.map
+
+	if [ "$SUBPROJECT" = minimize ] && [ -n "$partuuid" ]; then
+		echo "partuuid found for root device; forcing it in Grub"
+		mkdir -p mountpoint/etc/default/grub.d
+		echo "GRUB_FORCE_PARTUUID=$partuuid" >> mountpoint/etc/default/grub.d/40-force-partuuid
+		chroot mountpoint update-grub
+	fi
 fi
 
 if [ "$ARCH" = "s390x" ]; then

live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary

@@ -62,6 +62,14 @@ install_grub() {
     efi_boot_dir="/boot/efi/EFI/BOOT"
     chroot mountpoint mkdir -p "${efi_boot_dir}"
 
+    if [ "$SUBPROJECT" = minimize ] && [ -n "$partuuid" ]; then
+        # FIXME: code duplicated between 032-disk-image.binary
+        # and 033-disk-image-uefi.binary.  We want to fix this to not
+        # have initramfs-tools installed at all on these images.
+        echo "partuuid found for root device; omitting initrd"
+        echo "GRUB_FORCE_PARTUUID=$partuuid" >> mountpoint/etc/default/grub.d/40-force-partuuid
+    fi
+
     chroot mountpoint apt-get -y update
 
     # UEFI GRUB modules are meant to be used equally by Secure Boot and
@@ -122,6 +130,8 @@ create_empty_disk_image "${disk_image}"
 create_partitions "${disk_image}"
 mount_image "${disk_image}" 1
 
+partuuid=$(blkid -s PARTUUID -o value "$rootfs_dev_mapper")
+
 # Copy the chroot in to the disk
 make_ext4_partition "${rootfs_dev_mapper}"
 mkdir mountpoint

live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot

@@ -3,6 +3,8 @@ rootd="${1:-/}"
 root_fs_label=cloudimg-rootfs
 set -ex
 
+. /root/config/chroot
+
 CLOUD_IMG_STR="# CLOUD_IMG: This file was created/modified by the Cloud Image build process"
 
 LANG=C
@@ -57,7 +59,9 @@ _xchroot "${rootd}" sh -c 'sed -i "/^127.0.1.1/d" /etc/hosts'
 _xchroot "${rootd}" sh -c 'rm -f /etc/ssh/ssh_host_[rd]sa_key*'
 
 ## --------------
-_xchroot "${rootd}" locale-gen en_US.utf8
+if [ "${SUBPROJECT:-}" != minimize ]; then
+	_xchroot "${rootd}" locale-gen en_US.utf8
+fi
 
 ## --------------
 # We continue to pre-generate en_US.UTF-8 locale above, but the default locale