diff --git a/debian/changelog b/debian/changelog index 266c6d42..546f681e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,11 +12,36 @@ livecd-rootfs (2.408.19) UNRELEASED; urgency=medium downstream. * Also nuke the sleep / udevadm settle calls in the process, which should never be required and slow down the builds. + * Begin adding support for a project-independent 'minimized' subproject, + which (ironically) omits ubuntu-minimal in favor of using only the minbase + package set. + * Export the subproject into config/chroot and config/binary, so that this + information is available to per-project hooks that need to be + subproject-aware (e.g., to skip steps when SUBPROJECT=minimized) + * Make the 999-cpc-fixes.chroot subproject-aware, so we don't try to + locale-gen + * In a cloud environment, we can rely on the kernel being able to boot the + root filesystem directly, without an initramfs; enable this when building + minimized. + * If we're using SUBPROJECT=minimized, and tzdata is not installed, remove + files that have been left behind. This is a workaround for a bug that + should be fixed in tzdata. + * Fix a reference to an undefined variable in a script that's set -u. + * Use /bin/sh, not /bin/bash, for autopkgtest. [ Balint Reczey ] * Mount using --make-rslave to ensure safe unmounts for rbind mounts - - -- Victor Tapia Tue, 19 Sep 2017 10:55:11 +0200 + * Drop man pages and most of the documentation from minimized images + /usr/share/doc/*/copyright and changelog.Debian.gz files are still kept + * Add unminimize script for reverting minimization on running system + * Install ubuntu-minimal while unminimizing the system + * Bump needed live-build version which can build images without initrd + * Mention unminimize script in motd + * Warn users that unminimize may fail reinstalling packages + * Run autopkgtest for SUBPROJECT=minimized + * When SUBPROJECT environment variable is not set assume it to be "" + + -- Balint Reczey Tue, 03 Oct 2017 17:35:04 +0200 livecd-rootfs (2.408.18) xenial; urgency=medium diff --git a/debian/control b/debian/control index d9a7e581..de508def 100644 --- a/debian/control +++ b/debian/control @@ -8,7 +8,7 @@ Vcs-Bzr: lp:~ubuntu-core-dev/livecd-rootfs/xenial-proposed/ Package: livecd-rootfs Architecture: any -Depends: ${misc:Depends}, debootstrap, rsync, python-minimal | python, procps, squashfs-tools (>= 1:3.3-1), grep-dctrl, lsb-release, lzma, e2fsprogs, germinate (>= 1.25.1), apt-utils, gnupg, live-build (>= 3.0~a57-1ubuntu12~), android-tools-fsutils [armhf], python3-software-properties +Depends: ${misc:Depends}, debootstrap, rsync, python-minimal | python, procps, squashfs-tools (>= 1:3.3-1), grep-dctrl, lsb-release, lzma, e2fsprogs, germinate (>= 1.25.1), apt-utils, gnupg, live-build (>= 3.0~a57-1ubuntu25.5~), android-tools-fsutils [armhf], python3-software-properties Suggests: partimage Breaks: ubuntu-defaults-builder (<< 0.32) Description: construction script for the livecd rootfs diff --git a/debian/tests/control b/debian/tests/control index b40640f9..671e49e9 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -1,3 +1,7 @@ Tests: default-bootstraps Depends: @, lsb-release Restrictions: needs-root isolation-machine + +Tests: minimized +Depends: @, lsb-release +Restrictions: needs-root isolation-machine diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps index 23631d4c..53e5a3bb 100755 --- a/debian/tests/default-bootstraps +++ b/debian/tests/default-bootstraps @@ -30,6 +30,7 @@ ALL_TRIPLETS=" ubuntu-budgie-live:: ubuntu-core:system-image:ubuntu-core ubuntu-cpc::ubuntu-cpc + ubuntu-cpc:minimized:ubuntu-cpc ubuntu-desktop-next:system-image:ubuntu-desktop-next ubuntu-desktop-next::ubuntu-desktop-next ubuntu-dvd:: diff --git a/debian/tests/minimized b/debian/tests/minimized new file mode 100644 index 00000000..c5da93ee --- /dev/null +++ b/debian/tests/minimized @@ -0,0 +1,3 @@ +#!/bin/sh + +env SELECTED_TRIPLETS=ubuntu-cpc:minimized:ubuntu-cpc debian/tests/default-bootstraps diff --git a/live-build/auto/build b/live-build/auto/build index 9e96dd0f..443482e3 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -31,6 +31,111 @@ Expire-Date: 0 lb bootstrap "$@" + if [ "${SUBPROJECT:-}" = minimized ] \ + && ! Chroot chroot dpkg -l tzdata 2>&1 |grep -q ^ii; then + # workaround for tzdata purge not removing these files + rm -f chroot/etc/localtime chroot/etc/timezone + fi + + if [ "${SUBPROJECT:-}" = minimized ]; then + # set up dpkg filters to skip installing docs on minimizedd system + mkdir -p chroot/etc/dpkg/dpkg.cfg.d + cat > chroot/etc/dpkg/dpkg.cfg.d/excludes < chroot/usr/local/sbin/unminimize <<'EOF' +#!/bin/sh + +set -e + +echo "This system was optimized for smaller footprint by reducing the available" +echo "installed documentation, removal of rarely used tools, and by other means." +echo "" +echo "This script reverts part of the optimization and makes documentation and" +echo "tools available again to match the familiar Ubuntu minimial system." +echo "" +echo "Be warned that reinstallation of packages may fail due to changes to the" +echo "system configuration, due to installation of 3rd party packages, or for" +echo "other reasons." +read -p "Would you like to continue? [y/N]" REPLY +echo # (optional) move to a new line +if [ "$REPLY" != "y" ] && [ "$REPLY" != "Y" ] +then + exit 1 +fi + +if [ -f /etc/dpkg/dpkg.cfg.d/excludes ] || [ -f /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp ]; then + echo "Re-enabling installation of all documentation in dpkg..." + if [ -f /etc/dpkg/dpkg.cfg.d/excludes ]; then + mv /etc/dpkg/dpkg.cfg.d/excludes /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp + fi + echo "Updating package list and upgrading packages..." + apt-get update + # apt-get upgrade asks for confirmation before upgrading packages to let the user stop here + apt-get upgrade + echo "Restoring system documentation..." + echo "Reinstalling packages with files in /usr/share/man/ ..." + # Reinstallation takes place in two steps because a single dpkg --verified + # command generates very long parameter list for "xargs dpkg -S" and may go + # over ARG_MAX. Since many packages have man pages the second download + # handles a much smaller amount of packages. + dpkg -S /usr/share/man/ |sed 's|, |\n|g;s|: [^:]*$||' | DEBIAN_FRONTEND=noninteractive xargs apt-get install --reinstall -y + echo "Reinstalling packages with system documentation in /usr/share/doc/ .." + # This step processes the packages which still have missing documentation + dpkg --verify --verify-format rpm | awk '/..5...... \/usr\/share\/doc/ {print $2}' | sed 's|/[^/]*$||' | sort |uniq \ + | xargs dpkg -S | sed 's|, |\n|g;s|: [^:]*$||' | uniq | DEBIAN_FRONTEND=noninteractive xargs apt-get install --reinstall -y + if dpkg --verify --verify-format rpm | awk '/..5...... \/usr\/share\/doc/ {exit 1}'; then + echo "Documentation has been restored successfully." + rm /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp + else + echo "There are still files missing from /usr/share/doc/:" + dpkg --verify --verify-format rpm | awk '/..5...... \/usr\/share\/doc/ {print " " $2}' + echo "You may want to try running this script again or you can remove" + echo "/etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp and restore the files manually." + fi +fi + +if ! dpkg-query --show --showformat='${db:Status-Status}\n' ubuntu-minimal 2> /dev/null | grep -q '^installed$'; then + echo "Installing ubuntu-minimal package to provide the familiar Ubuntu minimal system..." + DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-minimal +fi + +# unminimization succeeded, there is no need to mention it in motd +rm -f /etc/update-motd.d/60-unminimize + +EOF + chmod +x chroot/usr/local/sbin/unminimize + fi + + # inform users about the unminimize script + cat > "chroot/etc/update-motd.d/60-unminimize" << EOF +#!/bin/sh +# +# This file is not managed by a package. If you no longer want to +# see this message you can safely remove the file. +echo "" +echo "This system does not provide part of the tools and documentation" +echo "which are available on standard Ubuntu systems." +echo "To make them available please run the "unminimize" command." +EOF + + chmod +x chroot/etc/update-motd.d/60-unminimize Chroot chroot "dpkg-divert --quiet --add \ --divert /usr/sbin/update-initramfs.REAL --rename \ /usr/sbin/update-initramfs" @@ -249,7 +354,7 @@ elif [ -e binary-tar.tar.gz ]; then cp -a binary-tar.tar.gz "$PREFIX.rootfs.tar.gz" fi -if [ "$PROJECT:$SUBPROJECT" = "ubuntu-core:system-image" ]; then +if [ "$PROJECT:${SUBPROJECT:-}" = "ubuntu-core:system-image" ]; then if [ -e "binary/$INITFS/filesystem.dir" ]; then rootfs="binary/$INITFS/filesystem.dir" @@ -297,7 +402,7 @@ fi # ubuntu-core and ubuntu-desktop-next splits kernel stuff into a "device" tarball so # at this point we reset it to "none" as all the work to extract it was done already # in a binary hook -case $PROJECT:$SUBPROJECT in +case $PROJECT:${SUBPROJECT:-} in ubuntu-core:system-image|ubuntu-desktop-next:system-image) # create device tarball (for snappy only atm) diff --git a/live-build/auto/config b/live-build/auto/config index cbdeea85..fe51505c 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -117,7 +117,7 @@ case $IMAGEFORMAT in ext2|ext3|ext4) OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT" PREINSTALLED=true - case $SUBPROJECT in + case ${SUBPROJECT:-} in wubi) add_package install lupin-support COMPONENTS='main restricted universe multiverse' @@ -196,6 +196,10 @@ HWE_X_PACKAGES="xserver-xorg-hwe-16.04 xserver-xorg-video-all-hwe-16.04 xserver- HWE_KERNEL_FLAVOUR="generic-hwe-16.04" HWE_SIGNED_KERNEL_PACKAGE="linux-signed-$HWE_KERNEL_FLAVOUR" +if [ "${SUBPROJECT:-}" = minimized ]; then + OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal" +fi + case $PROJECT in ubuntu|ubuntu-dvd) HWE_BUILD="yes" @@ -223,7 +227,7 @@ case $PROJECT in # CDIMAGE_PREINSTALLED is not passed from build.py # and PREINSTALLED means something different. So # we use SUBPROJECT to pass on the information - if [ "$SUBPROJECT" = "system-image" ]; then + if [ "${SUBPROJECT:-}" = "system-image" ]; then OPTS="${OPTS:+$OPTS }--linux-packages=linux-image" fi ;; @@ -477,7 +481,7 @@ case $PROJECT in # SUBPROJECT, but it's a handy thing that launchpad-buildd # already passes through to us that we weren't otherwise # using here. - case $SUBPROJECT in + case ${SUBPROJECT:-} in ubuntu-rtm/dogfood) MIRROR=http://derived-archive.dogfood.content.paddev.net/ubuntu-rtm/ OPTS="${OPTS:+$OPTS }--apt-secure false" @@ -495,8 +499,13 @@ case $PROJECT in ;; ubuntu-cpc) - add_task install minimal standard cloud-image - add_package install ubuntu-minimal + if [ "${SUBPROJECT:-}" = minimized ]; then + add_task install cloud-image + add_package install sudo + else + add_task install minimal standard cloud-image + add_package install ubuntu-minimal + fi BINARY_REMOVE_LINUX=false OPTS="${OPTS:+$OPTS }--initramfs=none" @@ -630,7 +639,7 @@ case $PROJECT in ;; esac -case $SUBPROJECT in +case ${SUBPROJECT:-} in wubi) add_binary_hook build-wubildr ;; @@ -656,8 +665,10 @@ lb config noauto \ "$@" echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot +echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary echo "BUILDSTAMP=\"$NOW\"" >> config/binary +echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary case $ARCH+$SUBARCH in armhf+raspi2) @@ -772,7 +783,7 @@ EOF config/archives/proposed.list.binary fi -case $PROJECT:$SUBPROJECT in +case $PROJECT:${SUBPROJECT:-} in *-dvd:*) . config/bootstrap @@ -858,7 +869,7 @@ EOF fi fi -case $SUBPROJECT in +case ${SUBPROJECT:-} in ubuntu-rtm|ubuntu-rtm/*) # debootstrap doesn't know about ubuntu-rtm series directly. Rather # than having to teach it, we employ a few hacks to make it use the diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 1cde487a..6e2cd034 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -273,6 +273,11 @@ replace_grub_root_with_label() { # Instead, we want grub to use the cloudimg-rootfs labelled disk CHROOT_ROOT="$1" + # If boot by partuuid has been requested, don't override. + if [ -f $CHROOT_ROOT/etc/default/grub.d/40-partuuid ] && \ + grep -q ^GRUB_FORCE_PARTUUID= $CHROOT_ROOT/etc/default/grub.d/40-partuuid; then + return 0 + fi sed -i -e "s,root=[^ ]\+,root=LABEL=cloudimg-rootfs," \ "$CHROOT_ROOT/boot/grub/grub.cfg" } diff --git a/live-build/ubuntu-cpc/hooks/032-disk-image.binary b/live-build/ubuntu-cpc/hooks/032-disk-image.binary index 9c7d42ea..7cde62f6 100755 --- a/live-build/ubuntu-cpc/hooks/032-disk-image.binary +++ b/live-build/ubuntu-cpc/hooks/032-disk-image.binary @@ -2,6 +2,8 @@ . config/functions +. config/binary + BOOTPART_START= BOOTPART_END= BOOT_MOUNTPOINT= @@ -64,6 +66,8 @@ create_empty_partition "${disk_image}" "$ROOTPART" "$ROOTPART_START" -1 ext2 "$R mount_image "${disk_image}" "$ROOTPART" +partuuid=$(blkid -s PARTUUID -o value "$rootfs_dev_mapper") + # Copy the chroot in to the disk make_ext4_partition "${rootfs_dev_mapper}" mkdir mountpoint @@ -111,6 +115,13 @@ if [ "${should_install_grub}" -eq 1 ]; then ${loop_device} rm mountpoint/tmp/device.map + + if [ "${SUBPROJECT:-}" = minimized ] && [ -n "$partuuid" ]; then + echo "partuuid found for root device; forcing it in Grub" + mkdir -p mountpoint/etc/default/grub.d + echo "GRUB_FORCE_PARTUUID=$partuuid" >> mountpoint/etc/default/grub.d/40-force-partuuid + chroot mountpoint update-grub + fi fi if [ "$ARCH" = "s390x" ]; then diff --git a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary index dadadb16..2e98e356 100755 --- a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary +++ b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary @@ -61,6 +61,14 @@ install_grub() { efi_boot_dir="/boot/efi/EFI/BOOT" chroot mountpoint mkdir -p "${efi_boot_dir}" + if [ "${SUBPROJECT:-}" = minimized ] && [ -n "$partuuid" ]; then + # FIXME: code duplicated between 032-disk-image.binary + # and 033-disk-image-uefi.binary. We want to fix this to not + # have initramfs-tools installed at all on these images. + echo "partuuid found for root device; omitting initrd" + echo "GRUB_FORCE_PARTUUID=$partuuid" >> mountpoint/etc/default/grub.d/40-force-partuuid + fi + chroot mountpoint apt-get -y update # The modules below only make sense on non-Secure Boot UEFI systems. @@ -132,6 +140,8 @@ create_empty_disk_image "${disk_image}" create_partitions "${disk_image}" mount_image "${disk_image}" 1 +partuuid=$(blkid -s PARTUUID -o value "$rootfs_dev_mapper") + # Copy the chroot in to the disk make_ext4_partition "${rootfs_dev_mapper}" mkdir mountpoint diff --git a/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot b/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot index cf6b6bdb..4a2df66e 100755 --- a/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot +++ b/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot @@ -3,6 +3,8 @@ rootd="${1:-/}" root_fs_label=cloudimg-rootfs set -ex +. /root/config/chroot + CLOUD_IMG_STR="# CLOUD_IMG: This file was created/modified by the Cloud Image build process" LANG=C @@ -87,7 +89,9 @@ _xchroot "${rootd}" sh -c 'sed -i "/^127.0.1.1/d" /etc/hosts' _xchroot "${rootd}" sh -c 'rm -f /etc/ssh/ssh_host_[rd]sa_key*' ## -------------- -_xchroot "${rootd}" locale-gen en_US.utf8 +if [ "${SUBPROJECT:-}" != minimized ]; then + _xchroot "${rootd}" locale-gen en_US.utf8 +fi ## -------------- # set cloud-init to be on