diff --git a/debian/changelog b/debian/changelog index 576982cd..2a084acd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,56 @@ +livecd-rootfs (2.408.20) xenial; urgency=medium + + * Now that grub-related diversions have been factored out in 2.466 + instead of having bogus root=stuff arg generated in grub.cfg, it is + actually empty. Therefore update the sed command to make the arg in + the root= token optional. This should resolve non-booting livecd cpc + images. + + -- Dimitri John Ledkov Tue, 10 Oct 2017 13:49:49 +0100 + +livecd-rootfs (2.408.19) xenial; urgency=medium + + [ Nishanth Aravamudan ] + * live-build/ubuntu-cpc/hooks/061-open-iscsi.chroot: generate iSCSI + Initiator Name at first iscsid run for cloud images to ensure it is + unique (LP: #1444992). + + [ Steve Langasek ] + * Improve teardown_mountpoint to recursively find all submounts and + unmount them, instead of working from a hard-coded list. This makes + the code resilient against other submounts being added later, including + downstream. LP: #1721279. + * Also nuke the sleep / udevadm settle calls in the process, which should + never be required and slow down the builds. + * Fix a reference to an undefined variable in a script that's set -u. + * Use /bin/sh, not /bin/bash, for autopkgtest. + * debian/tests/default-bootstraps: minor adjustments to shell syntax, + syncing with artful where this originated. + + [ Steve Langasek, Balint Reczey ] + * Introduce a new project-independent 'minimized' subproject + (LP: #1721261): + - omit ubuntu-minimal in favor of using only the minbase package set. + - boot directly by partuuid, avoiding the use of an initramfs. + - Bump needed live-build version which can build images without initrd + - drop man pages and most of the documentation from minimized images + (/usr/share/doc/*/copyright and changelog.Debian.gz files are still + kept) + - Add unminimize script for reverting minimization on a running system + - Mention unminimize script in motd + - Run autopkgtest for SUBPROJECT=minimized + - If we're using SUBPROJECT=minimized, and tzdata is not installed, + remove files that have been left behind. This is a workaround for a + bug that should be fixed in tzdata. + * Factor out grub-related diversions and use them consistently, so we + don't end up with wrong os-probe output in our grub.cfg. + + [ Balint Reczey ] + * Mount using --make-rslave to ensure safe unmounts for rbind mounts + * When SUBPROJECT environment variable is not set assume it to be "" + + -- Steve Langasek Wed, 04 Oct 2017 18:43:48 +0000 + livecd-rootfs (2.408.18) xenial; urgency=medium [ Robert C Jennings ] diff --git a/debian/control b/debian/control index d9a7e581..de508def 100644 --- a/debian/control +++ b/debian/control @@ -8,7 +8,7 @@ Vcs-Bzr: lp:~ubuntu-core-dev/livecd-rootfs/xenial-proposed/ Package: livecd-rootfs Architecture: any -Depends: ${misc:Depends}, debootstrap, rsync, python-minimal | python, procps, squashfs-tools (>= 1:3.3-1), grep-dctrl, lsb-release, lzma, e2fsprogs, germinate (>= 1.25.1), apt-utils, gnupg, live-build (>= 3.0~a57-1ubuntu12~), android-tools-fsutils [armhf], python3-software-properties +Depends: ${misc:Depends}, debootstrap, rsync, python-minimal | python, procps, squashfs-tools (>= 1:3.3-1), grep-dctrl, lsb-release, lzma, e2fsprogs, germinate (>= 1.25.1), apt-utils, gnupg, live-build (>= 3.0~a57-1ubuntu25.5~), android-tools-fsutils [armhf], python3-software-properties Suggests: partimage Breaks: ubuntu-defaults-builder (<< 0.32) Description: construction script for the livecd rootfs diff --git a/debian/tests/control b/debian/tests/control index b40640f9..671e49e9 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -1,3 +1,7 @@ Tests: default-bootstraps Depends: @, lsb-release Restrictions: needs-root isolation-machine + +Tests: minimized +Depends: @, lsb-release +Restrictions: needs-root isolation-machine diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps index 23631d4c..433cd327 100755 --- a/debian/tests/default-bootstraps +++ b/debian/tests/default-bootstraps @@ -5,7 +5,7 @@ set -e -if [ -z "$SUITE"]; then +if [ -z "$SUITE" ]; then SUITE=$(lsb_release -c -s) fi @@ -30,6 +30,7 @@ ALL_TRIPLETS=" ubuntu-budgie-live:: ubuntu-core:system-image:ubuntu-core ubuntu-cpc::ubuntu-cpc + ubuntu-cpc:minimized:ubuntu-cpc ubuntu-desktop-next:system-image:ubuntu-desktop-next ubuntu-desktop-next::ubuntu-desktop-next ubuntu-dvd:: @@ -66,19 +67,17 @@ live_build_rootfs() { if [ -n "$TEMPLATE" ]; then cp -a /usr/share/livecd-rootfs/live-build/$TEMPLATE . fi - (env PROJECT=$PROJECT \ + env PROJECT=$PROJECT \ SUBPROJECT=$SUBPROJECT \ SUITE=$SUITE \ ARCH=$ARCH \ lb config - ) mkdir chroot # this part needs root rights, but right now the whole script ran as root by autopkgtest - (env PROJECT=$PROJECT \ + env PROJECT=$PROJECT \ SUBPROJECT=$SUBPROJECT \ ARCH=$ARCH \ lb build - ) echo "Build results for project: '$PROJECT' subproject: '$SUBPROJECT' template: '$TEMPLATE' in $PWD" du -sh * echo "" diff --git a/debian/tests/minimized b/debian/tests/minimized new file mode 100644 index 00000000..0807a764 --- /dev/null +++ b/debian/tests/minimized @@ -0,0 +1,11 @@ +#!/bin/sh + +ARCH=$(dpkg --print-architecture) + +failure_code=1 +if [ "$ARCH" = ppc64el ]; then + # not a regression; don't block while debugging + failure_code=0 +fi + +env SELECTED_TRIPLETS=ubuntu-cpc:minimized:ubuntu-cpc debian/tests/default-bootstraps || exit $failure_code diff --git a/live-build/auto/build b/live-build/auto/build index 9e96dd0f..367e41f8 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -31,6 +31,112 @@ Expire-Date: 0 lb bootstrap "$@" + if [ "${SUBPROJECT:-}" = minimized ] \ + && ! Chroot chroot dpkg -l tzdata 2>&1 |grep -q ^ii; then + # workaround for tzdata purge not removing these files + rm -f chroot/etc/localtime chroot/etc/timezone + fi + + if [ "${SUBPROJECT:-}" = minimized ]; then + # set up dpkg filters to skip installing docs on minimized system + mkdir -p chroot/etc/dpkg/dpkg.cfg.d + cat > chroot/etc/dpkg/dpkg.cfg.d/excludes < chroot/usr/local/sbin/unminimize <<'EOF' +#!/bin/sh + +set -e + +echo "This system has been minimized by removing packages and content that are" +echo "not required on a system that users do not log into." +echo "" +echo "This script restores the content and packages that are found on a default" +echo "Ubuntu server system." +echo "" +echo "Reinstallation of packages may fail due to changes to the system" +echo "configuration, the presence of third-party packages, or for other" +echo "reasons." +read -p "Would you like to continue? [y/N]" REPLY +echo # (optional) move to a new line +if [ "$REPLY" != "y" ] && [ "$REPLY" != "Y" ] +then + exit 1 +fi + +if [ -f /etc/dpkg/dpkg.cfg.d/excludes ] || [ -f /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp ]; then + echo "Re-enabling installation of all documentation in dpkg..." + if [ -f /etc/dpkg/dpkg.cfg.d/excludes ]; then + mv /etc/dpkg/dpkg.cfg.d/excludes /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp + fi + echo "Updating package list and upgrading packages..." + apt-get update + # apt-get upgrade asks for confirmation before upgrading packages to let the user stop here + apt-get upgrade + echo "Restoring system documentation..." + echo "Reinstalling packages with files in /usr/share/man/ ..." + # Reinstallation takes place in two steps because a single dpkg --verified + # command generates very long parameter list for "xargs dpkg -S" and may go + # over ARG_MAX. Since many packages have man pages the second download + # handles a much smaller amount of packages. + dpkg -S /usr/share/man/ |sed 's|, |\n|g;s|: [^:]*$||' | DEBIAN_FRONTEND=noninteractive xargs apt-get install --reinstall -y + echo "Reinstalling packages with system documentation in /usr/share/doc/ .." + # This step processes the packages which still have missing documentation + dpkg --verify --verify-format rpm | awk '/..5...... \/usr\/share\/doc/ {print $2}' | sed 's|/[^/]*$||' | sort |uniq \ + | xargs dpkg -S | sed 's|, |\n|g;s|: [^:]*$||' | uniq | DEBIAN_FRONTEND=noninteractive xargs apt-get install --reinstall -y + if dpkg --verify --verify-format rpm | awk '/..5...... \/usr\/share\/doc/ {exit 1}'; then + echo "Documentation has been restored successfully." + rm /etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp + else + echo "There are still files missing from /usr/share/doc/:" + dpkg --verify --verify-format rpm | awk '/..5...... \/usr\/share\/doc/ {print " " $2}' + echo "You may want to try running this script again or you can remove" + echo "/etc/dpkg/dpkg.cfg.d/excludes.dpkg-tmp and restore the files manually." + fi +fi + +if ! dpkg-query --show --showformat='${db:Status-Status}\n' ubuntu-minimal 2> /dev/null | grep -q '^installed$'; then + echo "Installing ubuntu-minimal package to provide the familiar Ubuntu minimal system..." + DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-minimal +fi + +# unminimization succeeded, there is no need to mention it in motd +rm -f /etc/update-motd.d/60-unminimize + +EOF + chmod +x chroot/usr/local/sbin/unminimize + + # inform users about the unminimize script + cat > "chroot/etc/update-motd.d/60-unminimize" << EOF +#!/bin/sh +# +# This file is not managed by a package. If you no longer want to +# see this message you can safely remove the file. +echo "This system has been minimized by removing packages and content that are" +echo "not required on a system that users do not log into." +echo "" +echo "To restore this content, you can run the 'unminimize' command." +EOF + + chmod +x chroot/etc/update-motd.d/60-unminimize + fi + Chroot chroot "dpkg-divert --quiet --add \ --divert /usr/sbin/update-initramfs.REAL --rename \ /usr/sbin/update-initramfs" @@ -249,7 +355,7 @@ elif [ -e binary-tar.tar.gz ]; then cp -a binary-tar.tar.gz "$PREFIX.rootfs.tar.gz" fi -if [ "$PROJECT:$SUBPROJECT" = "ubuntu-core:system-image" ]; then +if [ "$PROJECT:${SUBPROJECT:-}" = "ubuntu-core:system-image" ]; then if [ -e "binary/$INITFS/filesystem.dir" ]; then rootfs="binary/$INITFS/filesystem.dir" @@ -297,7 +403,7 @@ fi # ubuntu-core and ubuntu-desktop-next splits kernel stuff into a "device" tarball so # at this point we reset it to "none" as all the work to extract it was done already # in a binary hook -case $PROJECT:$SUBPROJECT in +case $PROJECT:${SUBPROJECT:-} in ubuntu-core:system-image|ubuntu-desktop-next:system-image) # create device tarball (for snappy only atm) diff --git a/live-build/auto/config b/live-build/auto/config index cbdeea85..e73e2fe5 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -117,7 +117,7 @@ case $IMAGEFORMAT in ext2|ext3|ext4) OPTS="${OPTS:+$OPTS }--initramfs none --chroot-filesystem $IMAGEFORMAT" PREINSTALLED=true - case $SUBPROJECT in + case ${SUBPROJECT:-} in wubi) add_package install lupin-support COMPONENTS='main restricted universe multiverse' @@ -196,6 +196,10 @@ HWE_X_PACKAGES="xserver-xorg-hwe-16.04 xserver-xorg-video-all-hwe-16.04 xserver- HWE_KERNEL_FLAVOUR="generic-hwe-16.04" HWE_SIGNED_KERNEL_PACKAGE="linux-signed-$HWE_KERNEL_FLAVOUR" +if [ "${SUBPROJECT:-}" = minimized ]; then + OPTS="${OPTS:+$OPTS }--bootstrap-flavour=minimal --linux-packages=linux-image" +fi + case $PROJECT in ubuntu|ubuntu-dvd) HWE_BUILD="yes" @@ -223,7 +227,7 @@ case $PROJECT in # CDIMAGE_PREINSTALLED is not passed from build.py # and PREINSTALLED means something different. So # we use SUBPROJECT to pass on the information - if [ "$SUBPROJECT" = "system-image" ]; then + if [ "${SUBPROJECT:-}" = "system-image" ]; then OPTS="${OPTS:+$OPTS }--linux-packages=linux-image" fi ;; @@ -477,7 +481,7 @@ case $PROJECT in # SUBPROJECT, but it's a handy thing that launchpad-buildd # already passes through to us that we weren't otherwise # using here. - case $SUBPROJECT in + case ${SUBPROJECT:-} in ubuntu-rtm/dogfood) MIRROR=http://derived-archive.dogfood.content.paddev.net/ubuntu-rtm/ OPTS="${OPTS:+$OPTS }--apt-secure false" @@ -495,12 +499,31 @@ case $PROJECT in ;; ubuntu-cpc) - add_task install minimal standard cloud-image - add_package install ubuntu-minimal + if [ "${SUBPROJECT:-}" = minimized ]; then + add_task install cloud-image + add_package install sudo + # linux-kvm currently only exists in xenial, not in + # non-LTS suites. Fall back to virtual flavor, which + # may or may not boot initramfsless but enables us to + # test building and possibly build derivative images + # using other kernel flavors. + # If you enable an extra ppa, it is assumed that + # linux-kvm is available since you control the + # archive and can provide this metapackage as + # necessary. + if [ -z "$EXTRA_PPAS" ] && [ "$SUITE" != xenial ]; then + KERNEL_FLAVOURS=virtual + else + KERNEL_FLAVOURS=kvm + fi + else + add_task install minimal standard cloud-image + add_package install ubuntu-minimal + KERNEL_FLAVOURS=virtual + fi BINARY_REMOVE_LINUX=false OPTS="${OPTS:+$OPTS }--initramfs=none" - KERNEL_FLAVOURS=virtual case $ARCH in armhf) KERNEL_FLAVOURS=generic-lpae @@ -630,7 +653,7 @@ case $PROJECT in ;; esac -case $SUBPROJECT in +case ${SUBPROJECT:-} in wubi) add_binary_hook build-wubildr ;; @@ -656,8 +679,10 @@ lb config noauto \ "$@" echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot +echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary echo "BUILDSTAMP=\"$NOW\"" >> config/binary +echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary case $ARCH+$SUBARCH in armhf+raspi2) @@ -772,7 +797,7 @@ EOF config/archives/proposed.list.binary fi -case $PROJECT:$SUBPROJECT in +case $PROJECT:${SUBPROJECT:-} in *-dvd:*) . config/bootstrap @@ -858,7 +883,7 @@ EOF fi fi -case $SUBPROJECT in +case ${SUBPROJECT:-} in ubuntu-rtm|ubuntu-rtm/*) # debootstrap doesn't know about ubuntu-rtm series directly. Rather # than having to teach it, we employ a few hacks to make it use the diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 3e9e904f..246b839f 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -82,7 +82,7 @@ mount_image() { setup_mountpoint() { local mountpoint="$1" - mount --rbind /dev "$mountpoint/dev" + mount --rbind --make-rslave /dev "$mountpoint/dev" mount proc-live -t proc "$mountpoint/proc" mount sysfs-live -t sysfs "$mountpoint/sys" mount -t tmpfs none "$mountpoint/tmp" @@ -98,16 +98,13 @@ teardown_mountpoint() { # Reverse the operations from setup_mountpoint local mountpoint="$1" - umount "$mountpoint/var/cache/apt" - umount "$mountpoint/var/lib/apt" - umount "$mountpoint/tmp" - umount "$mountpoint/sys" - umount "$mountpoint/proc" - umount -R "$mountpoint/dev" - - apt-get install -qqy udev - udevadm settle - sleep 3 + # ensure we have exactly one trailing slash, and escape all slashes for awk + mountpoint_match=$(echo "$mountpoint" | sed -e's,/$,,; s,/,\\/,g;')'\/' + # sort -r ensures that deeper mountpoints are unmounted first + for submount in $(awk > mountpoint/etc/default/grub.d/40-force-partuuid.cfg + divert_grub mountpoint + chroot mountpoint update-grub + undivert_grub mountpoint + fi fi if [ "$ARCH" = "s390x" ]; then diff --git a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary index 81782b4b..da1d18dc 100755 --- a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary +++ b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary @@ -61,6 +61,14 @@ install_grub() { efi_boot_dir="/boot/efi/EFI/BOOT" chroot mountpoint mkdir -p "${efi_boot_dir}" + if [ "${SUBPROJECT:-}" = minimized ] && [ -n "$partuuid" ]; then + # FIXME: code duplicated between 032-disk-image.binary + # and 033-disk-image-uefi.binary. We want to fix this to not + # have initramfs-tools installed at all on these images. + echo "partuuid found for root device; omitting initrd" + echo "GRUB_FORCE_PARTUUID=$partuuid" >> mountpoint/etc/default/grub.d/40-force-partuuid.cfg + fi + chroot mountpoint apt-get -y update # The modules below only make sense on non-Secure Boot UEFI systems. @@ -111,19 +119,16 @@ EOF chroot mountpoint grub-install --target=i386-pc "${loop_device}" fi - chroot mountpoint dpkg-divert --local --rename /etc/grub.d/30_os-prober + divert_grub mountpoint chroot mountpoint update-grub replace_grub_root_with_label mountpoint - chroot mountpoint dpkg-divert --remove --local --rename /etc/grub.d/30_os-prober + undivert_grub mountpoint chroot mountpoint apt-get -y clean rm mountpoint/tmp/device.map - sync umount mountpoint/boot/efi - sleep 5 apt-get install -qqy udev - udevadm settle mount umount_partition mountpoint rmdir mountpoint @@ -135,6 +140,8 @@ create_empty_disk_image "${disk_image}" create_partitions "${disk_image}" mount_image "${disk_image}" 1 +partuuid=$(blkid -s PARTUUID -o value "$rootfs_dev_mapper") + # Copy the chroot in to the disk make_ext4_partition "${rootfs_dev_mapper}" mkdir mountpoint diff --git a/live-build/ubuntu-cpc/hooks/034-disk-image-ppc64el.binary b/live-build/ubuntu-cpc/hooks/034-disk-image-ppc64el.binary index 884626e8..7891f639 100755 --- a/live-build/ubuntu-cpc/hooks/034-disk-image-ppc64el.binary +++ b/live-build/ubuntu-cpc/hooks/034-disk-image-ppc64el.binary @@ -50,10 +50,10 @@ EOF --boot-directory=/boot \ --target=powerpc-ieee1275 - chroot mountpoint dpkg-divert --local --rename /etc/grub.d/30_os-prober + divert_grub mountpoint chroot mountpoint update-grub replace_grub_root_with_label mountpoint - chroot mountpoint dpkg-divert --remove --local --rename /etc/grub.d/30_os-prober + undivert_grub mountpoint umount_partition mountpoint rmdir mountpoint diff --git a/live-build/ubuntu-cpc/hooks/061-open-iscsi.chroot b/live-build/ubuntu-cpc/hooks/061-open-iscsi.chroot new file mode 100755 index 00000000..ba9b51e6 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/061-open-iscsi.chroot @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Set InitiatorName to be runtime generated when iscsid first starts, so +# that each cloud image gets a unique value +# + +if [ -f /etc/iscsi/initiatorname.iscsi ]; then + echo "GenerateName=yes" > /etc/iscsi/initiatorname.iscsi +fi diff --git a/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot b/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot index cf6b6bdb..4a2df66e 100755 --- a/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot +++ b/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot @@ -3,6 +3,8 @@ rootd="${1:-/}" root_fs_label=cloudimg-rootfs set -ex +. /root/config/chroot + CLOUD_IMG_STR="# CLOUD_IMG: This file was created/modified by the Cloud Image build process" LANG=C @@ -87,7 +89,9 @@ _xchroot "${rootd}" sh -c 'sed -i "/^127.0.1.1/d" /etc/hosts' _xchroot "${rootd}" sh -c 'rm -f /etc/ssh/ssh_host_[rd]sa_key*' ## -------------- -_xchroot "${rootd}" locale-gen en_US.utf8 +if [ "${SUBPROJECT:-}" != minimized ]; then + _xchroot "${rootd}" locale-gen en_US.utf8 +fi ## -------------- # set cloud-init to be on