diff --git a/debian/changelog b/debian/changelog index 102da40e..84c310ee 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,33 @@ +livecd-rootfs (2.525.12) bionic; urgency=medium + + * Key netplan delegation to NetworkManager on presence of + /usr/sbin/NetworkManager, not on /usr/lib/NetworkManager which may have + hooks from other packages (i.e., wpasupplicant). + + [ Cody Shepherd ] + * Include snaps in image manifests (LP: #1805497) + * Change call to add grub efi packages using new create_manifests() + function. + + -- Steve Langasek Mon, 10 Dec 2018 12:46:46 -0800 + +livecd-rootfs (2.525.11) bionic; urgency=medium + + * Backport all the required changes to enable Raspberry Pi 3 armhf and arm64 + preinstalled image builds. (LP: #1805668) + - Add support for raspi3 rootfs builds (based on Ryan Finnie's changes). + - For ubuntu-image consumption, export the kernel and initrd to + image/boot/uboot for raspi*. + - Avoid issues of hard-linking to a symbolic vmlinuz as this can lead to a + dangling symlink. + - Add raspi3 arm64 rootfs build support. + - Add git to the build dependencies for the gadget tree pull. + - Minor fixes to raspi3 builds: add arm64+raspi3 to the supported model + list, pass SUITE on to the build stage and use the git:// url for the + gadget tree. + + -- Łukasz 'sil2100' Zemczak Thu, 29 Nov 2018 16:24:23 +0100 + livecd-rootfs (2.525.10) bionic; urgency=medium [ Cody Shepherd ] diff --git a/debian/control b/debian/control index 74a03380..b0bd1030 100644 --- a/debian/control +++ b/debian/control @@ -17,12 +17,14 @@ Depends: ${misc:Depends}, gdisk, genisoimage, germinate (>= 1.25.1), + git, gnupg, grep-dctrl, kpartx, live-build (>= 3.0~a57-1ubuntu31~), lsb-release, lzma, + make, parted, procps, python-minimal | python, @@ -32,6 +34,7 @@ Depends: ${misc:Depends}, snapd, squashfs-tools (>= 1:3.3-1), sudo, + u-boot-tools [armhf arm64], ubuntu-image, vmdk-stream-converter [amd64 i386], xz-utils, diff --git a/live-build/auto/build b/live-build/auto/build index 110478a5..273f1a10 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -23,12 +23,31 @@ PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" if [ "${IMAGEFORMAT:-}" = "ubuntu-image" ]; then # Use ubuntu-image instead of live-build - CHANNEL="${CHANNEL:-edge}" - env SNAPPY_STORE_NO_CDN=1 \ - ubuntu-image -c "$CHANNEL" $UBUNTU_IMAGE_ARGS \ - -o "$PREFIX".img "$PREFIX".model-assertion - xz -0 -T4 "$PREFIX".img - mv seed.manifest "$PREFIX".manifest + if [ "$PROJECT" = "ubuntu-core"]; then + CHANNEL="${CHANNEL:-edge}" + env SNAPPY_STORE_NO_CDN=1 \ + ubuntu-image snap -c "$CHANNEL" $UBUNTU_IMAGE_ARGS \ + -O output "$PREFIX".model-assertion + # XXX: currently we only have one image generated, but really + # we should be supporting more than one for models that + # define those. + mv output/*.img "$PREFIX".img + xz -0 -T4 "$PREFIX".img + mv output/seed.manifest "$PREFIX".manifest + else + # First we need to build the gadget tree + make -C "config/$PREFIX-gadget" ARCH=$ARCH SERIES=$SUITE + ubuntu-image classic $UBUNTU_IMAGE_ARGS \ + -s $SUITE -p $PROJECT -a $ARCH --subarch $SUBARCH \ + -O output config/$PREFIX-gadget/install + # XXX: currently we only have one image generated, but really + # we should be supporting more than one for models that + # define those. + mv output/*.img "$PREFIX".img + xz -0 -T4 "$PREFIX".img + mv output/filesystem.manifest "$PREFIX".manifest + fi + exit 0 fi @@ -396,7 +415,7 @@ EOF # default. Installing NM on an existing system only manages wifi and wwan via # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting # the global backend to NM, netplan overrides that file. - if [ -d chroot/usr/lib/NetworkManager ]; then + if [ -e chroot/usr/sbin/NetworkManager ]; then echo "===== Enabling all devices in NetworkManager ====" mkdir -p chroot/etc/netplan cat < chroot/etc/netplan/01-network-manager-all.yaml @@ -503,6 +522,7 @@ fi # '--initramfs none' produces different manifest names. if [ -e "binary/$INITFS/filesystem.packages" ]; then + ./config/snap-seed-parse "chroot/" "binary/${INITFS}/filesystem.packages" ln "binary/$INITFS/filesystem.packages" "$PREFIX.manifest" chmod 644 "$PREFIX.manifest" fi @@ -800,13 +820,13 @@ done NUMFLAVOURS="$(set -- $LB_LINUX_FLAVOURS; echo $#)" if [ "$NUMFLAVOURS" = 1 ] && [ "$LB_LINUX_FLAVOURS" != "none" ]; then # only one kernel flavour - if [ -e "binary/$INITFS/vmlinuz" ]; then + if [ -f "binary/$INITFS/vmlinuz" ] && ! [ -h "binary/$INITFS/vmlinuz" ]; then ln "binary/$INITFS/vmlinuz" "$PREFIX.kernel" chmod 644 "$PREFIX.kernel" else ln -sf "$PREFIX.kernel-$LB_LINUX_FLAVOURS" "$PREFIX.kernel" fi - if [ -e "binary/$INITFS/initrd.lz" ]; then + if [ -f "binary/$INITFS/initrd.lz" ] && ! [ -h "binary/$INITFS/initrd.lz" ]; then ln "binary/$INITFS/initrd.lz" "$PREFIX.initrd" chmod 644 "$PREFIX.initrd" else @@ -814,36 +834,52 @@ if [ "$NUMFLAVOURS" = 1 ] && [ "$LB_LINUX_FLAVOURS" != "none" ]; then fi fi -if [ "$SUBARCH" = "ac100" ]; then - # create the md5sum and size files for which we are actually doing all this - md5sum $PREFIX.rootfs.tar.gz >chroot/installer.md5 - wc -c $PREFIX.rootfs.tar.gz >chroot/installer.size +case $SUBARCH in + ac100) + # create the md5sum and size files for which we are actually doing all this + md5sum $PREFIX.rootfs.tar.gz >chroot/installer.md5 + wc -c $PREFIX.rootfs.tar.gz >chroot/installer.size + + INFO_DESC="$(lsb_release -d -s)" + INFO_STAMP=$(date +20%y%m%d-%H:%M) + + echo "$INFO_DESC - $ARCH ($INFO_STAMP)" >chroot/media-info + + # make sure update-initramfs feels cosy and warm in the environment + lb chroot_proc install "$@" + lb chroot_sysfs install "$@" + lb chroot_devpts install "$@" + + # re-create initrd to contain the installer.md5 file + Chroot chroot "env FLASH_KERNEL_SKIP=1 update-initramfs -k all -t -u -v" + + # create boot.img + Chroot chroot "abootimg --create /boot/installer-${KVERS}.img -f /boot/bootimg.cfg-$SUBARCH -r /boot/initrd.img-${KVERS} -k /boot/vmlinuz-${KVERS}" + + # clean up + lb chroot_devpts remove "$@" + lb chroot_sysfs remove "$@" + lb chroot_proc remove "$@" + + cp "chroot/boot/installer-${KVERS}.img" "$PREFIX.bootimg-$FLAVOUR" + ln -sf "$PREFIX.bootimg-$FLAVOUR" "$PREFIX.bootimg" + ;; - INFO_DESC="$(lsb_release -d -s)" - INFO_STAMP=$(date +20%y%m%d-%H:%M) + raspi2|raspi3) + # copy the kernel and initrd to a predictable directory for + # ubuntu-image consumption. In some cases, like in pi2/3 + # u-boot, the bootloader needs to contain the kernel and initrd, + # so during rootfs build we copy it over to a directory that + # ubuntu-image looks for and shoves into the bootloader + # partition. + UBOOT_BOOT="image/boot/uboot" - echo "$INFO_DESC - $ARCH ($INFO_STAMP)" >chroot/media-info + mkdir -p $UBOOT_BOOT - # make sure update-initramfs feels cosy and warm in the environment - lb chroot_proc install "$@" - lb chroot_sysfs install "$@" - lb chroot_devpts install "$@" - - # re-create initrd to contain the installer.md5 file - Chroot chroot "env FLASH_KERNEL_SKIP=1 update-initramfs -k all -t -u -v" - - # create boot.img - Chroot chroot "abootimg --create /boot/installer-${KVERS}.img -f /boot/bootimg.cfg-$SUBARCH -r /boot/initrd.img-${KVERS} -k /boot/vmlinuz-${KVERS}" - - # clean up - lb chroot_devpts remove "$@" - lb chroot_sysfs remove "$@" - lb chroot_proc remove "$@" - - cp "chroot/boot/installer-${KVERS}.img" "$PREFIX.bootimg-$FLAVOUR" - ln -sf "$PREFIX.bootimg-$FLAVOUR" "$PREFIX.bootimg" - -fi + cp $PREFIX.initrd $UBOOT_BOOT/initrd.img || true + cp $PREFIX.kernel $UBOOT_BOOT/vmlinuz || true + ;; +esac if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then sourceslist="chroot/etc/apt/sources.list" diff --git a/live-build/auto/config b/live-build/auto/config index ba554125..fe82ccad 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -33,6 +33,7 @@ fi mkdir -p config cp -af /usr/share/livecd-rootfs/live-build/functions config/functions +cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse mkdir -p config/package-lists @@ -175,7 +176,7 @@ case $IMAGEFORMAT in MODEL=dragonboard ;; armhf+raspi2) MODEL=pi2 ;; - armhf+raspi3) + armhf+raspi3|arm64+raspi3) MODEL=pi3 ;; armhf+cm3) MODEL=cm3 ;; @@ -184,20 +185,39 @@ case $IMAGEFORMAT in exit 1 ;; esac - case $MODEL in - pc-amd64|pc-i386) - UBUNTU_IMAGE_ARGS="--image-size 3700M" ;; - *) - UBUNTU_IMAGE_ARGS="" ;; - esac - echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common - echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common - # Store model assertion in top dir to get it picked up later as a build artifact - env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 model="$MODEL" brand-id=canonical > "$PREFIX".model-assertion - echo "Configured ubuntu-image for the following model assertion:" - cat "$PREFIX".model-assertion - echo "----------------------------------------------------------" + if [ $PROJECT = "ubuntu-core" ]; then + # snap-based core images + + case $MODEL in + pc-amd64|pc-i386) + UBUNTU_IMAGE_ARGS="--image-size 3700M" ;; + *) + UBUNTU_IMAGE_ARGS="" ;; + esac + + echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common + echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common + # Store model assertion in top dir to get it picked up later as a build artifact + env SNAPPY_STORE_NO_CDN=1 snap known --remote model series=16 model="$MODEL" brand-id=canonical > "$PREFIX".model-assertion + echo "Configured ubuntu-image for the following model assertion:" + cat "$PREFIX".model-assertion + echo "----------------------------------------------------------" + else + # classic images + + UBUNTU_IMAGE_ARGS="" + UBUNTU_IMAGE_ARGS="$UBUNTU_IMAGE_ARGS${PROPOSED:+ --with-proposed}" + UBUNTU_IMAGE_ARGS="$UBUNTU_IMAGE_ARGS${EXTRA_PPAS:+ --extra-ppas \"$EXTRA_PPAS\"}" + + git clone git://git.launchpad.net/~canonical-foundations/snap-$MODEL/+git/github-mirror -b classic config/$PREFIX-gadget + + echo "IMAGEFORMAT=$IMAGEFORMAT" >> config/common + echo "SUITE=$SUITE" >> config/common + echo "UBUNTU_IMAGE_ARGS=\"$UBUNTU_IMAGE_ARGS\"" >> config/common + + echo "Configured ubuntu-image for the following gadget model: $MODEL" + fi # Fake finished configuration for lb build mkdir -p .build touch .build/config @@ -751,7 +771,7 @@ case $PROJECT in esac case $ARCH in - armel|armhf) + armel|armhf|arm64) KERNEL_FLAVOURS="${SUBARCH:-$KERNEL_FLAVOURS}" case $SUBARCH in dove) @@ -789,6 +809,13 @@ case $ARCH in add_package install linux-firmware-raspi2 u-boot-rpi flash-kernel u-boot-tools BINARY_REMOVE_LINUX=false ;; + raspi3) + COMPONENTS='main restricted universe multiverse' + KERNEL_FLAVOURS=raspi2 + add_package install raspi3-firmware u-boot-rpi flash-kernel u-boot-tools + BINARY_REMOVE_LINUX=false + ;; + esac ;; esac @@ -854,7 +881,7 @@ echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/binary case $ARCH+$SUBARCH in - armhf+raspi2) + armhf+raspi2|armhf+raspi3|arm64+raspi3) cat > config/hooks/01-firmware-directory.chroot_early < ${target_file} + echo "create_manifest call to dpkg-query finished." + ./config/snap-seed-parse "${chroot_root}" "${target_file}" + echo "create_manifest call to snap_seed_parse finished." + echo "create_manifest finished" +} + make_ext4_partition() { device="$1" label=${fs_label:+-L "${fs_label}"} diff --git a/live-build/snap-seed-parse.py b/live-build/snap-seed-parse.py new file mode 100755 index 00000000..d2c6ef28 --- /dev/null +++ b/live-build/snap-seed-parse.py @@ -0,0 +1,68 @@ +#!/usr/bin/python3 + +""" +Usage: snap-seed-parse [${chroot_dir}] + +This script looks for a seed.yaml path in the given root directory, parsing +it and appending the parsed lines to the given output file. + +The $chroot_dir argument is optional and will default to the empty string. +""" + +import argparse +import os.path +import re +import yaml + + +def log(msg): + print("snap-seed-parse: {}".format(msg)) + + +log("Parsing seed.yaml") + +parser = argparse.ArgumentParser() +parser.add_argument('chroot', nargs='?', default='', + help='root dir for the chroot from which to generate the ' + 'manifest') +parser.add_argument('file', help='Output manifest to this file') + +ARGS = parser.parse_args() +CHROOT_ROOT = ARGS.chroot +FNAME = ARGS.file + +# Trim any trailing slashes for correct appending +log("CHROOT_ROOT: {}".format(CHROOT_ROOT)) +if len(CHROOT_ROOT) > 0 and CHROOT_ROOT[-1] == '/': + CHROOT_ROOT = CHROOT_ROOT[:-1] + +# This is where we expect to find the seed.yaml file +YAML_PATH = CHROOT_ROOT + '/var/lib/snapd/seed/seed.yaml' + +# Snaps are prepended with this string in the manifest +LINE_PREFIX = 'snap:' + +log("yaml path: {}".format(YAML_PATH)) +if not os.path.isfile(YAML_PATH): + log("WARNING: yaml path not found; no seeded snaps found.") + exit(0) +else: + log("yaml path found.") + +with open(YAML_PATH, 'r') as fh: + yaml_lines = yaml.safe_load(fh)['snaps'] + +log('Writing manifest to {}'.format(FNAME)) + +with open(FNAME, 'a+') as fh: + for item in yaml_lines: + filestring = item['file'] + # Pull the revision number off the file name + revision = filestring[filestring.rindex('_')+1:] + revision = re.sub(r'[^0-9]', '', revision) + fh.write("{}{}\t{}\t{}\n".format(LINE_PREFIX, + item['name'], + item['channel'], + revision, + )) +log('Manifest output finished.') diff --git a/live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary b/live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary index a4d77bd7..6ba4fe9c 100644 --- a/live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary +++ b/live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary @@ -26,4 +26,4 @@ mkdir -p $rootfs_dir/lib/modules teardown_mountpoint $rootfs_dir -dpkg-query --admindir=$rootfs_dir/var/lib/dpkg -W > $rootfs_dir.manifest +create_manifest "${rootfs_dir}" "${rootfs_dir}.manifest" diff --git a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary index 7839c49e..2dff6d22 100755 --- a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary +++ b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary @@ -97,7 +97,7 @@ install_grub() { # grub-efi packages that otherwise would not make it into the base # manifest. filesystem.packages is moved into place via symlinking to # livecd.ubuntu-cpc.manifest by live-build/auto/build after lb_binary runs - dpkg-query --show --admindir="mountpoint/var/lib/dpkg" > "binary/boot/filesystem.packages" + create_manifest "mountpoint" "binary/boot/filesystem.packages" chroot mountpoint grub-install "${loop_device}" \ --boot-directory=/boot \ diff --git a/live-build/ubuntu-server/hooks/030-root-squashfs.binary b/live-build/ubuntu-server/hooks/030-root-squashfs.binary index ad34e76a..41c15001 100755 --- a/live-build/ubuntu-server/hooks/030-root-squashfs.binary +++ b/live-build/ubuntu-server/hooks/030-root-squashfs.binary @@ -28,7 +28,7 @@ cp -a chroot/* binary/boot/squashfs.dir squashfs_f="${PWD}/livecd.${PROJECT}.squashfs" squashfs_f_manifest="${squashfs_f}.manifest" -dpkg-query --admindir=binary/boot/squashfs.dir/var/lib/dpkg -W > ${squashfs_f_manifest} +create_manifest "binary/boot/squashfs.dir" "${squashfs_f_manifest}" (cd "binary/boot/squashfs.dir/" && mksquashfs . ${squashfs_f} \