From a65c1815965cd72347433a29b5a4fecc87161558 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 23 Sep 2022 10:58:43 +1200 Subject: [PATCH] auto/config: Rewrite add_task to use a Python script that cribs the logic from lp:ubuntu-archive-publishing's generate_extra_overrides.py. This means we can avoid some dubious hacks around seeding snaps and no longer depend on the Task headers in the archive. --- debian/changelog | 9 +++++ live-build/auto/config | 89 ++++++++++++++++-------------------------- live-build/expand-task | 84 +++++++++++++++++++++++++++++++++++++++ live-build/functions | 50 ------------------------ 4 files changed, 126 insertions(+), 106 deletions(-) create mode 100755 live-build/expand-task diff --git a/debian/changelog b/debian/changelog index 0906c7d6..52429984 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +livecd-rootfs (2.837) UNRELEASED; urgency=medium + + * auto/config: Rewrite add_task to use a Python script that cribs the logic + from lp:ubuntu-archive-publishing's generate_extra_overrides.py. This + means we can avoid some dubious hacks around seeding snaps and no longer + depend on the Task headers in the archive. (LP: #2019265) + + -- Michael Hudson-Doyle Fri, 23 Sep 2022 10:58:29 +1200 + livecd-rootfs (2.836) mantic; urgency=medium * canary: include cryptsetup in the live layer diff --git a/live-build/auto/config b/live-build/auto/config index ff0abd60..801e6aaf 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -36,6 +36,7 @@ mkdir -p config cp -af /usr/share/livecd-rootfs/live-build/functions config/functions cp -af /usr/share/livecd-rootfs/live-build/lb_*_layered config/ cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse +cp -af /usr/share/livecd-rootfs/live-build/expand-task config/expand-task cp -af /usr/share/livecd-rootfs/live-build/squashfs-exclude-files config/ mkdir -p config/package-lists @@ -106,55 +107,36 @@ add_task () { local pass="$1" shift - local task - local snap_list_file - local snap_list_files - local curseed + local file pkg_file snap_file task _check_immutable_passes_to_layers _register_pass "$pass" - # The removal of direct task installation support from live-build - # poses some problems. If the chroot has multiarch configured - for - # example, if we're building for amd64 - then dumpavail will show - # foreign-architecture packages which will have their own Task - # lines, but which we don't want to install. (Compare - # PackageContainerInterface::FromTask in apt, which restricts task - # expansion to the native architecture.) We therefore restrict our - # search to stanzas with Architecture: $ARCH or all. - # - # However, even this may not be accurate enough. At the moment I - # have no idea what happens if an Architecture: all package has - # different Task fields on different architectures. This is - # probably a lurking timebomb that we need to fix. In the meantime, - # the Architecture restriction at least saves us from abject - # failure. - # - # We want as well to grab the snap list for each PASS. Resolve for all - # given task, and deduplicate them to generate snaps for the PASS. + if [ ! -e config/germinate-output/structure ]; then + echo "add_task too soon" >&2 + exit 1 + fi + + pkg_file="config/package-lists/livecd-rootfs.list.chroot_$pass" + + if [ $PASSES_TO_LAYERS = "true" ]; then + snap_file="config/package-lists/livecd-rootfs.snaplist.chroot_$pass.full" + else + snap_file="config/seeded-snaps" + fi for task; do - # We need a ridiculous number of backslashes to protect - # parentheses from eval. - echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass" - - curseed=$(seed_from_task ${task}) - if [ -z "${curseed}" ]; then - echo "W: No seed matching task ${task}" - continue - fi - snap_list_file="config/package-lists/seed.${curseed}.snaplist.full" - snap_from_seed "${curseed}" $snap_list_file - if [ -e "$snap_list_file" ]; then - snap_list_files="${snap_list_files} $snap_list_file" + ./config/expand-task config/germinate-output $FLAVOUR $task packages >> "$pkg_file" + ./config/expand-task config/germinate-output $FLAVOUR $task snaps >> "$snap_file" + done + + for file in $pkg_file $snap_file; do + if [ -s $file ]; then + sort -u -o $file $file + else + rm -f $file fi done - # The snap list is one line, and could be duplicated between seeds via inheritance. - # Uniquely sort them and store them back in one line. - if [ -n "${snap_list_files}" ]; then - cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full" - rm ${snap_list_files} - fi } add_package () @@ -639,34 +621,34 @@ fi mkdir -p config/germinate-output case $PROJECT in kubuntu*) - SEED=kubuntu.$SUITE + FLAVOUR=kubuntu ;; xubuntu*) - SEED=xubuntu.$SUITE + FLAVOUR=xubuntu ;; ubuntu-mate*) - SEED=ubuntu-mate.$SUITE + FLAVOUR=ubuntu-mate ;; ubuntu-unity*) - SEED=ubuntu-unity.$SUITE + FLAVOUR=ubuntu-unity ;; lubuntu*) - SEED=lubuntu.$SUITE + FLAVOUR=lubuntu ;; ubuntu-budgie*) - SEED=ubuntu-budgie.$SUITE + FLAVOUR=ubuntu-budgie ;; ubuntukylin*) - SEED=ubuntukylin.$SUITE + FLAVOUR=ubuntukylin ;; ubuntustudio*) - SEED=ubuntustudio.$SUITE + FLAVOUR=ubuntustudio ;; ubuntucinnamon*) SEED=ubuntucinnamon.$SUITE ;; *) - SEED=ubuntu.$SUITE + FLAVOUR=ubuntu ;; esac @@ -694,7 +676,7 @@ if ! [ -e config/germinate-output/structure ]; then GERMINATE_ARG="-c $(echo $COMPONENTS | sed -e's/ \+/,/g')" fi (cd config/germinate-output && germinate --no-rdepends --no-installer \ - -S $SEEDMIRROR -m $MIRROR -d $SUITE -s $SEED \ + -S $SEEDMIRROR -m $MIRROR -d $SUITE -s $FLAVOUR.$SUITE \ $GERMINATE_ARG -a $ARCH) fi @@ -957,8 +939,6 @@ case $PROJECT in add_task ubuntu-server-minimal server-minimal add_package ubuntu-server-minimal lxd-installer add_task ubuntu-server-minimal.ubuntu-server minimal standard server - # add_task really should do this itself but for now... - snap_from_seed server config/package-lists/livecd-rootfs.snaplist.chroot_ubuntu-server-minimal.ubuntu-server.full add_package ubuntu-server-minimal.ubuntu-server cloud-init add_package ubuntu-server-minimal.ubuntu-server.installer linux-firmware casper openssh-server @@ -1188,9 +1168,6 @@ case $PROJECT:${SUBPROJECT:-} in ;; esac -if [ "$PASSES_TO_LAYERS" != "true" ] && [ -n "${BASE_SEED}" ]; then - snap_from_seed "${BASE_SEED}" config/seeded-snaps -fi if [ "$PROJECT:${SUBPROJECT:-}" = ubuntu-cpc:minimized ]; then # We install a lxc script that installs the snap when invoked. We don't # want any other snaps to come in without due consideration, so fail the diff --git a/live-build/expand-task b/live-build/expand-task new file mode 100755 index 00000000..fb309e7b --- /dev/null +++ b/live-build/expand-task @@ -0,0 +1,84 @@ +#!/usr/bin/python3 + +import argparse +import glob +import os +import re + +p = argparse.ArgumentParser() +p.add_argument('output_dir') +p.add_argument('flavour') +p.add_argument('task') +p.add_argument('what', choices=['packages', 'snaps']) +args = p.parse_args() + +if args.what == 'snaps': + ext = '.snaps' +else: + ext = '' + + +# begin copy/paste from ubuntu-archive-publishing's generate_extra_overrides. +def parseTaskHeaders(seedtext): + """Parse a seed for Task headers. + + seedtext is a file-like object. Return a dictionary of Task headers, + with keys canonicalised to lower-case. + """ + task_headers = {} + task_header_regex = re.compile( + r"task-(.*?):(.*)", flags=re.IGNORECASE) + for line in seedtext: + match = task_header_regex.match(line) + if match is not None: + key, value = match.groups() + task_headers[key.lower()] = value.strip() + return task_headers + + +def getTaskName(task_headers, flavour, seedname, primary_flavour): + """Work out the name of the Task to be generated from this seed. + + If there is a Task-Name header, it wins; otherwise, seeds with a + Task-Per-Derivative header are honoured for all flavours and put in + an appropriate namespace, while other seeds are only honoured for + the first flavour and have archive-global names. + """ + if "name" in task_headers: + return task_headers["name"] + elif "per-derivative" in task_headers: + return "%s-%s" % (flavour, seedname) + elif primary_flavour: + return seedname + else: + return None + + +def getTaskSeeds(task_headers, seedname): + """Return the list of seeds used to generate a task from this seed. + + The list of packages in this task comes from this seed plus any + other seeds listed in a Task-Seeds header. + """ + scan_seeds = set([seedname]) + if "seeds" in task_headers: + scan_seeds.update(task_headers["seeds"].split()) + return sorted(scan_seeds) +# end copy/paste from ubuntu-archive-publishing's generate_extra_overrides. + + +for seedtext in glob.glob(f'{args.output_dir}/*.seedtext'): + hs = parseTaskHeaders(open(seedtext)) + if not hs: + continue + seedname = os.path.splitext(os.path.basename(seedtext))[0] + tn = getTaskName(hs, args.flavour, seedname, args.flavour == 'ubuntu') + if tn != args.task: + continue + for seed in getTaskSeeds(hs, seedname): + for line in open(f'{args.output_dir}/{seed}{ext}'): + if re.match('^[a-z0-9]', line): + print(line.split()[0]) + break +else: + raise Exception("did not find task %r" % (args.task,)) diff --git a/live-build/functions b/live-build/functions index 3a25e995..208b8f28 100644 --- a/live-build/functions +++ b/live-build/functions @@ -766,56 +766,6 @@ snap_validate_seed() { fi } -snap_from_seed() { - local base_seed=$1 - local out=$2 - local all_snaps - local seeds_expanded - - seeds_expanded=$(inheritance ${base_seed}) - for seed in ${seeds_expanded}; do - echo "snap: considering ${seed}" - file=config/germinate-output/${seed}.snaps - [ -e "${file}" ] || continue - # extract the first column (snap package name) from germinate's output - # translate the human-readable "foo (classic)" into a - # more machine readable "foo/classic" - seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}") - for snap in ${seed_snaps}; do - echo "snap: found ${snap}" - all_snaps="${all_snaps:+${all_snaps} }${snap}" - done - done - if [ -n "${all_snaps}" ]; then - echo "${all_snaps}" > $out - fi -} - -seed_from_task () -{ - # Retrieve the name of the seed from a task name - local task=$1 - local seed - local seedfile - local seedfiles - - seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)" - if [ -n "$seedfile" ]; then - basename $seedfile .seedtext - return - fi - - seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)" - if [ -n "$seedfiles" ]; then - for seed in $(echo $seedfiles | xargs basename -s .seedtext); do - if [ ${PROJECT}-${seed} = $task ]; then - echo ${seed} - return - fi - done - fi -} - list_packages_from_seed () { # Store all packages for a given seed, including its seed dependency # $1: Name of the seed to expand to a package list