mirror of
https://git.launchpad.net/livecd-rootfs
synced 2025-03-06 08:51:10 +00:00
auto/config: Rewrite add_task to use a Python script that cribs the logic from lp:ubuntu-archive-publishing's generate_extra_overrides.py. This means we can avoid some dubious hacks around seeding snaps and no longer depend on the Task headers in the archive.
This commit is contained in:
Michael Hudson-Doyle
parent
db029cd387
commit
a65c181596
9
debian/changelog
vendored
9
debian/changelog
vendored
@ -1,3 +1,12 @@
|
||||
livecd-rootfs (2.837) UNRELEASED; urgency=medium
|
||||
|
||||
* auto/config: Rewrite add_task to use a Python script that cribs the logic
|
||||
from lp:ubuntu-archive-publishing's generate_extra_overrides.py. This
|
||||
means we can avoid some dubious hacks around seeding snaps and no longer
|
||||
depend on the Task headers in the archive. (LP: #2019265)
|
||||
|
||||
-- Michael Hudson-Doyle <michael.hudson@ubuntu.com> Fri, 23 Sep 2022 10:58:29 +1200
|
||||
|
||||
livecd-rootfs (2.836) mantic; urgency=medium
|
||||
|
||||
* canary: include cryptsetup in the live layer
|
||||
|
||||
@ -36,6 +36,7 @@ mkdir -p config
|
||||
cp -af /usr/share/livecd-rootfs/live-build/functions config/functions
|
||||
cp -af /usr/share/livecd-rootfs/live-build/lb_*_layered config/
|
||||
cp -af /usr/share/livecd-rootfs/live-build/snap-seed-parse.py config/snap-seed-parse
|
||||
cp -af /usr/share/livecd-rootfs/live-build/expand-task config/expand-task
|
||||
cp -af /usr/share/livecd-rootfs/live-build/squashfs-exclude-files config/
|
||||
|
||||
mkdir -p config/package-lists
|
||||
@ -106,55 +107,36 @@ add_task ()
|
||||
{
|
||||
local pass="$1"
|
||||
shift
|
||||
local task
|
||||
local snap_list_file
|
||||
local snap_list_files
|
||||
local curseed
|
||||
local file pkg_file snap_file task
|
||||
|
||||
_check_immutable_passes_to_layers
|
||||
_register_pass "$pass"
|
||||
|
||||
# The removal of direct task installation support from live-build
|
||||
# poses some problems. If the chroot has multiarch configured - for
|
||||
# example, if we're building for amd64 - then dumpavail will show
|
||||
# foreign-architecture packages which will have their own Task
|
||||
# lines, but which we don't want to install. (Compare
|
||||
# PackageContainerInterface::FromTask in apt, which restricts task
|
||||
# expansion to the native architecture.) We therefore restrict our
|
||||
# search to stanzas with Architecture: $ARCH or all.
|
||||
#
|
||||
# However, even this may not be accurate enough. At the moment I
|
||||
# have no idea what happens if an Architecture: all package has
|
||||
# different Task fields on different architectures. This is
|
||||
# probably a lurking timebomb that we need to fix. In the meantime,
|
||||
# the Architecture restriction at least saves us from abject
|
||||
# failure.
|
||||
#
|
||||
# We want as well to grab the snap list for each PASS. Resolve for all
|
||||
# given task, and deduplicate them to generate snaps for the PASS.
|
||||
if [ ! -e config/germinate-output/structure ]; then
|
||||
echo "add_task too soon" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pkg_file="config/package-lists/livecd-rootfs.list.chroot_$pass"
|
||||
|
||||
if [ $PASSES_TO_LAYERS = "true" ]; then
|
||||
snap_file="config/package-lists/livecd-rootfs.snaplist.chroot_$pass.full"
|
||||
else
|
||||
snap_file="config/seeded-snaps"
|
||||
fi
|
||||
|
||||
for task; do
|
||||
# We need a ridiculous number of backslashes to protect
|
||||
# parentheses from eval.
|
||||
echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass"
|
||||
./config/expand-task config/germinate-output $FLAVOUR $task packages >> "$pkg_file"
|
||||
./config/expand-task config/germinate-output $FLAVOUR $task snaps >> "$snap_file"
|
||||
done
|
||||
|
||||
curseed=$(seed_from_task ${task})
|
||||
if [ -z "${curseed}" ]; then
|
||||
echo "W: No seed matching task ${task}"
|
||||
continue
|
||||
fi
|
||||
snap_list_file="config/package-lists/seed.${curseed}.snaplist.full"
|
||||
snap_from_seed "${curseed}" $snap_list_file
|
||||
if [ -e "$snap_list_file" ]; then
|
||||
snap_list_files="${snap_list_files} $snap_list_file"
|
||||
for file in $pkg_file $snap_file; do
|
||||
if [ -s $file ]; then
|
||||
sort -u -o $file $file
|
||||
else
|
||||
rm -f $file
|
||||
fi
|
||||
done
|
||||
# The snap list is one line, and could be duplicated between seeds via inheritance.
|
||||
# Uniquely sort them and store them back in one line.
|
||||
if [ -n "${snap_list_files}" ]; then
|
||||
cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full"
|
||||
rm ${snap_list_files}
|
||||
fi
|
||||
}
|
||||
|
||||
add_package ()
|
||||
@ -639,34 +621,34 @@ fi
|
||||
mkdir -p config/germinate-output
|
||||
case $PROJECT in
|
||||
kubuntu*)
|
||||
SEED=kubuntu.$SUITE
|
||||
FLAVOUR=kubuntu
|
||||
;;
|
||||
xubuntu*)
|
||||
SEED=xubuntu.$SUITE
|
||||
FLAVOUR=xubuntu
|
||||
;;
|
||||
ubuntu-mate*)
|
||||
SEED=ubuntu-mate.$SUITE
|
||||
FLAVOUR=ubuntu-mate
|
||||
;;
|
||||
ubuntu-unity*)
|
||||
SEED=ubuntu-unity.$SUITE
|
||||
FLAVOUR=ubuntu-unity
|
||||
;;
|
||||
lubuntu*)
|
||||
SEED=lubuntu.$SUITE
|
||||
FLAVOUR=lubuntu
|
||||
;;
|
||||
ubuntu-budgie*)
|
||||
SEED=ubuntu-budgie.$SUITE
|
||||
FLAVOUR=ubuntu-budgie
|
||||
;;
|
||||
ubuntukylin*)
|
||||
SEED=ubuntukylin.$SUITE
|
||||
FLAVOUR=ubuntukylin
|
||||
;;
|
||||
ubuntustudio*)
|
||||
SEED=ubuntustudio.$SUITE
|
||||
FLAVOUR=ubuntustudio
|
||||
;;
|
||||
ubuntucinnamon*)
|
||||
SEED=ubuntucinnamon.$SUITE
|
||||
;;
|
||||
*)
|
||||
SEED=ubuntu.$SUITE
|
||||
FLAVOUR=ubuntu
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -694,7 +676,7 @@ if ! [ -e config/germinate-output/structure ]; then
|
||||
GERMINATE_ARG="-c $(echo $COMPONENTS | sed -e's/ \+/,/g')"
|
||||
fi
|
||||
(cd config/germinate-output && germinate --no-rdepends --no-installer \
|
||||
-S $SEEDMIRROR -m $MIRROR -d $SUITE -s $SEED \
|
||||
-S $SEEDMIRROR -m $MIRROR -d $SUITE -s $FLAVOUR.$SUITE \
|
||||
$GERMINATE_ARG -a $ARCH)
|
||||
fi
|
||||
|
||||
@ -957,8 +939,6 @@ case $PROJECT in
|
||||
add_task ubuntu-server-minimal server-minimal
|
||||
add_package ubuntu-server-minimal lxd-installer
|
||||
add_task ubuntu-server-minimal.ubuntu-server minimal standard server
|
||||
# add_task really should do this itself but for now...
|
||||
snap_from_seed server config/package-lists/livecd-rootfs.snaplist.chroot_ubuntu-server-minimal.ubuntu-server.full
|
||||
add_package ubuntu-server-minimal.ubuntu-server cloud-init
|
||||
|
||||
add_package ubuntu-server-minimal.ubuntu-server.installer linux-firmware casper openssh-server
|
||||
@ -1188,9 +1168,6 @@ case $PROJECT:${SUBPROJECT:-} in
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$PASSES_TO_LAYERS" != "true" ] && [ -n "${BASE_SEED}" ]; then
|
||||
snap_from_seed "${BASE_SEED}" config/seeded-snaps
|
||||
fi
|
||||
if [ "$PROJECT:${SUBPROJECT:-}" = ubuntu-cpc:minimized ]; then
|
||||
# We install a lxc script that installs the snap when invoked. We don't
|
||||
# want any other snaps to come in without due consideration, so fail the
|
||||
|
||||
84
live-build/expand-task
Executable file
84
live-build/expand-task
Executable file
@ -0,0 +1,84 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('output_dir')
|
||||
p.add_argument('flavour')
|
||||
p.add_argument('task')
|
||||
p.add_argument('what', choices=['packages', 'snaps'])
|
||||
args = p.parse_args()
|
||||
|
||||
if args.what == 'snaps':
|
||||
ext = '.snaps'
|
||||
else:
|
||||
ext = ''
|
||||
|
||||
|
||||
# begin copy/paste from ubuntu-archive-publishing's generate_extra_overrides.
|
||||
def parseTaskHeaders(seedtext):
|
||||
"""Parse a seed for Task headers.
|
||||
|
||||
seedtext is a file-like object. Return a dictionary of Task headers,
|
||||
with keys canonicalised to lower-case.
|
||||
"""
|
||||
task_headers = {}
|
||||
task_header_regex = re.compile(
|
||||
r"task-(.*?):(.*)", flags=re.IGNORECASE)
|
||||
for line in seedtext:
|
||||
match = task_header_regex.match(line)
|
||||
if match is not None:
|
||||
key, value = match.groups()
|
||||
task_headers[key.lower()] = value.strip()
|
||||
return task_headers
|
||||
|
||||
|
||||
def getTaskName(task_headers, flavour, seedname, primary_flavour):
|
||||
"""Work out the name of the Task to be generated from this seed.
|
||||
|
||||
If there is a Task-Name header, it wins; otherwise, seeds with a
|
||||
Task-Per-Derivative header are honoured for all flavours and put in
|
||||
an appropriate namespace, while other seeds are only honoured for
|
||||
the first flavour and have archive-global names.
|
||||
"""
|
||||
if "name" in task_headers:
|
||||
return task_headers["name"]
|
||||
elif "per-derivative" in task_headers:
|
||||
return "%s-%s" % (flavour, seedname)
|
||||
elif primary_flavour:
|
||||
return seedname
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def getTaskSeeds(task_headers, seedname):
|
||||
"""Return the list of seeds used to generate a task from this seed.
|
||||
|
||||
The list of packages in this task comes from this seed plus any
|
||||
other seeds listed in a Task-Seeds header.
|
||||
"""
|
||||
scan_seeds = set([seedname])
|
||||
if "seeds" in task_headers:
|
||||
scan_seeds.update(task_headers["seeds"].split())
|
||||
return sorted(scan_seeds)
|
||||
# end copy/paste from ubuntu-archive-publishing's generate_extra_overrides.
|
||||
|
||||
|
||||
for seedtext in glob.glob(f'{args.output_dir}/*.seedtext'):
|
||||
hs = parseTaskHeaders(open(seedtext))
|
||||
if not hs:
|
||||
continue
|
||||
seedname = os.path.splitext(os.path.basename(seedtext))[0]
|
||||
tn = getTaskName(hs, args.flavour, seedname, args.flavour == 'ubuntu')
|
||||
if tn != args.task:
|
||||
continue
|
||||
for seed in getTaskSeeds(hs, seedname):
|
||||
for line in open(f'{args.output_dir}/{seed}{ext}'):
|
||||
if re.match('^[a-z0-9]', line):
|
||||
print(line.split()[0])
|
||||
break
|
||||
else:
|
||||
raise Exception("did not find task %r" % (args.task,))
|
||||
@ -766,56 +766,6 @@ snap_validate_seed() {
|
||||
fi
|
||||
}
|
||||
|
||||
snap_from_seed() {
|
||||
local base_seed=$1
|
||||
local out=$2
|
||||
local all_snaps
|
||||
local seeds_expanded
|
||||
|
||||
seeds_expanded=$(inheritance ${base_seed})
|
||||
for seed in ${seeds_expanded}; do
|
||||
echo "snap: considering ${seed}"
|
||||
file=config/germinate-output/${seed}.snaps
|
||||
[ -e "${file}" ] || continue
|
||||
# extract the first column (snap package name) from germinate's output
|
||||
# translate the human-readable "foo (classic)" into a
|
||||
# more machine readable "foo/classic"
|
||||
seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}")
|
||||
for snap in ${seed_snaps}; do
|
||||
echo "snap: found ${snap}"
|
||||
all_snaps="${all_snaps:+${all_snaps} }${snap}"
|
||||
done
|
||||
done
|
||||
if [ -n "${all_snaps}" ]; then
|
||||
echo "${all_snaps}" > $out
|
||||
fi
|
||||
}
|
||||
|
||||
seed_from_task ()
|
||||
{
|
||||
# Retrieve the name of the seed from a task name
|
||||
local task=$1
|
||||
local seed
|
||||
local seedfile
|
||||
local seedfiles
|
||||
|
||||
seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)"
|
||||
if [ -n "$seedfile" ]; then
|
||||
basename $seedfile .seedtext
|
||||
return
|
||||
fi
|
||||
|
||||
seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)"
|
||||
if [ -n "$seedfiles" ]; then
|
||||
for seed in $(echo $seedfiles | xargs basename -s .seedtext); do
|
||||
if [ ${PROJECT}-${seed} = $task ]; then
|
||||
echo ${seed}
|
||||
return
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
list_packages_from_seed () {
|
||||
# Store all packages for a given seed, including its seed dependency
|
||||
# $1: Name of the seed to expand to a package list
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user