diff --git a/debian/changelog b/debian/changelog index 8baeea3e..9fb010f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +livecd-rootfs (2.542.4) UNRELEASED; urgency=medium + + * ubuntu-cpc: parallel builds + - Replace "snap download" with tool that uses snap store's coherence feature + - Inject a proxy into the build providing a snapshot view of the package repo. + - Use series files with dependency handling to generate hook symlinks dynamically + + -- Robert C Jennings Wed, 24 Apr 2019 21:17:17 -0500 + livecd-rootfs (2.542.3) cosmic; urgency=medium * Remove device nodes later for ubuntu-base:minimized (i.e. docker) builds. diff --git a/debian/install b/debian/install index 4eb70070..339c4f2c 100644 --- a/debian/install +++ b/debian/install @@ -1,3 +1,6 @@ live-build usr/share/livecd-rootfs get-ppa-fingerprint usr/share/livecd-rootfs minimize-manual usr/share/livecd-rootfs +magic-proxy usr/share/livecd-rootfs +lp-in-release usr/share/livecd-rootfs +snap-tool usr/share/livecd-rootfs diff --git a/live-build/auto/build b/live-build/auto/build index eec51250..1d36ed06 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -17,6 +17,30 @@ fi . config/functions +if [ -n "$REPO_SNAPSHOT_STAMP" ]; then + if [ "`whoami`" != "root" ]; then + echo "Magic repo snapshots only work when running as root." >&2 + exit 1 + fi + + apt-get -qyy install iptables + + # Redirect all outgoing traffic to port 80 to proxy instead. + iptables -t nat -A OUTPUT -p tcp --dport 80 -m owner ! --uid-owner daemon \ + -j REDIRECT --to 8080 + + # Run proxy as "daemon" to avoid infinite loop. + /usr/share/livecd-rootfs/magic-proxy \ + --address="127.0.0.1" \ + --port=8080 \ + --run-as=daemon \ + --cutoff-time="$REPO_SNAPSHOT_STAMP" \ + --log-file=/build/livecd.magic-proxy.log \ + --pid-file=config/magic-proxy.pid \ + --background \ + --setsid +fi + # Link output files somewhere BuildLiveCD will be able to find them. PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}" @@ -974,3 +998,12 @@ case $PROJECT in fi ;; esac + +if [ -f "config/magic-proxy.pid" ]; then + kill -TERM $(cat config/magic-proxy.pid) + rm -f config/magic-proxy.pid + + # Remove previously-inserted iptables rule. + iptables -t nat -D OUTPUT -p tcp --dport 80 -m owner ! --uid-owner daemon \ + -j REDIRECT --to 8080 +fi diff --git a/live-build/auto/config b/live-build/auto/config index bffbb7f7..7d39ad9f 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -1000,6 +1000,19 @@ EOF ubuntu-touch:*|ubuntu-touch-custom:*|ubuntu-core:system-image|ubuntu-desktop-next:system-image|ubuntu-cpc:*|ubuntu-server:live) cp -af /usr/share/livecd-rootfs/live-build/${PROJECT}/* \ config/ + + if [ "$PROJECT" = "ubuntu-cpc" ]; then + case ${IMAGE_TARGETS:-} in + "") + config/hooks.d/make-hooks --hooks-dir config/hooks all + ;; + *) + config/hooks.d/make-hooks --hooks-dir config/hooks \ + "$IMAGE_TARGETS" + ;; + esac + fi + if [ "$IMAGEFORMAT" = none ]; then rm -f config/hooks/*.binary* fi diff --git a/live-build/functions b/live-build/functions index 2cbe3e02..2757d10c 100644 --- a/live-build/functions +++ b/live-build/functions @@ -387,11 +387,12 @@ _snap_preseed() { # Download the snap & assertion local snap_download_failed=0 - chroot $CHROOT_ROOT sh -c " + sh -c " set -x; - cd /var/lib/snapd/seed; - SNAPPY_STORE_NO_CDN=1 snap download \ - --channel=$CHANNEL \"$SNAP_NAME\"" || snap_download_failed=1 + cd \"$CHROOT_ROOT/var/lib/snapd/seed\"; + SNAPPY_STORE_NO_CDN=1 /usr/share/livecd-rootfs/snap-tool download \ + --cohort-key=\"${COHORT_KEY:-}\" \ + --channel=\"$CHANNEL\" \"$SNAP_NAME\"" || snap_download_failed=1 if [ $snap_download_failed = 1 ] ; then echo "If the channel ($CHANNEL) includes '*/ubuntu-##.##' track per " echo "Ubuntu policy (ex. stable/ubuntu-18.04) the publisher will need " diff --git a/live-build/ubuntu-cpc/README.cpc.md b/live-build/ubuntu-cpc/README.cpc.md new file mode 100644 index 00000000..9d61b2d5 --- /dev/null +++ b/live-build/ubuntu-cpc/README.cpc.md @@ -0,0 +1,68 @@ +# TL;DR + +In order to generate the hooks for a specific image target set, call the +`make-hooks` script, located in `hooks.d` as + + ./make-hooks --hooks-dir ../hooks + +where `image_set` is the name of a series file (e.g. "vagrant") without leading +path components. Do *not* check in the `hooks` folder, it is automatically +generated by `auto/config` during Live Build runs. + + +# Hook placement and ordering + +Scripts live in subfolders below the `hooks.d` folder. Currently the folders +`chroot` and `base` exist. The folder with the name `extra` is reserved for +private scripts, which are not included in the source of livecd-rootfs. The +scripts are not numbered, instead the order of their execution depends on the +order in which they are listed in a *series* file. + +Series files are placed in subfolders `hooks.d/base/series` or +`hooks.d/extra/series`. Each series file contains a list of scripts to be +executed. Empty lines and lines starting with a `#` are ignored. + +Series files in `extra/series` override files in `base/series` with the same +name. For example, if a series file `base/series/cloudA` exists and a series +file `extra/series/cloudA`, then the latter will be preferred. + +A series file in `extra/series` may also list scripts that are located in the +`chroot` and `base` folders. In addition, series files can *depend* on other +series files. For example, the series files for most custom images look similar +to this: + + depends disk-image + depends extra-settings + extra/cloudB.binary + +Where `disk-image` and `extra-settings` may list scripts and dependencies which +are to be processed before the script `extra/cloudB.binary` is called. + +ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate +from scripts ending in ".binary". Even if you arrange them interleaved in your +series files, the chroot scripts will be run before the binary scripts. + +# Image set selection for Live Build + +During a Live Build, enumerated symbolic links are generated based on the +contents of one or more series files. The series files are selected according +to the contents of the `IMAGE_TARGETS` environment variable. For example, in +order to trigger the build of `squashfs` and `vagrant`, list them in the +`IMAGE_TARGETS` variable as `squashfs,vagrant`. The separator can be a comma, +a semi-colon or whitespace. + +The generation of the symbolic links is triggered from the `auto/config` script, +from where the contents of the `IMAGE_TARGETS` environment variable are passed +on to the `make-hooks` script. + + +# Symlink generation + +Since Live Build itself does not know about series files, a traditional `hooks` +folder is generated using the `make-hooks` script. The script takes as arguments +the names of the series files to be processed. + +The script parses the series files and generates enumerated symbolic links for +all entries. Per default, these are placed into a directory named `hooks` next +to the `hooks.d` directory. This can be changed using the `--hooks-dir` +parameter. diff --git a/live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary b/live-build/ubuntu-cpc/hooks.d/base/create-root-dir.binary similarity index 100% rename from live-build/ubuntu-cpc/hooks/031-0-create-root-dir.binary rename to live-build/ubuntu-cpc/hooks.d/base/create-root-dir.binary diff --git a/live-build/ubuntu-cpc/hooks/034-disk-image-ppc64el.binary b/live-build/ubuntu-cpc/hooks.d/base/disk-image-ppc64el.binary similarity index 100% rename from live-build/ubuntu-cpc/hooks/034-disk-image-ppc64el.binary rename to live-build/ubuntu-cpc/hooks.d/base/disk-image-ppc64el.binary diff --git a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary b/live-build/ubuntu-cpc/hooks.d/base/disk-image-uefi.binary similarity index 100% rename from live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary rename to live-build/ubuntu-cpc/hooks.d/base/disk-image-uefi.binary diff --git a/live-build/ubuntu-cpc/hooks/032-disk-image.binary b/live-build/ubuntu-cpc/hooks.d/base/disk-image.binary similarity index 98% rename from live-build/ubuntu-cpc/hooks/032-disk-image.binary rename to live-build/ubuntu-cpc/hooks.d/base/disk-image.binary index f7516dde..19978ff0 100755 --- a/live-build/ubuntu-cpc/hooks/032-disk-image.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/disk-image.binary @@ -12,6 +12,8 @@ BOOTPART_END= BOOT_MOUNTPOINT= ROOTPART_START=1 +my_d=$(dirname $(readlink -f ${0})) + case $ARCH:$SUBARCH in ppc64el:*|powerpc:*) echo "POWER disk images are handled separately" @@ -97,7 +99,7 @@ case $ARCH:$SUBARCH in # not the best place for this, but neither flash-kernel nor # u-boot have provisions for installing u-boot via maintainer # script - config/hooks/raspi2/mkknlimg --dtok \ + ${my_d}/raspi2/mkknlimg --dtok \ mountpoint/usr/lib/u-boot/rpi_2/u-boot.bin \ mountpoint/boot/firmware/uboot.bin ;; diff --git a/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl b/live-build/ubuntu-cpc/hooks.d/base/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl similarity index 100% rename from live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl rename to live-build/ubuntu-cpc/hooks.d/base/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl diff --git a/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-vmdk.tmpl b/live-build/ubuntu-cpc/hooks.d/base/ovf/ubuntu-ova-v1-vmdk.tmpl similarity index 100% rename from live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-vmdk.tmpl rename to live-build/ubuntu-cpc/hooks.d/base/ovf/ubuntu-ova-v1-vmdk.tmpl diff --git a/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ovf-v1-img.tmpl b/live-build/ubuntu-cpc/hooks.d/base/ovf/ubuntu-ovf-v1-img.tmpl similarity index 100% rename from live-build/ubuntu-cpc/hooks/ovf/ubuntu-ovf-v1-img.tmpl rename to live-build/ubuntu-cpc/hooks.d/base/ovf/ubuntu-ovf-v1-img.tmpl diff --git a/live-build/ubuntu-cpc/hooks/040-qcow2-image.binary b/live-build/ubuntu-cpc/hooks.d/base/qcow2-image.binary similarity index 83% rename from live-build/ubuntu-cpc/hooks/040-qcow2-image.binary rename to live-build/ubuntu-cpc/hooks.d/base/qcow2-image.binary index b9c40a88..c827a2a6 100755 --- a/live-build/ubuntu-cpc/hooks/040-qcow2-image.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/qcow2-image.binary @@ -1,14 +1,5 @@ #!/bin/bash -ex -case $IMAGE_TARGETS in - ""|*qcow2*) - ;; - *) - echo "Skipping qcow2 image build" - exit 0 - ;; -esac - case $ARCH:$SUBARCH in # Not sure if any other cloud images use subarch for something that # should take qcow2 format, so only skipping this on raspi2 for now. diff --git a/live-build/ubuntu-cpc/hooks/raspi2/mkknlimg b/live-build/ubuntu-cpc/hooks.d/base/raspi2/mkknlimg similarity index 100% rename from live-build/ubuntu-cpc/hooks/raspi2/mkknlimg rename to live-build/ubuntu-cpc/hooks.d/base/raspi2/mkknlimg diff --git a/live-build/ubuntu-cpc/hooks/031-2-root-squashfs.binary b/live-build/ubuntu-cpc/hooks.d/base/root-squashfs.binary similarity index 81% rename from live-build/ubuntu-cpc/hooks/031-2-root-squashfs.binary rename to live-build/ubuntu-cpc/hooks.d/base/root-squashfs.binary index f4c2ac3d..c6e2273e 100755 --- a/live-build/ubuntu-cpc/hooks/031-2-root-squashfs.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/root-squashfs.binary @@ -3,15 +3,6 @@ # # Generate a squashfs root and manifest -case $IMAGE_TARGETS in - ""|*squashfs*) - ;; - *) - echo "Skipping squashfs build" - exit 0 - ;; -esac - if [ -n "$SUBARCH" ]; then echo "Skipping rootfs build for subarch flavor build" exit 0 diff --git a/live-build/ubuntu-cpc/hooks/031-1-root-xz.binary b/live-build/ubuntu-cpc/hooks.d/base/root-xz.binary similarity index 100% rename from live-build/ubuntu-cpc/hooks/031-1-root-xz.binary rename to live-build/ubuntu-cpc/hooks.d/base/root-xz.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/all b/live-build/ubuntu-cpc/hooks.d/base/series/all new file mode 120000 index 00000000..8681f8b8 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/all @@ -0,0 +1 @@ +base \ No newline at end of file diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/base b/live-build/ubuntu-cpc/hooks.d/base/series/base new file mode 100644 index 00000000..43262570 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/base @@ -0,0 +1,7 @@ +depends root-dir +depends tarball +depends squashfs +depends disk-image +depends qcow2 +depends vmdk +depends vagrant diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/disk-image b/live-build/ubuntu-cpc/hooks.d/base/series/disk-image new file mode 100644 index 00000000..355c010c --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/disk-image @@ -0,0 +1,3 @@ +base/disk-image.binary +base/disk-image-uefi.binary +base/disk-image-ppc64el.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 b/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 new file mode 100644 index 00000000..cc3ced35 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/qcow2 @@ -0,0 +1,2 @@ +depends disk-image +base/qcow2-image.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/root-dir b/live-build/ubuntu-cpc/hooks.d/base/series/root-dir new file mode 100644 index 00000000..b5d3b4e4 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/root-dir @@ -0,0 +1 @@ +base/create-root-dir.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/squashfs b/live-build/ubuntu-cpc/hooks.d/base/series/squashfs new file mode 100644 index 00000000..60332761 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/squashfs @@ -0,0 +1,2 @@ +depends root-dir +base/root-squashfs.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/tarball b/live-build/ubuntu-cpc/hooks.d/base/series/tarball new file mode 100644 index 00000000..2ea30bf2 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/tarball @@ -0,0 +1,2 @@ +depends root-dir +base/root-xz.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/vagrant b/live-build/ubuntu-cpc/hooks.d/base/series/vagrant new file mode 100644 index 00000000..a4eeb86f --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/vagrant @@ -0,0 +1,2 @@ +depends disk-image +base/vagrant.binary diff --git a/live-build/ubuntu-cpc/hooks.d/base/series/vmdk b/live-build/ubuntu-cpc/hooks.d/base/series/vmdk new file mode 100644 index 00000000..ba0acbae --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/base/series/vmdk @@ -0,0 +1,3 @@ +depends disk-image +base/vmdk-image.binary +base/vmdk-ova-image.binary diff --git a/live-build/ubuntu-cpc/hooks/042-vagrant.binary b/live-build/ubuntu-cpc/hooks.d/base/vagrant.binary similarity index 98% rename from live-build/ubuntu-cpc/hooks/042-vagrant.binary rename to live-build/ubuntu-cpc/hooks.d/base/vagrant.binary index cb6c593c..5e95c334 100755 --- a/live-build/ubuntu-cpc/hooks/042-vagrant.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/vagrant.binary @@ -24,15 +24,6 @@ case ${SUBPROJECT:-} in ;; esac -case $IMAGE_TARGETS in - ""|*vagrant*) - ;; - *) - echo "Skipping Vagrant image build" - exit 0 - ;; -esac - cur_d=${PWD} my_d=$(dirname $(readlink -f ${0})) diff --git a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary b/live-build/ubuntu-cpc/hooks.d/base/vmdk-image.binary similarity index 85% rename from live-build/ubuntu-cpc/hooks/040-vmdk-image.binary rename to live-build/ubuntu-cpc/hooks.d/base/vmdk-image.binary index 32740127..3c2a6449 100755 --- a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/vmdk-image.binary @@ -18,15 +18,6 @@ case $ARCH in exit 0;; esac -case ${IMAGE_TARGETS:-} in - ""|*vmdk*) - ;; - *) - echo "Skipping VMDK image build" - exit 0 - ;; -esac - . config/functions if [ -e binary/boot/disk-uefi.ext4 ]; then diff --git a/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary b/live-build/ubuntu-cpc/hooks.d/base/vmdk-ova-image.binary similarity index 96% rename from live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary rename to live-build/ubuntu-cpc/hooks.d/base/vmdk-ova-image.binary index 77a4b11a..f3414f71 100755 --- a/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary +++ b/live-build/ubuntu-cpc/hooks.d/base/vmdk-ova-image.binary @@ -32,15 +32,6 @@ case $ARCH in exit 0;; esac -case ${IMAGE_TARGETS:-} in - ""|*vmdk*) - ;; - *) - echo "Skipping OVA image build" - exit 0 - ;; -esac - cur_d=${PWD} my_d=$(dirname $(readlink -f ${0})) diff --git a/live-build/ubuntu-cpc/hooks/001-divert-sync.chroot_early b/live-build/ubuntu-cpc/hooks.d/chroot/001-divert-sync.chroot_early similarity index 100% rename from live-build/ubuntu-cpc/hooks/001-divert-sync.chroot_early rename to live-build/ubuntu-cpc/hooks.d/chroot/001-divert-sync.chroot_early diff --git a/live-build/ubuntu-cpc/hooks/010-write-etc-ec2-version.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/010-write-etc-ec2-version.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/010-write-etc-ec2-version.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/010-write-etc-ec2-version.chroot diff --git a/live-build/ubuntu-cpc/hooks/020-pkg-configure.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/020-pkg-configure.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/020-pkg-configure.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/020-pkg-configure.chroot diff --git a/live-build/ubuntu-cpc/hooks/025-create-groups.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/025-create-groups.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/025-create-groups.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/025-create-groups.chroot diff --git a/live-build/ubuntu-cpc/hooks/052-ssh_authentication.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/052-ssh_authentication.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/052-ssh_authentication.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/052-ssh_authentication.chroot diff --git a/live-build/ubuntu-cpc/hooks/060-ipv6.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/060-ipv6.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/060-ipv6.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/060-ipv6.chroot diff --git a/live-build/ubuntu-cpc/hooks/061-open-iscsi.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/061-open-iscsi.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/061-open-iscsi.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/061-open-iscsi.chroot diff --git a/live-build/ubuntu-cpc/hooks/099-cleanup.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/099-cleanup.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/099-cleanup.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/099-cleanup.chroot diff --git a/live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/999-cpc-fixes.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/999-cpc-fixes.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/999-cpc-fixes.chroot diff --git a/live-build/ubuntu-cpc/hooks/999-undivert-sync.chroot b/live-build/ubuntu-cpc/hooks.d/chroot/999-undivert-sync.chroot similarity index 100% rename from live-build/ubuntu-cpc/hooks/999-undivert-sync.chroot rename to live-build/ubuntu-cpc/hooks.d/chroot/999-undivert-sync.chroot diff --git a/live-build/ubuntu-cpc/hooks.d/make-hooks b/live-build/ubuntu-cpc/hooks.d/make-hooks new file mode 100755 index 00000000..7796be3d --- /dev/null +++ b/live-build/ubuntu-cpc/hooks.d/make-hooks @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +#-*- encoding: utf-8 -*- +""" +This script parses a series file and its dependencies and generates a hooks +folder containing symbolic links to the scripts that need to be invoked for +a given image target set. + +For example, if you wish to build the image target sets "vmdk" and "vagrant", +you would call this script as + +./make-hooks --hooks-dir hooks vmdk vagrant + +Scripts live in subfolders below the "hooks.d" folder. Currently the folders +"chroot" and "base" exist. The folder with the name "extra" is reserved for +private scripts, which are not included in the source of livecd-rootfs. The +scripts are not numbered, instead the order of their execution depends on the +order in which they are listed in a series file. + +Series files are placed into the subfolders "base/series" or "extra/series". +Each series file contains a list of scripts to be executed. Empty lines and +lines starting with a '#' are ignored. Series files in "extra/series" override +files in "base/series" with the same name. For example, if a series file +"base/series/cloudA" exists and a series file "extra/series/cloudA", then the +latter will be preferred. + +A series file in "extra/series" may also list scripts that are located in the +"chroot" and "base" folders. In addition, series files can depend on other +series files. For example, the series files for most custom images look similar +to this: + + depends disk-image + depends extra-settings + extra/cloudB.binary + +Where "disk-image" and "extra-settings" may list scripts and dependencies which +are to be processed before the script "extra/cloudB.binary" is called. + +ACHTUNG: live build runs scripts with the suffix ".chroot" in a batch separate +from scripts ending in ".binary". Even if you arrange them interleaved in your +series files, the chroot scripts will be run before the binary scripts. +""" + +import argparse +import os +import re +import shutil +import sys +import yaml + +SCRIPT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(sys.argv[0]))) +HOOKS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "hooks")) + +EXIT_OK = 0 +EXIT_ERR = 1 + +class MakeHooksError(Exception): + pass + +class MakeHooks: + """This class provides series file parsing and symlink generator + functionality.""" + + def __init__(self, hooks_dir=None, quiet=False): + """The hooks_dir parameter can be used to specify the path to the + directory, into which the hook symlinks to the actual script files + should be placed. + + If quiet is set to True, info messages during symlink creation will + be suppressed. Use this if your build is not private, but you would + like to hide which scripts are being run. + """ + self._script_dir = SCRIPT_DIR + self._hooks_dir = hooks_dir or HOOKS_DIR + self._quiet = quiet + self._hooks_list = [] + self._included = set() + + def reset(self): + """Reset the internal state allowing instance to be reused for + another run.""" + self._hooks_list.clear() + self._included.clear() + + def print_usage(self): + print( + "CPC live build hook generator script \n" + " \n" + "Usage: ./make-hooks.sh [OPTIONS] \n" + " \n" + "Options: \n" + " \n" + " --help, -h Show this message and exit. \n" + " --quiet, -q Only show warnings and error messages. \n" + " --hooks-dir, -d The directory where to write the symlinks.\n" + ) + + def find_series_file(self, image_set): + """Search for the series file requested in the image_set parameter. + + The image_set parameter should be a string containing the name of an + image target set represented by a series file. First the "extra/series" + folder is searched followed by the "base/series" folder. + + When a file with the given name is found, the search stops and the + full path to the file is returned. + """ + for subdir in ["extra", "base"]: + series_file = os.path.join(self._script_dir, subdir, "series", + image_set) + if os.path.isfile(series_file): + return series_file + return None + + def make_hooks(self, image_sets): + """Entry point for parsing series files and their dependencies and + for generating the symlinks in the hooks folder. + + The image_sets parameter must be an iterable containing the names of + the series files representing the corresponding image target sets, + e.g. "vmdk" or "vagrant". + """ + self.collect_chroot_hooks() + self.collect_binary_hooks(image_sets) + self.create_symlinks() + + def collect_chroot_hooks(self): + """Chroot hooks are numbered and not explicitly mentioned in series + files. Collect them, sort them and add them to the internal list of + paths to hook sripts. + """ + chroot_hooks_dir = os.path.join(self._script_dir, "chroot") + + chroot_entries = os.listdir(chroot_hooks_dir) + chroot_entries.sort() + + for entry in chroot_entries: + if not (entry.endswith(".chroot_early") or + entry.endswith(".chroot")): + continue + self._hooks_list.append(os.path.join("chroot", entry)) + + def collect_binary_hooks(self, image_sets): + """Search the series files for the given image_sets and parse them + and their dependencies to generate a list of hook scripts to be run + during image build. + + The image_sets parameter must be an iterable containing the names of + the series files representing the corresponding image target sets, + e.g. "vmdk" or "vagrant". + + Populates the internal list of paths to hook scripts in the order in + which the scripts are to be run. + """ + for image_set in image_sets: + series_file = self.find_series_file(image_set) + + if not series_file: + raise MakeHooksError( + "Series file for image set '%s' not found." % image_set) + + with open(series_file, "r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line: + continue + m = re.match(r"^\s*depends\s+(\S+.*)$", line) + if m: + include_set = m.group(1) + if include_set in self._included: + continue + self._included.add(include_set) + self.collect_binary_hooks([include_set,]) + continue + if not line in self._hooks_list: + self._hooks_list.append(line) + + def create_symlinks(self): + """Once the internal list of hooks scripts has been populated by a + call to collect_?_hooks, this method is used to populate the hooks + folder with enumerated symbolic links to the hooks scripts. If the + folder does not exist, it will be created. If it exists, it must be + empty or a MakeHooksError will be thrown. + """ + if os.path.isdir(self._hooks_dir) and os.listdir(self._hooks_dir): + # Only print a warning, because directory might have been created + # by auto/config voodoo. + sys.stderr.write("WARNING: Hooks directory exists and is not empty.\n") + os.makedirs(self._hooks_dir, exist_ok=True) + + for counter, hook in enumerate(self._hooks_list, start=1): + hook_basename = os.path.basename(hook) + + m = re.match(r"^\d+-(?:\d+-)?(?P.*)$", hook_basename) + if m: + hook_basename = m.group("basename") + + linkname = ("%03d-" % counter) + hook_basename + linksrc = os.path.join(self._hooks_dir, linkname) + linkdest = os.path.relpath(os.path.join(self._script_dir, hook), + self._hooks_dir) + + if not self._quiet: + print("[HOOK] %s => %s" % (linkname, hook)) + os.symlink(linkdest, linksrc) + + def cli(self, args): + """Command line interface to the hooks generator.""" + parser = argparse.ArgumentParser() + + parser.add_argument("-q", "--quiet", dest="quiet", type=bool, + help="Only show warnings and error messages.") + parser.add_argument("-d", "--hooks-dir", dest="hooks_dir", type=str, + help="The directory where to create the symlinks.") + parser.add_argument("image_target", nargs="+", type=str, + help="") + + self.reset() + options = parser.parse_args(args) + + # Copy options to object attributes. + for key, value in vars(options).items(): + if value and hasattr(self, "_" + key): + setattr(self, "_" + key, value) + + # Take remaining command line arguments, sanitize and turn into list. + image_sets = re.sub(r";|,", " ", " ".join(options.image_target))\ + .split() + + self.make_hooks(image_sets) + + +if __name__ == "__main__": + try: + MakeHooks().cli(sys.argv[1:]) + except MakeHooksError as e: + sys.stderr.write("%s: %s\n" % (os.path.basename(sys.argv[0]), str(e))) + sys.exit(EXIT_ERR) diff --git a/live-build/ubuntu-cpc/hooks/999-extras.binary b/live-build/ubuntu-cpc/hooks/999-extras.binary deleted file mode 100755 index 0d506ea4..00000000 --- a/live-build/ubuntu-cpc/hooks/999-extras.binary +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Execute extra binary hooks. - -my_dir=$(dirname $(readlink -f ${0})) -extra_d=${my_dir}/extra - -if [ ! -d ${my_dir}/extra ]; then - exit 0 -fi - -export IMAGE_STR="# CLOUD_IMG: This file was created/modified by the Cloud Image build process" -export CLOUD_IMG_STR="$IMAGE_STR" -export FS_LABEL="cloudimg-rootfs" - -# Cleaner execution -/bin/run-parts --exit-on-error --regex ".*\.binary" "${extra_d}" diff --git a/lp-in-release b/lp-in-release new file mode 120000 index 00000000..fbc6dd86 --- /dev/null +++ b/lp-in-release @@ -0,0 +1 @@ +magic-proxy \ No newline at end of file diff --git a/magic-proxy b/magic-proxy new file mode 100755 index 00000000..50481e28 --- /dev/null +++ b/magic-proxy @@ -0,0 +1,970 @@ +#!/usr/bin/python3 -u +#-*- encoding: utf-8 -*- +""" +This script can be called as "lp-in-release" or as "magic-proxy". When called +under the former name, it acts as a CLI tool, when called under the latter name +it will act as a transparent HTTP proxy. + +The CLI tool parses the directory listing of + + http:///dists/suite/by-hash/SHA256 + +and figures out which hashes belong to an InRelease file. For example, to list +all available hashes for "cosmic" run + + ./lp-in-release list --suite cosmic + +Per default the script scans archive.ubuntu.com, but you can tell it to use a +different mirror with the --mirror-url command line parameter. Analogously, you +can list the hashes for "cosmic-updates" or "cosmic-security". The script can +also find the hash that was valid at a given timestamp via + + ./lp-in-release select --suite cosmic --cutoff-time + +Finally, you can use the script to inject inrelease-path settings into a +sources.list file via + + ./lp-in-release inject --cutoff-time /etc/apt/sources.list + +The proxy is just an extension to this functionality. Whenever a URL points at +an InRelease file or a path listed in an InRelease file, the proxy will +automatically inject the by hash URL for the resource according to the timestamp +it was configured for. The proxy works in transparent and non-transparent mode. +""" +from datetime import datetime, timedelta, tzinfo + +import argparse +import copy +import fcntl +import getopt +import hashlib +import http.client +import http.server +import json +import os +import pwd +import re +import shutil +import socketserver +import sys +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +EXIT_OK = 0 +EXIT_ERR = 1 + +class LPInReleaseBaseError(Exception): + pass + +class LPInReleaseIndexError(LPInReleaseBaseError): + pass + +class LPInReleaseCacheError(LPInReleaseBaseError): + pass + +class LPInReleaseProxyError(LPInReleaseBaseError): + pass + +class InRelease: + """This class represents an InRelease file.""" + + def __init__(self, mirror, suite, data, hash_=None, last_modified=None): + """mirror must contain the proper URL of the package repository up to + the "dists" folder, e.g. + + http://archive.ubuntu.com/ubuntu + + suite is the name of the suite this InRelease file belongs to, e.g. + , -updates or -security. + + data must contain the full contents of the InReleaes file as a unicode + string. + + If supplied, then hash_ will be used as the sha256 hexdigest of the + binary encoding of the InRelease file. If not supplied, the hash will + be calculated. This is just used as a time-saver, when cache contents + are read back in. + + last_modified must be a string of format + + Thu, 26 Apr 2018 23:37:48 UTC + + representing the publication time of the InRelease file. If not given, + the generation time stored in the InRelease file will be used. Below, + this is set explicitly to correspond to the Last-Modified header spat + out by the Web server. + """ + self.mirror = mirror + self.suite = suite + self.data = data + self.dict = {} + + if hash_: + self.hash = hash_ + else: + h = hashlib.sha256() + h.update(data.encode("utf-8")) + self.hash = h.hexdigest() + + if last_modified: + self.published = self._parse_datetime(last_modified) + else: + self.published = self._extract_timestamp(data) + + @property + def datetime(self): + """Return the publication time of this InRelease file as a string in + YYYY-MM-DD HH:MM:SS ISO format. The result is always in GMT.""" + return datetime \ + .utcfromtimestamp(self.published) \ + .strftime('%Y-%m-%d %H:%M:%S') + + @property + def normalized_address(self): + """Return the "normalized" address of the mirror URL, consisting of + only the hostname and the path. This may be used as an index into an + InReleaseCache.""" + result = urllib.parse.urlparse(self.mirror) + address = result.hostname + result.path.rstrip("/") + return address + + @property + def contents(self): + """Return the pure contents of the InRelease file with the signature + stripped off.""" + return self._split_release_and_sig(self.data)[0] + + @property + def signature(self): + """Return the ASCII-armored PGP signature of the InRelease file.""" + return self._split_release_and_sig(self.data)[1] + + def serialize(self): + """Serializes the InRelease object into Python structures to be stored + in an InReleaseCache.""" + month_names = [ "_ignore_", + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ] + + wkday_names = [ + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", + ] + + dt = datetime.utcfromtimestamp(self.published) + + published = "{}, {:02} {} {} {:02}:{:02}:{:02} GMT".format( + wkday_names[dt.weekday()], + dt.day, + month_names[dt.month], + dt.year, + dt.hour, + dt.minute, + dt.second + ) + + return { + "mirror": self.mirror, + "suite": self.suite, + "hash": self.hash, + "published": published, + "data": self.data, + } + + def get_hash_for(self, path): + """Check if the given path is listed in this InRelease file and if so + return the corresponding hash in hexdigest format. If the path is not + listed, None is returned.""" + if not self.dict: + self._parse_contents() + return self.dict.get(path) + + def _parse_contents(self): + """This method parses out all lines containing SHA256 hashes and creates + an internal dict, mapping resources to hashes.""" + regex = re.compile( + r" (?P[0-9a-f]{64})\s+(?P\d+)\s+(?P\S+)") + + for line in self.contents.splitlines(): + m = regex.match(line) + if not m: + continue + self.dict[m.group("path")] = m.group("hash") + + def _parse_datetime(self, datetime_string): + """Because the behavior of Python's strptime's would be + locale-dependent, we parse datetime strings of the format found in + Last-Modified HTTP headers ourselves. This returns an integer + representing a posix timestamp or None, if the parsing failed.""" + class UTC(tzinfo): + def utcoffset(self, dt): + return timedelta(0) + + # we need a map, because strptime would be locale-dependent + month_name_to_number = { + "Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6, + "Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12 + } + + rexpr = r"""^\s*\w+,\s+ + (?P\d+) \s+ + (?P\w+) \s+ + (?P\d+) \s+ + (?P\d+) : + (?P\d+) : + (?P\d+) .*$""" + + m = re.match(rexpr, datetime_string, flags=re.VERBOSE) + if not m: + return None + + parts = list(m.group("year", "month", "day", "hour", "min", "sec")) + parts[1] = month_name_to_number[m.group("month")] + parts = [int(s) for s in parts] + dt = datetime(*parts, tzinfo=UTC()) + epoch = datetime(1970, 1, 1, tzinfo=UTC()) + posix = (dt - epoch).total_seconds() + + return int(posix) + + def _extract_timestamp(self, data): + """Parse the contents of the InRelease file to find the time it was + generated. Returns a POSIX timestamp if found or None otherwise.""" + for line in data.splitlines(): + if line.startswith("Date:"): + return self._parse_datetime(line.split(":", 1)[1]) + + return None + + def _split_release_and_sig(self, data): + """Split the InRelease file into content and signature parts and return + a tuple of unicode strings (content, signature).""" + rexpr = re.escape("-----BEGIN PGP SIGNED MESSAGE-----") + r"\r?\n|" + \ + re.escape("-----BEGIN PGP SIGNATURE-----" ) + r"\r?\n|" + \ + re.escape("-----END PGP SIGNATURE-----" ) + + # returns content and signature + return re.split(rexpr, data)[1:3] + + +class LPInReleaseCache: + """A cache for InRelease files that can optionally be saved to and + loaded from disk.""" + + def __init__(self, filename=None): + """If filename is given, it is the name of the file that cache contents + will be saved to or loaded from when the save and load methods are + called, respectively.""" + self._filename = filename + self._data = {} + self._lock = threading.Lock() + + self.load() + + def load(self): + """Load the cache contents from disk performing some rudimentary file + locking to prevent corruption.""" + if not self._filename: + return + + buf = [] + fd = None + try: + fd = os.open(self._filename, os.O_CREAT | os.O_RDWR) + + fcntl.flock(fd, fcntl.LOCK_EX) + + while True: + tmp = os.read(fd, 4096) + if not tmp: + break + buf.append(tmp) + + fcntl.flock(fd, fcntl.LOCK_UN) + except OSError as e: + raise LPInReleaseCacheError("Failed to load cache file: {}" + .format(str(e))) + finally: + if fd: + os.close(fd) + + cache_data = {} if not buf else json.loads( + b"".join(buf).decode("utf-8")) + + with self._lock: + self._data = cache_data + + def save(self): + """Save the cache contents to disk performing some rudimentary file + locking to prevent corruption.""" + if not self._filename: + return + + with self._lock: + buf = json \ + .dumps(self._data, ensure_ascii=False, indent=4, + sort_keys=True) \ + .encode("utf-8") + + fd = None + try: + fd = os.open(self._filename, os.O_CREAT | os.O_RDWR) + + fcntl.flock(fd, fcntl.LOCK_EX) + + os.ftruncate(fd, 0) + os.write(fd, buf) + + fcntl.flock(fd, fcntl.LOCK_UN) + except OSError as e: + raise LPInReleaseCacheError("Failed to store cache file: {}" + .format(str(e))) + finally: + if fd: + os.close(fd) + + def add(self, inrelease): + """Add the given InRelease object to the cache.""" + with self._lock: + self._data \ + .setdefault(inrelease.normalized_address, {}) \ + .setdefault(inrelease.suite, {}) \ + .setdefault(inrelease.hash, inrelease.serialize()) + + def get_one(self, mirror, suite, hash_): + """Return a single InRelease object for the given mirror and suite, + corresponding to the hash or None if such an entry does not exist.""" + with self._lock: + url_obj = urllib.parse.urlparse(mirror) + address = url_obj.hostname + url_obj.path.rstrip("/") + + inrel = self._data\ + .get(address, {})\ + .get(suite, {})\ + .get(hash_) + + if not inrel: + return None + + return InRelease( + inrel["mirror"], + inrel["suite"], + inrel["data"], + hash_=inrel["hash"], + last_modified=inrel["published"] + ) + + def get_all(self, mirror, suite): + """Retrieve a list of InRelease objects for the given mirror and suite. + Return a list of all known InRelease objects for the given mirror and + suite.""" + with self._lock: + url_obj = urllib.parse.urlparse(mirror) + address = url_obj.hostname + url_obj.path.rstrip("/") + + inrel_by_hash = self._data\ + .get(address, {})\ + .get(suite, {}) + + inrelease_list = [] + + for hash_, inrel in inrel_by_hash.items(): + inrelease_list.append( + InRelease( + inrel["mirror"], + inrel["suite"], + inrel["data"], + hash_=inrel["hash"], + last_modified=inrel["published"] + ) + ) + + return inrelease_list + + +class LPInReleaseIndex: + """Abstraction to the build system's view of the "by hash" database. + Currently, that interface is the by-hash directory listing of the Web + server.""" + + def __init__(self, mirror, suite, cache=None): + """The mirror is the base URL of the repository up to the "dists" + folder, e.g. + + http://archive.ubuntu.com/ubuntu + + suite is the name of the suite this InReleaseIndex object operates on, + e.g. , -updates or -security. + + Optionally, cache can be initialized to a LPInReleaseCache object, in + which case all look-ups will first go to the cache and only cache + misses will result in requests to the Web server. + """ + self._mirror = mirror + self._suite = suite + self._cache = cache + + self._base_url = "/".join([self._mirror, "dists", self._suite, + "by-hash/SHA256"]) + + def inrelease_files(self): + """Iterate over all InRelease files found in the archive for the mirror + and suite this index has been configured to operate on.""" + hashes = self._retrieve_hashes() + + for h in hashes: + inrelease = None + + if self._cache: + inrelease = self._cache.get_one(self._mirror, + self._suite, hash_=h) + if not inrelease: + inrelease = self._retrieve_inrelease(h) + if not inrelease: + continue + + yield inrelease + + def get_inrelease_for_timestamp(self, time_gmt): + """Find and return the InRelease file that was valid at the given Posix + timestamp.""" + candidate = None + + for inrelease in self.inrelease_files(): + if inrelease.published > time_gmt: + continue + if not candidate or inrelease.published > candidate.published: + candidate = inrelease + + return candidate + + def _retrieve_inrelease(self, hash_): + """Retrieve the contents of the file identified by hash_. Check if the + file is an InRelease file and return a corresponding InRelease object. + If the hash_ does not belong to an InRelease file, None is returned.""" + _500KB = 500 * 1024 + + buf = b"" + inrelease = None + url = self._base_url + "/" + hash_ + + try: + with urllib.request.urlopen(url) as response: + + # InRelease files seem to be around 200-300KB + + content_length = response.headers.get("Content-Length") + last_modified = response.headers.get("Last-Modified") + + if not content_length: + buf = response.read(_500KB + 1) + content_length = len(buf) + else: + content_length = int(content_length) + + # Slightly silly heuristic, but does the job + + if content_length > _500KB or content_length < 1024: + return None + + buf += response.read() + + content_encoding = self \ + ._guess_content_encoding_for_response(response) + + # few additional checks to see if this is an InRelease file + + try: + buf = buf.decode(content_encoding) + except UnicodeError: + return None + + if not buf.startswith("-----BEGIN PGP SIGNED MESSAGE-----"): + return None + + for kw in ["Origin:", "Label:", "Suite:", "Acquire-By-Hash:"]: + if not kw in buf: + return None + + inrelease = InRelease(self._mirror, self._suite, buf, + hash_=hash_, last_modified=last_modified) + + if self._cache: + self._cache.add(inrelease) + except urllib.error.HTTPError as e: + if not e.code in [404,]: + raise LPInReleaseIndexError("Error retrieving {}: {}" + .format(url, str(e))) + + return inrelease + + def _guess_content_encoding_for_response(self, response): + """Guess the content encoding of the given HTTPResponse object.""" + content_encoding = response.headers.get("Content-Encoding") + content_type = response.headers.get("Content-Type", + "text/html;charset=UTF-8") + + if not content_encoding: + m = re.match(r"^.*charset=(\S+)$", content_type) + + if m: + content_encoding = m.group(1) + else: + content_encoding = "UTF-8" + + return content_encoding + + def _retrieve_hashes(self): + """Retrieve all available by-hashes for the mirror and suite that this + index is configured to operate on.""" + hashes = [] + + if self._cache: + cache_entry = self._cache.get_all(self._mirror, self._suite) + if cache_entry: + return [inrel.hash for inrel in cache_entry] + + try: + with urllib.request.urlopen(self._base_url) as response: + content_encoding = self._guess_content_encoding_for_response( + response) + + body = response.read().decode(content_encoding) + hashes = list(set(re.findall(r"[a-z0-9]{64}", body))) + except urllib.error.URLError as e: + raise LPInReleaseIndexError("Could not retrieve hash listing: {}" + .format(str(e))) + + return hashes + + +class LPInReleaseIndexCli: + """A CLI interface for LPInReleaseIndex.""" + + def __init__(self, name): + self._name = name + self._mirror = None + self._suite = None + self._timestamp = None + self._cachefile = None + self._cache = None + self._infile = None + self._outfile = None + + def __call__(self, args): + options = vars(self._parse_opts(args)) + + # Copy settings to object attributes + for key, value in options.items(): + if hasattr(self, "_" + key): + setattr(self, "_" + key, value) + + if self._cachefile: + self._cache = LPInReleaseCache(self._cachefile) + + try: + options["func"]() + except LPInReleaseIndexError as e: + sys.stderr.write("{}: {}\n".format(self._name, str(e))) + sys.exit(EXIT_ERR) + + if self._cache: + self._cache.save() + + def list(self): + """List all InRelease hashes for a given mirror and suite.""" + for inrelease in self._list(self._mirror, self._suite): + if self._timestamp and inrelease.published > self._timestamp: + continue + + print("{} {} ({})".format( + inrelease.hash, + inrelease.datetime, + inrelease.published, + )) + + def select(self): + """Find the hash of the InRelease file valid at a given timestamp.""" + candidate = self._select(self._mirror, self._suite) + + if candidate: + print("{} {} ({})".format( + candidate.hash, + candidate.datetime, + candidate.published, + )) + + def inject(self): + """Inject by-hash and inrelease-path settings into a sources.list.""" + sources_list = self._infile + + if not os.path.exists(sources_list): + sys.stderr.write("{}: No such file: {}.\n" + .format(self._name, sources_list)) + sys.exit(EXIT_ERR) + + with open(sources_list, "r", encoding="utf-8") as fp: + buf = fp.read() + + rexpr = re.compile(r"""^ + (?Pdeb(?:-src)?)\s+ + (?P\[[^\]]+\]\s+)? + (?P(?P\S+):\S+)\s+ + (?P\S+)\s+ + (?P.*)$""", flags=re.VERBOSE) + + lines = buf.splitlines(True) + + for i, line in enumerate(lines): + line = lines[i] + m = rexpr.match(line) + + if not m: + continue + if m.group("scheme") not in ["http", "https", "ftp"]: + continue + + opts = {} + if m.group("opts"): + for entry in m.group("opts").strip().strip("[]").split(): + k, v = entry.split("=") + opts[k] = v + + inrelease = self._select(m.group("mirror"), m.group("suite")) + if inrelease: + opts["by-hash"] = "yes" + opts["inrelease-path"] = "by-hash/SHA256/" + inrelease.hash + + groupdict = m.groupdict() + groupdict["opts"] = " ".join(["{0}={1}".format(*o) for o in + opts.items()]) + + lines[i] = "{type} [{opts}] {mirror} {suite} {comps}\n"\ + .format(**groupdict) + + outfile = None + try: + if not self._outfile or self._outfile == "-": + outfile = sys.stdout + else: + outfile = open(self._outfile, "w+", encoding="utf-8") + outfile.write("".join(lines)) + finally: + if outfile and outfile != sys.stdout: + outfile.close() + + def _parse_opts(self, args): + """Parse command line arguments and initialize the CLI object.""" + main_parser = argparse.ArgumentParser() + subparsers = main_parser.add_subparsers(dest="command") + + parser_inject = subparsers.add_parser("inject", + help="Rewrite a sources.list file injecting appropriate hashes.") + parser_list = subparsers.add_parser("list", + help="List InRelease hashes for a given release and suite.") + parser_select = subparsers.add_parser("select", + help="Select hash to use for a given timestamp, release, suite.") + + parser_inject.set_defaults(func=self.inject) + parser_list.set_defaults(func=self.list) + parser_select.set_defaults(func=self.select) + + # Options common to all commands + for parser in [parser_inject, parser_list, parser_select]: + cutoff_time_required = True if parser != parser_list else False + + parser.add_argument("-t", "--cutoff-time", dest="timestamp", + type=int, required=cutoff_time_required, + help="A POSIX timestamp to pin the repo to.") + parser.add_argument("--cache-file", dest="cachefile", type=str, + help="A file where to cache intermediate results (optional).") + + mirror = "http://archive.ubuntu.com/ubuntu" + + # Options common to list, select commands + for parser in [parser_list, parser_select]: + parser.add_argument("-m", "--mirror", dest="mirror", type=str, + default=mirror, help="The URL of the mirror to use.") + parser.add_argument("-s", "--suite", + dest="suite", type=str, required=True, + help="The suite to scan (e.g. 'bionic', 'bionic-updates').") + + # Extra option for inject command + parser_inject.add_argument("-o", "--output-file", dest="outfile", + type=str, help="") + parser_inject.add_argument("infile", type=str, + help="The sources.list file to modify.") + + if not args: + main_parser.print_help() + sys.exit(EXIT_ERR) + + return main_parser.parse_args(args) + + def _list(self, mirror, suite): + """Internal helper for the list command. This is also used + implicitly by the _select method.""" + index = LPInReleaseIndex(mirror, suite, cache=self._cache) + + inrelease_files = \ + reversed( + sorted( + list(index.inrelease_files()), + key=lambda x: x.published + ) + ) + + return inrelease_files + + def _select(self, mirror, suite): + """Internal helper for the select command.""" + candidate = None + + for inrelease in self._list(mirror, suite): + if inrelease.published > self._timestamp: + continue + if not candidate or inrelease.published > candidate.published: + candidate = inrelease + + return candidate + + +class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler): + """Request handler providing a virtual snapshot of the package + repositories.""" + + def do_HEAD(self): + """Process a HEAD request.""" + self.__get_request(verb="HEAD") + + def do_GET(self): + """Process a GET request.""" + self.__get_request() + + def __get_request(self, verb="GET"): + """Pass all requests on to the destination server 1:1 except when the + target is an InRelease file or a resource listed in an InRelease files. + + In that case we silently download the resource via the by-hash URL + which was most recent at the cutoff (or repo snapshot) time and inject + it into the response. + + It is important to understand that there is no status 3xx HTTP redirect + happening here, the client does not know that what it receives is not + exactly what it requested.""" + + host, path = self.__get_host_path() + + m = re.match( + r"^(?P.*?)/dists/(?P[^/]+)/(?P.*)$", + path + ) + + if m: + mirror = "http://" + host + m.group("base") + base = m.group("base") + suite = m.group("suite") + target = m.group("target") + + index = LPInReleaseIndex(mirror, suite, + cache=self.server.inrelease_cache) + inrelease = index.get_inrelease_for_timestamp( + self.server.snapshot_stamp) + + if inrelease is None: + self.__send_error(404, "No InRelease file found for given " + "mirror, suite and timestamp.") + return + + if target == "InRelease": + # If target is InRelease, send back contents directly. + data = inrelease.data.encode("utf-8") + + self.log_message( + "Inject InRelease '{}'".format(inrelease.hash)) + + self.send_response(200) + self.send_header("Content-Length", len(data)) + self.end_headers() + + if verb == "GET": + self.wfile.write(data) + + return + else: + # If target hash is listed, then redirect to by-hash URL. + hash_ = inrelease.get_hash_for(target) + + if hash_: + self.log_message( + "Inject {} for {}".format(hash_, target)) + + target_path = target.rsplit("/", 1)[0] + + path = "{}/dists/{}/{}/by-hash/SHA256/{}"\ + .format(base, suite, target_path, hash_) + + try: + client = http.client.HTTPConnection(host) + client.request(verb, path) + except Exception as e: + self.log_error("Failed to retrieve http://{}{}: {}" + .format(host, path, str(e))) + return + + try: + self.__send_response(client.getresponse()) + except Exception as e: + self.log_error("Error delivering response: {}".format(str(e))) + + def __get_host_path(self): + """Figure out the host to contact and the path of the resource that is + being requested.""" + host = self.headers.get("host") + url = urllib.parse.urlparse(self.path) + path = url.path + + return host, path + + def __send_response(self, response): + """Pass on upstream response headers and body to the client.""" + self.send_response(response.status) + + for name, value in response.getheaders(): + self.send_header(name, value) + + self.end_headers() + shutil.copyfileobj(response, self.wfile) + + def __send_error(self, status, message): + """Return an HTTP error status and a message in the response body.""" + self.send_response(status) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.wfile.write(message.encode("utf-8")) + + +class MagicHTTPProxy(socketserver.ThreadingMixIn, http.server.HTTPServer): + """Tiny HTTP server using ProxyingHTTPRequestHandler instances to provide + a snapshot view of the package repositories.""" + + def __init__(self, server_address, server_port, cache_file=None, + repo_snapshot_stamp=time.time(), run_as=None): + + try: + super(http.server.HTTPServer, self).__init__( + (server_address, server_port), ProxyingHTTPRequestHandler) + except OSError as e: + raise LPInReleaseProxyError( + "Could not initialize proxy: {}".format(str(e))) + + self.inrelease_cache = LPInReleaseCache(filename=cache_file) + self.snapshot_stamp = repo_snapshot_stamp + + +class MagicHTTPProxyCli: + """A CLI interface for the MagicHTTPProxy.""" + + def __init__(self, name): + self._name = name + self._address = "127.0.0.1" + self._port = 8080 + self._timestamp = time.time() + self._run_as = None + self._pid_file = None + self._log_file = None + self._background = False + self._setsid = False + + def __call__(self, args): + options = self._parse_opts(args) + + proxy = MagicHTTPProxy( + options.address, + options.port, + cache_file=None, + repo_snapshot_stamp=options.timestamp + ) + + # Detach, but keep all streams open. + if options.background: + pid = os.fork() + if pid: + os._exit(EXIT_OK) + + if options.log_file: + fd = open(options.log_file, "wb+") + os.dup2(fd.fileno(), sys.stdout.fileno()) + os.dup2(fd.fileno(), sys.stderr.fileno()) + + # Become session leader and give up controlling terminal. + if options.setsid: + if not options.log_file: + fd = open(os.devnull, "wb+") + os.dup2(fd.fileno(), sys.stdout.fileno()) + os.dup2(fd.fileno(), sys.stderr.fileno()) + os.setsid() + + if options.pid_file: + with open(options.pid_file, "w+", encoding="utf-8") as fp: + fp.write(str(os.getpid())) + + if options.run_as is not None: + try: + uid = pwd.getpwnam(options.run_as).pw_uid + os.setuid(uid) + except KeyError as e: + sys.stderr.write("Failed to lookup {}: {}\n" + .format(options.run_as, str(e))) + sys.exit(EXIT_ERR) + except PermissionError as e: + sys.stderr.write("Cannot setuid: {}\n".format(str(e))) + sys.exit(EXIT_ERR) + + proxy.serve_forever() + + def _parse_opts(self, args): + """Parse command line arguments and initialize the CLI object.""" + parser = argparse.ArgumentParser() + + parser.add_argument("--address", dest="address", type=str, + default="127.0.0.1", help="The address of the interface to " + "bind to (default: 127.0.0.1)") + parser.add_argument("--port", dest="port", type=int, default=8080, + help="The port to listen on (default: 8080)") + parser.add_argument("-t", "--cutoff-time", dest="timestamp", type=int, + required=True, help="A POSIX timestamp to pin the repo to.") + parser.add_argument("--run-as", dest="run_as", type=str, + help="Drop privileges and run as this user.") + parser.add_argument("--pid-file", dest="pid_file", type=str, + help="Store the PID to this file.") + parser.add_argument("--log-file", dest="log_file", type=str, + help="Re-direct all streams to this file.") + parser.add_argument("--background", dest="background", + action="store_true", + help="Whether to go into the background.") + parser.add_argument("--setsid", dest="setsid", + action="store_true", + help="Become session leader and drop controlling TTY.") + + return parser.parse_args(args) + +if __name__ == "__main__": + name = os.path.basename(sys.argv[0]) + + try: + if name == "lp-in-release": + cli = LPInReleaseIndexCli(name) + else: + cli = MagicHTTPProxyCli(name) + + cli(sys.argv[1:]) + except LPInReleaseBaseError as e: + sys.stderr.write("{}: {}\n".format(name, str(e))) + sys.exit(EXIT_ERR) + except KeyboardInterrupt: + sys.stderr.write("{}: Caught keyboard interrupt, exiting...\n" + .format(name)) + sys.exit(EXIT_ERR) diff --git a/snap-tool b/snap-tool new file mode 100755 index 00000000..44979c17 --- /dev/null +++ b/snap-tool @@ -0,0 +1,433 @@ +#!/usr/bin/python3 +#-*- encoding: utf-8 -*- +""" +This script can be used instead of the traditional `snap` command to download +snaps and accompanying assertions. It uses the new store API (v2) which allows +creating temporary snapshots of the channel map. + +To create such a snapshot run + + snap-tool cohort-create + +This will print a "cohort-key" to stdout, which can then be passed to future +invocations of `snap-tool download`. Whenever a cohort key is provided, the +store will provide a view of the channel map as it existed when the key was +created. +""" + +from textwrap import dedent + +import argparse +import base64 +import binascii +import getopt +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import urllib.error +import urllib.request + +EXIT_OK = 0 +EXIT_ERR = 1 + + +class SnapError(Exception): + """Generic error thrown by the Snap class.""" + pass + + +class SnapCraftError(SnapError): + """Error thrown on problems with the snapcraft APIs.""" + pass + + +class SnapAssertionError(SnapError): + """Error thrown on problems with the assertions API.""" + pass + + +class Snap: + """This class provides methods to retrieve information about a snap and + download it together with its assertions.""" + + def __init__(self, name, channel="stable", arch="amd64", series=16, + cohort_key=None, assertion_url="https://assertions.ubuntu.com", + snapcraft_url="https://api.snapcraft.io", **kwargs): + """ + :param name: + The name of the snap. + :param channel: + The channel to operate on. + :param arch: + The Debian architecture of the snap (e.g. amd64, armhf, arm64, ...). + :param series: + The device series. This should always be 16. + :param cohort_key: + A cohort key to access a snapshot of the channel map. + """ + self._name = name + self._channel = channel + self._arch = arch + self._series = series + self._cohort_key = cohort_key + self._assertion_url = assertion_url + self._snapcraft_url = snapcraft_url + self._details = None + self._assertions = {} + + @classmethod + def cohort_create(cls): + """Get a cohort key for the current moment. A cohort key is valid + across all snaps, channels and architectures.""" + return Snap("core")\ + .get_details(cohort_create=True)\ + .get("cohort-key") + + def download(self, download_assertions=True): + """Download the snap container. If download_assertions is True, the + corresponding assertions will be downloaded, as well.""" + snap = self.get_details() + + snap_name = snap["name"] + snap_revision = snap["revision"] + publisher_id = snap["publisher"]["id"] + snap_download_url = snap["download"]["url"] + snap_byte_size = snap["download"]["size"] + filename = snap_name + "_" + str(snap_revision) + snap_filename = filename + ".snap" + assert_filename = filename + ".assert" + + skip_snap_download = False + + if os.path.exists(snap_filename) and os.path.getsize(snap_filename) \ + == snap_byte_size: + skip_snap_download = True + + headers = {} + + if os.environ.get("SNAPPY_STORE_NO_CDN", "0") != "0": + headers.update({ + "X-Ubuntu-No-Cdn": "true", + "Snap-CDN": "none", + }) + + request = urllib.request.Request(snap_download_url, headers=headers) + + if not skip_snap_download: + with urllib.request.urlopen(request) as response, \ + open(snap_filename, "wb+") as fp: + shutil.copyfileobj(response, fp) + + if not download_assertions: + return + + required_assertions = [ + "account-key", + "account", + "snap-declaration", + "snap-revision", + ] + + if publisher_id == "canonical": + required_assertions.remove("account") + + for assertion_name in required_assertions: + attr_name = "get_assertion_" + assertion_name.replace("-", "_") + # This will populate self._assertions[]. + getattr(self, attr_name)() + + with open(assert_filename, "w+", encoding="utf-8") as fp: + fp.write("\n".join(self._assertions[a] for a in + required_assertions)) + + def get_details(self, cohort_create=False): + """Get details about the snap. On subsequent calls, the cached results + are returned. If cohort_create is set to True, a cohort key will be + created and included in the result.""" + if self._details and not cohort_create: + return self._details + + if self.is_core_snap() and self._channel.startswith("stable/ubuntu-"): + sys.stderr.write( + "WARNING: switching channel from '{}' to 'stable' for '{}' " + "snap.\n".format(self._channel, self._name) + ) + self._channel = "stable" + + path = "/v2/snaps/refresh" + + data = { + "context": [], + "actions": [ + { + "action": "download", + "instance-key": "0", + "name": self._name, + "channel": self._channel, + } + ], + "fields": [ + "base", + "created-at", + "download", + "license", + "name", + "prices", + "publisher", + "revision", + "snap-id", + "summary", + "title", + "type", + "version", + ], + } + + # These are mutually exclusive. + if cohort_create: + data["actions"][0]["cohort-create"] = True + elif self._cohort_key: + data["actions"][0]["cohort-key"] = self._cohort_key + + request_json = json.dumps(data, ensure_ascii=False).encode("utf-8") + + try: + response_dict = self._do_snapcraft_request(path, data=request_json) + except SnapCraftError as e: + raise SnapError("failed to get details for '{}': {}" + .format(self._name, str(e))) + + snap_data = response_dict["results"][0] + + # Have "base" initialized to something meaningful. + if self.is_core_snap(): + snap_data["snap"]["base"] = "" + elif snap_data["snap"].get("base") is None: + snap_data["snap"]["base"] = "core" + + # Copy the key into the snap details. + if "cohort-key" in snap_data: + snap_data["snap"]["cohort-key"] = snap_data["cohort-key"] + + if "error" in snap_data: + raise SnapError( + "failed to get details for '{}' in '{}' on '{}': {}" + .format(self._name, self._channel, self._arch, + snap_data["error"]["message"]) + ) + + self._details = snap_data["snap"] + return self._details + + def get_assertion_snap_revision(self): + """Download the snap-revision assertion associated with this snap. The + assertion is returned as a string.""" + if "snap-revision" in self._assertions: + return self._assertions["snap-revision"] + snap = self.get_details() + + snap_sha3_384 = base64.urlsafe_b64encode( + binascii.a2b_hex(snap["download"]["sha3-384"]) + ).decode("us-ascii") + + data = self._do_assertion_request("/v1/assertions/snap-revision/{}" + .format(snap_sha3_384)) + self._assertions["snap-revision"] = data + return data + + def get_assertion_snap_declaration(self): + """Download the snap-declaration assertion associated with this snap. + The assertion is returned as a string.""" + if "snap-declaration" in self._assertions: + return self._assertions["snap-declaration"] + snap = self.get_details() + series = self._series + snap_id = snap["snap-id"] + + data = self._do_assertion_request( + "/v1/assertions/snap-declaration/{}/{}" + .format(series, snap_id)) + + self._assertions["snap-declaration"] = data + return data + + def get_assertion_account(self): + """Download the account assertion associated with this snap. The + assertion is returned as a string.""" + if "account" in self._assertions: + return self._assertions["account"] + snap = self.get_details() + publisher_id = snap["publisher"]["id"] + data = self._do_assertion_request("/v1/assertions/account/{}" + .format(publisher_id)) + self._assertions["account"] = data + return data + + def get_assertion_account_key(self): + """Download the account-key assertion associated with this snap. The + assertion will be returned as a string.""" + if "account-key" in self._assertions: + return self._assertions["account-key"] + + declaration_data = self.get_assertion_snap_declaration() + sign_key_sha3 = None + + for line in declaration_data.splitlines(): + if line.startswith("sign-key-sha3-384:"): + sign_key_sha3 = line.split(":")[1].strip() + + data = self._do_assertion_request("/v1/assertions/account-key/{}" + .format(sign_key_sha3)) + + self._assertions["account-key"] = data + return data + + def is_core_snap(self): + return re.match(r"^core\d*$", self._name) != None + + def _do_assertion_request(self, path): + url = self._assertion_url + path + + headers = { + "Accept": "application/x.ubuntu.assertion", + } + + request = urllib.request.Request(url, headers=headers) + + try: + with urllib.request.urlopen(request) as response: + body = response.read() + except urllib.error.HTTPError as e: + raise SnapAssertionError(str(e)) + + return body.decode("utf-8") + + def _do_snapcraft_request(self, path, data=None): + url = self._snapcraft_url + "/" + path + + headers = { + "Snap-Device-Series": str(self._series), + "Snap-Device-Architecture": self._arch, + "Content-Type": "application/json", + } + + request = urllib.request.Request(url, data=data, headers=headers) + + try: + with urllib.request.urlopen(request) as response: + body = response.read() + except urllib.error.HTTPError as e: + raise SnapCraftError(str(e)) + + try: + response_data = json.loads(body, encoding="utf-8") + except json.JSONDecodeError as e: + raise SnapCraftError("failed to decode response body: " + str(e)) + + return response_data + + +class SnapCli: + + def __call__(self, args): + """Parse the command line arguments and execute the selected command.""" + options = self._parse_opts(args) + + try: + options.func(getattr(options, "snap", None), **vars(options)) + except SnapError as e: + sys.stderr.write("snap-tool {}: {}\n".format( + options.command, str(e))) + return EXIT_ERR + return EXIT_OK + + @staticmethod + def _get_host_deb_arch(): + result = subprocess.run(["dpkg", "--print-architecture"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, check=True) + + return result.stdout.strip() + + def _parse_opts(self, args): + main_parser = argparse.ArgumentParser() + subparsers = main_parser.add_subparsers(dest="command") + + parser_cohort_create = subparsers.add_parser("cohort-create", + help="Create a cohort key for the snap store channel map.") + parser_cohort_create.set_defaults(func=self._cohort_create) + + parser_download = subparsers.add_parser("download", + help="Download a snap from the store.") + parser_download.set_defaults(func=self._download) + + parser_info = subparsers.add_parser("info", + help="Retrieve information about a snap.") + parser_info.set_defaults(func=self._info) + + # Add common parameters. + for parser in [parser_download, parser_info]: + parser.add_argument("--cohort-key", dest="cohort_key", + help="A cohort key to pin the channel map to.", type=str) + parser.add_argument("--channel", dest="channel", + help="The publication channel to query (default: stable).", + type=str, default="stable") + parser.add_argument("--series", dest="series", + help="The device series (default: 16)", + type=int, default=16) + parser.add_argument("--arch", dest="arch", + help="The Debian architecture (default: amd64).", + type=str, default=self._get_host_deb_arch()) + parser.add_argument("snap", help="The name of the snap.") + + if not args: + main_parser.print_help() + sys.exit(EXIT_ERR) + + return main_parser.parse_args(args) + + def _cohort_create(self, _, **kwargs): + print(Snap.cohort_create()) + + def _download(self, snap_name, **kwargs): + Snap(snap_name, **kwargs).download() + + def _info(self, snap_name, **kwargs): + snap = Snap(snap_name, **kwargs) + info = snap.get_details() + + print(dedent("""\ + name: {} + summary: {} + arch: {} + base: {} + channel: {} + publisher: {} + license: {} + snap-id: {} + revision: {}""" + .format( + snap_name, + info.get("summary", ""), + snap._arch, + info.get("base"), + snap._channel, + info.get("publisher", {}).get("display-name", ""), + info.get("license", ""), + info.get("snap-id", ""), + info.get("revision", "") + )) + ) + + +if __name__ == "__main__": + try: + rval = SnapCli()(sys.argv[1:]) + except KeyboardInterrupt: + sys.stderr.write("snap-tool: caught keyboard interrupt, exiting.\n") + sys.exit(EXIT_ERR) + sys.exit(rval)