From 138465c476e70202a5ec9fa81a68328de5126f7d Mon Sep 17 00:00:00 2001 From: Steve Langasek Date: Fri, 10 Oct 2014 00:11:26 -0700 Subject: [PATCH 01/12] Add support to the ubuntu-touch target to generate a separate custom tarball as part of the build, moving most click packages into it. --- debian/changelog | 7 +++++ live-build/auto/build | 6 +++++ .../hooks/60-install-click.binary | 9 +++++++ .../hooks/60-install-click.chroot | 26 ++++++++++++++++++- .../90-precompile-apparmor-policies.chroot | 4 +++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100755 live-build/ubuntu-touch/hooks/60-install-click.binary diff --git a/debian/changelog b/debian/changelog index 2a5b439d..84e7ed7f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +livecd-rootfs (2.253) UNRELEASED; urgency=medium + + * Add support to the ubuntu-touch target to generate a separate custom + tarball as part of the build, moving most click packages into it. + + -- Steve Langasek Thu, 09 Oct 2014 20:46:46 -0700 + livecd-rootfs (2.252) utopic; urgency=low [ James Hunt ] diff --git a/live-build/auto/build b/live-build/auto/build index 197e3113..c1036ec1 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -298,6 +298,12 @@ elif [ -e binary-tar.tar.gz ]; then cp -a binary-tar.tar.gz "$PREFIX.rootfs.tar.gz" fi +if [ "$PROJECT" = "ubuntu-touch" ]; then + (cd "binary/$INITFS/custom.dir/" && tar -c *) | \ + gzip -9 --rsyncable > "$PREFIX.custom.tar.gz" + chmod 644 "$PREFIX.custom.tar.gZ" +fi + # '--initramfs none' produces different manifest names. if [ -e "binary/$INITFS/filesystem.packages" ]; then ln "binary/$INITFS/filesystem.packages" "$PREFIX.manifest" diff --git a/live-build/ubuntu-touch/hooks/60-install-click.binary b/live-build/ubuntu-touch/hooks/60-install-click.binary new file mode 100755 index 00000000..c57b372e --- /dev/null +++ b/live-build/ubuntu-touch/hooks/60-install-click.binary @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +echo "Splitting out custom tarball" + +rm -f binary/boot/custom.dir +mkdir binary/boot/custom.dir +mv binary/boot/filesystem.dir/custom binary/boot/custom.dir/custom diff --git a/live-build/ubuntu-touch/hooks/60-install-click.chroot b/live-build/ubuntu-touch/hooks/60-install-click.chroot index 0b4b2b7d..92537c3b 100755 --- a/live-build/ubuntu-touch/hooks/60-install-click.chroot +++ b/live-build/ubuntu-touch/hooks/60-install-click.chroot @@ -7,10 +7,15 @@ echo "Setting up click packages" click_uri=http://archive-team.internal/click_packages click_list=$click_uri/click_list click_db=/usr/share/click/preinstalled +click_db_custom=/custom/click mkdir -p -m 755 "$click_db" chown clickpkg:clickpkg "$click_db" +# some of these get installed to /custom/click +mkdir -p -m 755 "$click_db_custom" +chown clickpkg:clickpkg "$click_db_custom" + tmpdir="$(mktemp -d)" cleanup () { rm -rf "$tmpdir"; } trap cleanup EXIT @@ -23,7 +28,26 @@ do if echo $package | egrep -q "_$CLICKARCH.click|_all.click|_unknown.click"; then echo "Setting up $package" wget --no-verbose -O "$tmpdir/$package" "$click_uri/$package" - click install --force-missing-framework --root="$click_db" --all-users \ + # FIXME: first attempt, a hard-coded list of the packages that go to + # the custom tarball + case $package in + com.ubuntu.developer.webapps.webapp-amazon_*|\ + com.ubuntu.dropping-letters_*|\ + com.ubuntu.developer.webapps.webapp-ebay_*|\ + com.ubuntu.filemanager_*|\ + com.ubuntu.developer.webapps.webapp-gmail_*|\ + com.ubuntu.reminders_*|\ + com.ubuntu.sudoku_*|\ + com.ubuntu.terminal_*|\ + com.ubuntu.developer.webapps.webapp-facebook_*|\ + com.ubuntu.developer.webapps.webapp-twitter_*) + root="$click_db_custom" + ;; + *) + root="$click_db" + ;; + esac + click install --force-missing-framework --root="$root" --all-users \ "$tmpdir/$package" fi done diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index 36779203..8d92cea5 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -7,3 +7,7 @@ echo "I: precompiling click apparmor policies" echo "I: precompiling deb apparmor policies" /sbin/apparmor_parser -M ${FEATURES} -Q --write-cache --cache-loc=/etc/apparmor.d/cache/ `find /etc/apparmor.d/ -maxdepth 1 -type f -not -path '*/\.*'` + +echo "I: precompiling custom click apparmor policies" +mkdir -p /custom/cache/apparmor +/sbin/apparmor_parser -M ${FEATURES} -Q --write-cache --cache-loc=/custom/cache/apparmor/ `find /var/lib/apparmor/profiles/ -maxdepth 1 -type f -not -path '*/\.*'` From b423980cd592b8b78c01d9258fd065536e778665 Mon Sep 17 00:00:00 2001 From: Steve Langasek Date: Fri, 10 Oct 2014 07:18:52 +0000 Subject: [PATCH 02/12] Make sure the apparmor profiles and manifests end up in the custom directory, not in the rootfs --- .../hooks/90-precompile-apparmor-policies.chroot | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index 8d92cea5..0b53e2f7 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -11,3 +11,18 @@ echo "I: precompiling deb apparmor policies" echo "I: precompiling custom click apparmor policies" mkdir -p /custom/cache/apparmor /sbin/apparmor_parser -M ${FEATURES} -Q --write-cache --cache-loc=/custom/cache/apparmor/ `find /var/lib/apparmor/profiles/ -maxdepth 1 -type f -not -path '*/\.*'` + + +#get the apparmor manifests and profiles +mkdir -p /custom/lib/apparmor/clicks +mkdir -p /custom/lib/apparmor/profiles + +for manifest in /var/lib/apparmor/clicks/*; do + case $(readlink -f $manifest) in + /custom/*) + mv $manifest /custom/lib/apparmor/clicks/$manifest + profile=$(basename $manifest .json) + mv /var/lib/apparmor/profiles/click_$profile /custom/lib/apparmor/profiles + ;; + esac +done From 23f92bce16afdd3f595c8e92c1a4515604b38ce3 Mon Sep 17 00:00:00 2001 From: Steve Langasek Date: Fri, 10 Oct 2014 07:26:06 +0000 Subject: [PATCH 03/12] Whoops, fix the directory inclusion --- .../ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index 0b53e2f7..9bdb1fd7 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -20,7 +20,7 @@ mkdir -p /custom/lib/apparmor/profiles for manifest in /var/lib/apparmor/clicks/*; do case $(readlink -f $manifest) in /custom/*) - mv $manifest /custom/lib/apparmor/clicks/$manifest + mv $manifest /custom/lib/apparmor/clicks/$(basename $manifest) profile=$(basename $manifest .json) mv /var/lib/apparmor/profiles/click_$profile /custom/lib/apparmor/profiles ;; From b6d1950e37e9f74ddf3429ef710514fdbab94e9b Mon Sep 17 00:00:00 2001 From: Steve Langasek Date: Fri, 10 Oct 2014 07:27:42 +0000 Subject: [PATCH 04/12] Fix a typo --- live-build/auto/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/auto/build b/live-build/auto/build index c1036ec1..0d79b5c7 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -301,7 +301,7 @@ fi if [ "$PROJECT" = "ubuntu-touch" ]; then (cd "binary/$INITFS/custom.dir/" && tar -c *) | \ gzip -9 --rsyncable > "$PREFIX.custom.tar.gz" - chmod 644 "$PREFIX.custom.tar.gZ" + chmod 644 "$PREFIX.custom.tar.gz" fi # '--initramfs none' produces different manifest names. From 5407721469be1ba635ce8c4d12ecd717f67b7c35 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 12:44:01 +0100 Subject: [PATCH 05/12] sort by package name --- live-build/ubuntu-touch/hooks/60-install-click.chroot | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/live-build/ubuntu-touch/hooks/60-install-click.chroot b/live-build/ubuntu-touch/hooks/60-install-click.chroot index 92537c3b..7c005714 100755 --- a/live-build/ubuntu-touch/hooks/60-install-click.chroot +++ b/live-build/ubuntu-touch/hooks/60-install-click.chroot @@ -32,15 +32,15 @@ do # the custom tarball case $package in com.ubuntu.developer.webapps.webapp-amazon_*|\ - com.ubuntu.dropping-letters_*|\ com.ubuntu.developer.webapps.webapp-ebay_*|\ - com.ubuntu.filemanager_*|\ + com.ubuntu.developer.webapps.webapp-facebook_*|\ com.ubuntu.developer.webapps.webapp-gmail_*|\ + com.ubuntu.developer.webapps.webapp-twitter_*|\ + com.ubuntu.dropping-letters_*|\ + com.ubuntu.filemanager_*|\ com.ubuntu.reminders_*|\ com.ubuntu.sudoku_*|\ - com.ubuntu.terminal_*|\ - com.ubuntu.developer.webapps.webapp-facebook_*|\ - com.ubuntu.developer.webapps.webapp-twitter_*) + com.ubuntu.terminal_*) root="$click_db_custom" ;; *) From a10e22f77b2baba1f1279861a727cd79dad8c309 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 12:48:04 +0100 Subject: [PATCH 06/12] Temporarily install the to-be-moved click packages in both core and custom databases, to simplify the landing process. --- debian/changelog | 5 +++++ live-build/ubuntu-touch/hooks/60-install-click.chroot | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index 2bccc901..39681a63 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,13 @@ livecd-rootfs (2.255) UNRELEASED; urgency=medium + [ Steve Langasek ] * Add support to the ubuntu-touch target to generate a separate custom tarball as part of the build, moving most click packages into it. + [ Colin Watson ] + * Temporarily install the to-be-moved click packages in both core and + custom databases, to simplify the landing process. + -- Steve Langasek Thu, 09 Oct 2014 20:46:46 -0700 livecd-rootfs (2.254) utopic; urgency=medium diff --git a/live-build/ubuntu-touch/hooks/60-install-click.chroot b/live-build/ubuntu-touch/hooks/60-install-click.chroot index 7c005714..daae6a14 100755 --- a/live-build/ubuntu-touch/hooks/60-install-click.chroot +++ b/live-build/ubuntu-touch/hooks/60-install-click.chroot @@ -41,13 +41,15 @@ do com.ubuntu.reminders_*|\ com.ubuntu.sudoku_*|\ com.ubuntu.terminal_*) - root="$click_db_custom" + roots="$click_db $click_db_custom" ;; *) - root="$click_db" + roots="$click_db" ;; esac - click install --force-missing-framework --root="$root" --all-users \ - "$tmpdir/$package" + for root in $roots; do + click install --force-missing-framework --root="$root" --all-users \ + "$tmpdir/$package" + done fi done From 5d707813f0a711ca58e3184d654c43bd1a292564 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 14:22:41 +0100 Subject: [PATCH 07/12] Move default click database configuration aside while installing click packages, otherwise click gets clever and stops us installing things into multiple databases. --- live-build/ubuntu-touch/hooks/60-install-click.chroot | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/live-build/ubuntu-touch/hooks/60-install-click.chroot b/live-build/ubuntu-touch/hooks/60-install-click.chroot index daae6a14..626aad3b 100755 --- a/live-build/ubuntu-touch/hooks/60-install-click.chroot +++ b/live-build/ubuntu-touch/hooks/60-install-click.chroot @@ -16,8 +16,15 @@ chown clickpkg:clickpkg "$click_db" mkdir -p -m 755 "$click_db_custom" chown clickpkg:clickpkg "$click_db_custom" +# move default configuration aside to stop click being clever about symlink +# layering +mv /etc/click/databases /etc/click/databases.tmp + tmpdir="$(mktemp -d)" -cleanup () { rm -rf "$tmpdir"; } +cleanup () { + rm -rf "$tmpdir" + mv /etc/click/databases.tmp /etc/click/databases +} trap cleanup EXIT CLICKARCH=$(dpkg --print-architecture) From 95c970f52f393f4fa5460e83cddf2f8207ee94e9 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 16:29:52 +0100 Subject: [PATCH 08/12] Take a different approach to working around deduplication cleverness in click, this time hopefully without breaking hooks. --- .../hooks/60-install-click.chroot | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/live-build/ubuntu-touch/hooks/60-install-click.chroot b/live-build/ubuntu-touch/hooks/60-install-click.chroot index 626aad3b..d8be455b 100755 --- a/live-build/ubuntu-touch/hooks/60-install-click.chroot +++ b/live-build/ubuntu-touch/hooks/60-install-click.chroot @@ -16,15 +16,8 @@ chown clickpkg:clickpkg "$click_db" mkdir -p -m 755 "$click_db_custom" chown clickpkg:clickpkg "$click_db_custom" -# move default configuration aside to stop click being clever about symlink -# layering -mv /etc/click/databases /etc/click/databases.tmp - tmpdir="$(mktemp -d)" -cleanup () { - rm -rf "$tmpdir" - mv /etc/click/databases.tmp /etc/click/databases -} +cleanup () { rm -rf "$tmpdir"; } trap cleanup EXIT CLICKARCH=$(dpkg --print-architecture) @@ -55,8 +48,20 @@ do ;; esac for root in $roots; do + if [ "$root" = "$click_db_custom" ]; then + # FIXME: there is no good way to stop click from + # deduplicating things when installing the same package in + # multiple databases; the best we can do is to temporarily + # pretend that the core database does not exist + mv /etc/click/databases/10_core.conf \ + /etc/click/databases/10_core.conf.tmp + fi click install --force-missing-framework --root="$root" --all-users \ "$tmpdir/$package" + if [ "$root" = "$click_db_custom" ]; then + mv /etc/click/databases/10_core.conf.tmp \ + /etc/click/databases/10_core.conf + fi done fi done From 448c8f9fc0dda98c94c5464c3e482a55498dc6c7 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 16:51:48 +0100 Subject: [PATCH 09/12] Rearrange AppArmor policy precompilation to handle a click package being in multiple locations. --- .../90-precompile-apparmor-policies.chroot | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index 9bdb1fd7..4cd60c7b 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -18,11 +18,29 @@ mkdir -p /custom/lib/apparmor/clicks mkdir -p /custom/lib/apparmor/profiles for manifest in /var/lib/apparmor/clicks/*; do - case $(readlink -f $manifest) in - /custom/*) - mv $manifest /custom/lib/apparmor/clicks/$(basename $manifest) - profile=$(basename $manifest .json) - mv /var/lib/apparmor/profiles/click_$profile /custom/lib/apparmor/profiles - ;; - esac + # FIXME: if this code survives for very long, it should probably be + # rewritten using click's Python bindings + pkgdir="$(click pkgdir "$manifest")" + manifest_tail="${manifest#$pkgdir}" + # Does this package exist in the custom tarball? If so, move its + # profiles there (if it only exists in custom) or copy them (if it + # also exists in core). + version="${pkgdir##*/}" + pkgdir_noversion="${pkgdir%/*}" + name="${pkgdir_noversion##*/}" + if [ -d "/custom/click/$name/$version" ]; then + # Clone into custom. + ln -nsf "/custom/click/$name/$version$manifest_tail" "/custom/lib/apparmor/clicks$manifest_tail" + profile="$(basename "$manifest" .json)" + cp -a "/var/lib/apparmor/profiles/click_$profile" /custom/lib/apparmor/profiles/ + if [ -d "/usr/share/click/preinstalled/$name/$version" ]; then + # Ensure that the version in the rootfs points to + # the core database. + ln -nsf "/usr/share/click/preinstalled/$name/$version$manifest_tail" "$manifest" + else + # Remove from the rootfs. + rm -f "$manifest" + rm -f "/var/lib/apparmor/profiles/click_$profile" + fi + fi done From 68e78ecd529a41fdb5f8e00511045efb52c819c6 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 18:41:23 +0100 Subject: [PATCH 10/12] Fix manifest_tail computation. --- .../ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index 4cd60c7b..264324fe 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -21,7 +21,8 @@ for manifest in /var/lib/apparmor/clicks/*; do # FIXME: if this code survives for very long, it should probably be # rewritten using click's Python bindings pkgdir="$(click pkgdir "$manifest")" - manifest_tail="${manifest#$pkgdir}" + manifest_real="$(readlink -f "$manifest")" + manifest_tail="${manifest_real#$pkgdir}" # Does this package exist in the custom tarball? If so, move its # profiles there (if it only exists in custom) or copy them (if it # also exists in core). From f4589e7c31f96b2dee0633c5d0ea5f2f31138a17 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 20:08:50 +0100 Subject: [PATCH 11/12] Fix linking of manifest into /custom/lib/apparmor/clicks/. --- .../ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index 264324fe..f17a937b 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -31,7 +31,7 @@ for manifest in /var/lib/apparmor/clicks/*; do name="${pkgdir_noversion##*/}" if [ -d "/custom/click/$name/$version" ]; then # Clone into custom. - ln -nsf "/custom/click/$name/$version$manifest_tail" "/custom/lib/apparmor/clicks$manifest_tail" + ln -nsf "/custom/click/$name/$version$manifest_tail" "/custom/lib/apparmor/clicks/${manifest##*/}" profile="$(basename "$manifest" .json)" cp -a "/var/lib/apparmor/profiles/click_$profile" /custom/lib/apparmor/profiles/ if [ -d "/usr/share/click/preinstalled/$name/$version" ]; then From b41729958ac419db4b527804101d14d4fc826b40 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 13 Oct 2014 20:11:42 +0100 Subject: [PATCH 12/12] Clean up duplicate AppArmor cache entries between rootfs and custom. --- .../hooks/90-precompile-apparmor-policies.chroot | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot index f17a937b..664379b0 100755 --- a/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot +++ b/live-build/ubuntu-touch/hooks/90-precompile-apparmor-policies.chroot @@ -29,10 +29,10 @@ for manifest in /var/lib/apparmor/clicks/*; do version="${pkgdir##*/}" pkgdir_noversion="${pkgdir%/*}" name="${pkgdir_noversion##*/}" + profile="$(basename "$manifest" .json)" if [ -d "/custom/click/$name/$version" ]; then # Clone into custom. ln -nsf "/custom/click/$name/$version$manifest_tail" "/custom/lib/apparmor/clicks/${manifest##*/}" - profile="$(basename "$manifest" .json)" cp -a "/var/lib/apparmor/profiles/click_$profile" /custom/lib/apparmor/profiles/ if [ -d "/usr/share/click/preinstalled/$name/$version" ]; then # Ensure that the version in the rootfs points to @@ -42,6 +42,10 @@ for manifest in /var/lib/apparmor/clicks/*; do # Remove from the rootfs. rm -f "$manifest" rm -f "/var/lib/apparmor/profiles/click_$profile" + rm -f "/var/cache/apparmor/click_$profile" fi + else + # Remove from custom. + rm -f "/custom/cache/apparmor/click_$profile" fi done