Added sublayers support

Adds support for sublayers by defining package lists corresponding to
each sublayer. The name of the package list defines the name of the
layer.

live-build/auto/build

@@ -17,6 +17,57 @@ fi
 
 . config/functions
 
+build_layered_squashfs() {
+	local pass=$1   # install|install_subpass|install_subpass_subsubpass|…
+	local prevpass=$2  # install|install_subpass|…
+	local prefix=$3 # 01-|02-|…
+	local lowerlayers=$4
+
+	base="${PWD}/livecd.${PROJECT}.${prefix}${pass}"
+	squashfs_f="${base}.squashfs"
+
+	# We have already treated that pass
+	if [ -f "${squashfs_f}" ]; then
+		return
+	fi
+
+	if [ -n "${lowerlayers}" ]; then
+		mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/
+	else
+		# first pass
+		ln -s "chroot.${pass}/" chroot
+	fi
+
+	(cd "chroot.${pass}/" &&
+		mksquashfs . ${squashfs_f} \
+		-no-progress -xattrs -comp xz )
+
+	# Full manifest until that PASS
+	squashfs_f_manifest="${base}.manifest"
+	create_manifest "chroot" "${squashfs_f_manifest}.full"
+
+	# Delta manifest
+	diff -NU0 ${PWD}/livecd.${PROJECT}.${prefix}${prevpass}.manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
+
+	squashfs_f_size="${base}.size"
+	du -B 1 -s "chroot.${pass}/" | cut -f1 > "${squashfs_f_size}"
+
+	if [ -n "${lowerlayers}" ]; then
+		umount chroot
+	else
+		rm chroot
+		mkdir chroot/
+	fi
+
+	# Handle direct sublayer of current one
+	# Extract the name of the pass corresponding to the sublayer
+	for subpass in $(ls -d chroot.${pass}_* 2>/dev/null | sed -e "s/chroot\.\(${pass}_[^_]\+\).*/\1/"); do
+		lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}"
+		lowerlayers_for_subpass="${lowerlayers_for_subpass%:}"
+		build_layered_squashfs "${subpass}" "${pass}" "${prefix}" "${lowerlayers_for_subpass}"
+	done
+}
+
 # Link output files somewhere launchpad-buildd will be able to find them.
 PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
 
@@ -479,59 +530,31 @@ EOF
 
 	lb binary "$@"
 
+	# Teardown mount points in layered mode
+	if [ -x config/teardown_chroot_layered ]; then
+		config/teardown_chroot_layered
+	fi
+
 	# Create layered squashfs system
-	CURPASS=1
-	PREVPASS=""
-	LASTPASS=$(echo $PASSES|wc -w)
-	prev_manifest=""
-	to_remove=""
 	if [ "${IMAGEFORMAT:-}" = live-layered ]; then
+		CURPASS=1
+		PREVPASS=""
+		PREFIX=""
+		LOWER_LAYERS=""
+	
 		for _PASS in $PASSES
 		do
-			base="${PWD}/livecd.${PROJECT}.$(printf "%02g" $CURPASS)-${_PASS}"
+			PREFIX="$(printf "%02g" $CURPASS)-"
-			squashfs_f="${base}.squashfs"
+			build_layered_squashfs "${_PASS}" "${PREVPASS}" "$PREFIX" ${LOWER_LAYERS}
-
-			(cd "chroot.${_PASS}/" &&
-      			mksquashfs . ${squashfs_f} \
-        		-no-progress -xattrs -comp xz )
-
-			# Full manifest until that PASS
-			# Note: snaps won't be listed in the new layer by snap-seed-parse
-			# if no newly snap was installed in that layer. Copy and then remove it
-			# just to create the correct full manifest and avoid a new layer
-			# chroot mount dance.
-			SNAPPATH=var/lib/snapd/seed/seed.yaml
-			if [ ! -f "chroot.${_PASS}/${SNAPPATH}" ] && [ -n "${PREVPASS}" ] && [ -f "chroot.${PREVPASS}/${SNAPPATH}" ]; then
-				mkdir -p chroot.${_PASS}/$(dirname ${SNAPPATH})
-				cp "chroot.${PREVPASS}/${SNAPPATH}" "chroot.${_PASS}/${SNAPPATH}"
-				to_remove="chroot.${_PASS}/var/lib/snapd ${to_remove}"
-			fi
-			squashfs_f_manifest="${base}.manifest"
-			create_manifest "chroot.${_PASS}/" "${squashfs_f_manifest}.full"
-
-			# Delta manifest
-			diff -NU0 ${prev_manifest}.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
-			prev_manifest=$squashfs_f_manifest
-
-			# Full ISO manifest
-			if [ $CURPASS -eq $LASTPASS ]; then
-				cp "${squashfs_f_manifest}.full" livecd.${PROJECT}.manifest
-			fi
-
-			squashfs_f_size="${base}.size"
-			du -B 1 -s "chroot.${_PASS}/" | cut -f1 > "${squashfs_f_size}"
 
+			LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS"
+			LOWER_LAYERS="${LOWER_LAYERS%:}"
 			PREVPASS=${_PASS}
 			CURPASS=$(( CURPASS + 1 ))
 		done
-		if [ -n "${to_remove}" ]; then
-			rm -r ${to_remove}
-		fi
-	fi
 
-	# Teardown mount points in layered mode
+		# Full ISO manifest
-	if [ -x config/teardown_chroot_layered ]; then
+		cp "${PWD}/livecd.${PROJECT}.${PREFIX}${_PASS}.manifest.full" livecd.${PROJECT}.manifest
-		config/teardown_chroot_layered
 	fi
 
 	touch binary.success

live-build/lb_chroot_layered

@@ -81,18 +81,23 @@ lb_chroot_install_snaps() {
 	done < $snaplist_file
 }
 
-PASSES="${PASSES:-install live}"
+create_chroot_pass() {
-CURPASS=1
+	local pass=$1
-PREVPASS=""
+	local prevpass=$2
-LASTPASS=$(echo $PASSES|wc -w)
+	local lowerlayers=$3
-LOWER_LAYERS=""
+	local passtype=$4  # "first"|"last"|"" empty string
-for _PASS in $PASSES
+	shift 4 # restore ${*}
-do
+
-	export PASS=${_PASS}
+	# We have already treated that pass
+	if [ -d "chroot.${pass}/" ]; then
+		return
+	fi
 
-	if [ $CURPASS -gt 1 ]; then
+	export PASS=${pass}
-		mkdir chroot.${_PASS}
+
-		mount_overlay $LOWER_LAYERS "chroot.${_PASS}/" chroot/
+	if [ "${passtype}" != "first" ]; then
+		mkdir chroot.${pass}
+		mount_overlay ${lowerlayers} "chroot.${pass}/" chroot/
 	fi
 
 	# Configuring chroot
@@ -117,15 +122,15 @@ do
 	lb chroot_preseed ${*}
 	lb chroot_early_hooks ${*}
 
-	lb chroot_package-lists ${_PASS} ${*}
+	lb chroot_package-lists ${pass} ${*}
-	lb chroot_install-packages ${_PASS} ${*}
+	lb chroot_install-packages ${pass} ${*}
 
 	# Snap management
-	lb_chroot_snap_lists ${_PASS} ${PREVPASS}
+	lb_chroot_snap_lists ${pass} ${prevpass}
-	lb_chroot_install_snaps ${_PASS} ${*}
+	lb_chroot_install_snaps ${pass} ${*}
 
 	# Kernel should be in first layer
-	if [ $CURPASS -eq 1 ]; then
+	if [ "${passtype}" = "first" ]; then
 		Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \
 			| awk '/^i/ {print $2}' > chroot.headers
 		for i in $(cat chroot.headers); do
@@ -136,7 +141,7 @@ do
 	Chroot chroot "apt-get --purge -y autoremove"
 
 	# Add live packages to top layer
-	if [ $CURPASS -eq $LASTPASS ]; then
+	if [ "${passtype}" = "last" ]; then
 		lb chroot_live-packages ${*}
 	fi
 
@@ -145,7 +150,7 @@ do
 	lb chroot_hacks ${*}
 	lb chroot_interactive ${*}
 
-	Chroot chroot "dpkg-query -W" > chroot.packages.${_PASS}
+	Chroot chroot "dpkg-query -W" > chroot.packages.${pass}
 
 	# Deconfiguring chroot
 	lb chroot_archives chroot remove ${*}
@@ -164,13 +169,38 @@ do
 	lb chroot_devpts remove ${*}
 	lb chroot_cache save ${*}
 
-	if [ $CURPASS -eq 1 ]; then
+	if [ "${passtype}" = "first" ]; then
-		mv chroot chroot.${_PASS}
+		mv chroot chroot.${pass}
 		mkdir chroot
-	elif [ $CURPASS -ne $LASTPASS ]; then
+	else
-		# we will unmount chroot/ at teardown for upper layer
 		umount chroot
 	fi
+
+	# Handle direct sublayer of current one
+	# Extract the name of the pass corresponding to the sublayer
+	for subpass in $(ls config/package-lists/*.list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do
+		lowerlayers_for_subpass="chroot.${pass}:${lowerlayers}"
+		lowerlayers_for_subpass="${lowerlayers_for_subpass%:}"
+		create_chroot_pass "${subpass}" "${pass}" "${lowerlayers_for_subpass}" "" ${*}
+	done
+}
+
+PASSES="${PASSES:-install live}"
+CURPASS=1
+PREVPASS=""
+LASTPASS=$(echo $PASSES|wc -w)
+LOWER_LAYERS=""
+for _PASS in $PASSES
+do
+	PASSTYPE=""
+	if [ $CURPASS -eq 1 ]; then
+		PASSTYPE="first"
+	elif [ $CURPASS -eq $LASTPASS ]; then
+		PASSTYPE="last"
+	fi
+
+	create_chroot_pass "$_PASS" "$PREVPASS" "$LOWER_LAYERS" "$PASSTYPE" ${*}
+	
 	LOWER_LAYERS="chroot.${_PASS}:$LOWER_LAYERS"
 	LOWER_LAYERS="${LOWER_LAYERS%:}"
 	PREVPASS=${_PASS}
@@ -178,8 +208,10 @@ do
 	CURPASS=$(( CURPASS + 1 ))
 done
 
-# Remove unused chroot binary corresponding to bottom layer (once everything is unmount)
+# remount last pass on chroot for lb binary
+mount_overlay "${LOWER_LAYERS}" "chroot.${_PASS}/" chroot/
 
+# Remove unused chroot binary corresponding to bottom layer (once everything is unmount)
 TEARDOWNPATH="config/teardown_chroot_layered"
 cat > $TEARDOWNPATH << EOF
 #!/bin/sh