From a40cfd61809fc8019fdd8e2caa830afe4b0dbfc2 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 14:45:07 -0700 Subject: [PATCH 01/15] ubuntu-cpc: Move apt-get to the top of uefi disk creation --- live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary index ccc6cca9..e0e90c18 100755 --- a/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary +++ b/live-build/ubuntu-cpc/hooks/033-disk-image-uefi.binary @@ -12,9 +12,10 @@ esac . /build/config/functions +apt-get -qqy install dosfstools gdisk + create_partitions() { disk_image="$1" - apt-get install -qqy gdisk sgdisk "${disk_image}" --zap-all case $architecture in arm64) @@ -39,7 +40,6 @@ create_partitions() { create_and_mount_uefi_partition() { uefi_dev="/dev/mapper${loop_device///dev/}p15" - apt-get -qqy install dosfstools mountpoint="$1" mkfs.vfat -F 32 -n UEFI "${uefi_dev}" From e4553137acba9bbd475920dff49194c0fca2f630 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 14:46:00 -0700 Subject: [PATCH 02/15] ubuntu-cpc: generate manifest for root tarball --- live-build/ubuntu-cpc/hooks/031-root-xz.binary | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/live-build/ubuntu-cpc/hooks/031-root-xz.binary b/live-build/ubuntu-cpc/hooks/031-root-xz.binary index b2503adb..a3ee963d 100755 --- a/live-build/ubuntu-cpc/hooks/031-root-xz.binary +++ b/live-build/ubuntu-cpc/hooks/031-root-xz.binary @@ -1,3 +1,9 @@ #!/bin/bash -ex +# vi: ts=4 noexpandtab +# +# Generate the rootfs.tar.gz and manifest + +dpkg-query --admindir=binary/boot/filesystem.dir/var/lib/dpkg -W > livecd.ubuntu-cpc.rootfs.manifest + (cd "binary/boot/filesystem.dir/" && tar -c *) | \ xz > "livecd.ubuntu-cpc.rootfs.tar.xz" From bc5dbaaca51486e22e3316c87db7c869db89bc97 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 14:48:01 -0700 Subject: [PATCH 03/15] ubuntu-cpc: generate squashfs based on the rootfs tarball --- .../ubuntu-cpc/hooks/032-root-squashfs.binary | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 live-build/ubuntu-cpc/hooks/032-root-squashfs.binary diff --git a/live-build/ubuntu-cpc/hooks/032-root-squashfs.binary b/live-build/ubuntu-cpc/hooks/032-root-squashfs.binary new file mode 100755 index 00000000..37d612f9 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/032-root-squashfs.binary @@ -0,0 +1,15 @@ +#!/bin/bash -eux +# vi: ts=4 noexpandtab +# +# Generate a squashfs root and manifest + +apt-get -qqy install squashfs-tools + +squashfs_f="${PWD}/livecd.ubuntu-cpc.squashfs" +squashfs_f_manifest="${squashfs_f}.manifest" + +dpkg-query --admindir=binary/boot/filesystem.dir/var/lib/dpkg -W > ${squashfs_f_manifest} + +(cd "binary/boot/filesystem.dir/" && + mksquashfs . ${squashfs_f} \ + -no-progress -xattrs -comp xz ) From b31233589e3eed47c5832fea2e94c06aebb1a440 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 14:48:41 -0700 Subject: [PATCH 04/15] ubuntu-cpc: generate VMDK disks --- .../ubuntu-cpc/hooks/040-vmdk-image.binary | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 live-build/ubuntu-cpc/hooks/040-vmdk-image.binary diff --git a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary new file mode 100755 index 00000000..51d91768 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary @@ -0,0 +1,83 @@ +#!/bin/bash -eux +# vi: ts=4 noexpandtab +# +# Generate VMDK files +# +# There is no real good way to create a _compressed_ VMDK using open source +# tooling that works across multiple VMDK-capable platforms. This hook uses +# vmdk-stream-converter and then modifies the header manually to produce a +# compatiable VMDK. + +architecture=$(chroot chroot dpkg --print-architecture) + +extension="disk1.vmdk" +case ${architecture} in + i386) image_target="binary/boot/disk.ext4";; + amd64) image_target="binary/boot/disk-uefi.ext4"; extension="uefi1.vmdk";; + *) echo "VMDK images are not supported for ${architecture} yet."; + exit 0;; +esac + +apt-get install -qqy qemu-utils vmdk-stream-converter + +streamconverter="/usr/share/pyshared/VMDKstream.py" + +# Lets be safe about this +scratch_d=$(mktemp -d) +trap "rm -rf ${scratch_d}" EXIT + +modify_vmdk_header() { + # Modify the VMDK headers so that both VirtualBox _and_ VMware can + # read the vmdk and import them. The vodoo here is _not_ documented + # anywhere....so this will have to do. + + vmdk_name="${1}" + + # Extract the vmdk header for manipulation + dd if="${vmdk_name}" of=descriptor.txt bs=1 skip=512 count=1024 + + # The sed lines below is where the magic is. Specifically: + # ddb.toolsVersion: sets the open-vm-tools so that VMware shows + # the tooling as current + # ddb.virtualHWVersion: set the version to 7, which covers most + # current versions of VMware + # createType: make sure its set to stream Optimized + # remove the vmdk-stream-converter comment and replace with + # # Disk DescriptorFile. This is needed for Virtualbox + # remove the comments from vmdk-stream-converter which causes + # VirtualBox and others to fail VMDK validation + + sed -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ + -e 's|ddb.virtualHWVersion.*|ddb.virtualHWVersion = "7"|' \ + -e 's|createType.*|createType="streamOptimized"|' \ + -e 's|# Description file.*|# Disk DescriptorFile|' \ + -e '/# Believe.*/d' \ + -e '/# Indicates no parent/d' \ + descriptor.txt > new_descriptor.txt + + # The header is cannot be bigger than 1024 + expr $(stat --format=%s new_descriptor.txt) \< 1024 || { + echo "descriptor is too large, VMDK will be invalid!"; exit 1; } + + # Overwrite the vmdk header with our new, modified one + dd conv=notrunc,nocreat \ + if=new_descriptor.txt of="${vmdk_name}" \ + bs=1 seek=512 count=1024 +} + +convert_image() { + src="$1" + destination="$2" + + cp ${src} ${scratch_d}/resize.img + qemu-img resize ${scratch_d}/resize.img 10G + python ${streamconverter} ${scratch_d}/resize.img "${destination}" + modify_vmdk_header "${destination}" + qemu-img info "${destination}" +} + +convert_image binary/boot/disk.ext4 livecd.ubuntu-cpc.disk1.vmdk + +if [ -e binary/boot/disk-uefi.ext4 ]; then + convert_image binary/boot/disk-uefi.ext4 livecd.ubuntu-cpc.uefi.vmdk +fi From 92f63b3e6bd57d54ecb8fd199b7ab9c6c5677bf2 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 14:49:25 -0700 Subject: [PATCH 05/15] ubuntu-cpc: added OVF templates for generating OVA and Vagrant Images --- .../ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl | 179 ++++++++++++++++++ .../hooks/ovf/ubuntu-ova-v1-vmdk.tmpl | 177 +++++++++++++++++ .../hooks/ovf/ubuntu-ovf-v1-img.tmpl | 130 +++++++++++++ 3 files changed, 486 insertions(+) create mode 100644 live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl create mode 100644 live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-vmdk.tmpl create mode 100644 live-build/ubuntu-cpc/hooks/ovf/ubuntu-ovf-v1-img.tmpl diff --git a/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl b/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl new file mode 100644 index 00000000..f91b75b2 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-cloudcfg-vmdk.tmpl @@ -0,0 +1,179 @@ + + + + + + + + Virtual disk information + + + + + The list of logical networks + + The VM Network network + + + + A virtual machine + @@NAME@@ + + The kind of installed guest operating system + + + + Cloud-Init customization + Ubuntu @@VERSION@@ Server (@@DATE@@) + + + Specifies the instance id. This is required and used to determine if the machine should take "first boot" actions + + + Specifies the hostname for the appliance + + + + This field is optional, but indicates that the instance should 'seed' user-data and meta-data from the given url. If set to 'http://tinyurl.com/sm-' is given, meta-data will be pulled from http://tinyurl.com/sm-meta-data and user-data from http://tinyurl.com/sm-user-data. Leave this empty if you do not want to seed from a url. + + + + This field is optional, but indicates that the instance should populate the default user's 'authorized_keys' with this value + + + + In order to fit into a xml attribute, this value is base64 encoded . It will be decoded, and then processed normally as user-data. + + + + + If set, the default user's password will be set to this value to allow password based login. The password will be good for only a single login. If set to the string 'RANDOM' then a random password will be generated, and written to the console. + + + + + Virtual hardware requirements + + Virtual Hardware Family + 0 + @@NAME@@ + vmx-10 + + + hertz * 10^6 + Number of Virtual CPUs + @@NUM_CPUS@@ virtual CPU(s) + 1 + 3 + @@NUM_CPUS@@ + + + byte * 2^20 + Memory Size + @@MEM_SIZE@@MB of memory + 2 + 4 + @@MEM_SIZE@@ + + + 0 + SCSI Controller + SCSI Controller 0 + 3 + VirtualSCSI + 6 + + + 1 + IDE Controller + VirtualIDEController 1 + 4 + 5 + + + 0 + IDE Controller + VirtualIDEController 0 + 5 + 5 + + + false + VirtualVideoCard + 6 + 24 + + + + + + + + false + VirtualVMCIDevice + 7 + vmware.vmci + 1 + + + + 0 + false + CD-ROM 1 + 8 + 4 + vmware.cdrom.remotepassthrough + 15 + + + + 0 + Hard Disk 1 + ovf:/disk/vmdisk1 + 9 + 3 + 17 + + + + 1 + ConfigDriveDisk + ovf:/disk/vmdisk2 + 10 + 3 + 17 + + + + 7 + true + VM Network + VmxNet3 ethernet adapter on "VM Network" + Ethernet 1 + 11 + VmxNet3 + 10 + + + + + + + + + + + + + + + + + + + + + + diff --git a/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-vmdk.tmpl b/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-vmdk.tmpl new file mode 100644 index 00000000..0eaa85a4 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ova-v1-vmdk.tmpl @@ -0,0 +1,177 @@ + + + + + + + Virtual disk information + + + + The list of logical networks + + The VM Network network + + + + A virtual machine + @@NAME@@ + + The kind of installed guest operating system + + + + Cloud-Init customization + Ubuntu @@VERSION@@ Server (@@DATE@@) + + + Specifies the instance id. This is required and used to determine if the machine should take "first boot" actions + + + Specifies the hostname for the appliance + + + + This field is optional, but indicates that the instance should 'seed' user-data and meta-data from the given url. If set to 'http://tinyurl.com/sm-' is given, meta-data will be pulled from http://tinyurl.com/sm-meta-data and user-data from http://tinyurl.com/sm-user-data. Leave this empty if you do not want to seed from a url. + + + + This field is optional, but indicates that the instance should populate the default user's 'authorized_keys' with this value + + + + In order to fit into a xml attribute, this value is base64 encoded . It will be decoded, and then processed normally as user-data. + + + + + If set, the default user's password will be set to this value to allow password based login. The password will be good for only a single login. If set to the string 'RANDOM' then a random password will be generated, and written to the console. + + + + + Virtual hardware requirements + + Virtual Hardware Family + 0 + @@NAME@@ + vmx-10 + + + hertz * 10^6 + Number of Virtual CPUs + @@NUM_CPUS@@ virtual CPU(s) + 1 + 3 + @@NUM_CPUS@@ + + + byte * 2^20 + Memory Size + @@MEM_SIZE@@MB of memory + 2 + 4 + @@MEM_SIZE@@ + + + 0 + SCSI Controller + SCSI Controller 0 + 3 + VirtualSCSI + 6 + + + 1 + IDE Controller + VirtualIDEController 1 + 4 + 5 + + + 0 + IDE Controller + VirtualIDEController 0 + 5 + 5 + + + false + VirtualVideoCard + 6 + 24 + + + + + + + + false + VirtualVMCIDevice + 7 + vmware.vmci + 1 + + + + 0 + false + CD-ROM 1 + 8 + 4 + vmware.cdrom.remotepassthrough + 15 + + + + 0 + Hard Disk 1 + ovf:/disk/vmdisk1 + 9 + 3 + 17 + + + + 0 + false + Floppy Drive + Floppy 1 + 10 + vmware.floppy.remotedevice + 14 + + + 7 + true + VM Network + VmxNet3 ethernet adapter on "VM Network" + Ethernet 1 + 11 + VmxNet3 + 10 + + + + + + + + + + + + + + + + + + + + + + diff --git a/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ovf-v1-img.tmpl b/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ovf-v1-img.tmpl new file mode 100644 index 00000000..161b8f61 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/ovf/ubuntu-ovf-v1-img.tmpl @@ -0,0 +1,130 @@ + + + + + + + Virtual disk information + + + + The list of logical networks + + The bridged network + + + + A virtual machine + $VIRTUAL_SYSTEM_NAME + + $VIRTUAL_SYSTEM_OS_INFO + + + Cloud-Init customization + $VIRTUAL_SYSTEM_OS_INFO + + + Specifies the instance id. This is required and used to determine if the machine should take "first boot" actions + + + Specifies the hostname for the appliance + + + + This field is optional, but indicates that the instance should 'seed' user-data and meta-data from the given url. If set to 'http://tinyurl.com/sm-' is given, meta-data will be pulled from http://tinyurl.com/sm-meta-data and user-data from http://tinyurl.com/sm-user-data. Leave this empty if you do not want to seed from a url. + + + + This field is optional, but indicates that the instance should populate the default user's 'authorized_keys' with this value + + + + In order to fit into a xml attribute, this value is base64 encoded . It will be decoded, and then processed normally as user-data. + + + + + If set, the default user's password will be set to this value to allow password based login. The password will be good for only a single login. If set to the string 'RANDOM' then a random password will be generated, and written to the console. + + + + Virtual hardware requirements + + Virtual Hardware Family + 0 + $VIRTUAL_SYSTEM_IDENTIFIER + $VIRTUAL_SYSTEM_TYPES + + + hertz * 10^6 + Number of Virtual CPUs + 1 virtual CPU(s) + 1 + 3 + 1 + + + byte * 2^20 + Memory Size + 256MB of memory + 2 + 4 + 256 + + + 0 + USB Controller + usb + 3 + 23 + + + 0 + SCSI Controller + scsiController0 + 4 + lsilogic + 6 + + + 1 + IDE Controller + ideController1 + 5 + 5 + + + 0 + false + cdrom1 + 6 + 5 + 15 + + + 0 + disk1 + ovf:/disk/vmdisk1 + 7 + 4 + 17 + + + 2 + true + bridged + ethernet adapter on "bridged" + ethernet0 + 8 + E1000 + 10 + + + + $ANNOTATION_INFO + $ANNOTATION + + + From e96df0c156b887d4345de249995ee5283a5fd574 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 14:50:00 -0700 Subject: [PATCH 06/15] ubuntu-cpc: generate OVA images. --- .../hooks/041-vmdk-ova-image.binary | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary diff --git a/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary b/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary new file mode 100755 index 00000000..895d3b27 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary @@ -0,0 +1,88 @@ +#!/bin/bash -eux +# vi: ts=4 noexpandtab +# +# Generate OVA images +# +# OVA images are, by defintiion a tarball consisting of a disk image, OVF file +# and checksums. This step produces an OVA that is suitable for use with +# Cloud's that support the OVF specification. +# +# For this step, we re-use the VMDK's made in 040-vmdk-image.binary + +cur_d=${PWD} +my_d=$(dirname $(readlink -f ${0})) + +architecture=$(chroot chroot dpkg --print-architecture) + +base_vmdk="livecd.ubuntu-cpc.disk1.vmdk" +case ${architecture} in + amd64) base_vmdk="livecd.ubuntu-cpc.uefi.vmdk";; + *) echo "OVA images are not supported for ${architecture} yet."; + exit 0;; +esac + +if [ ! -e ${base_vmdk} ]; then + find . | grep vmdk + exit 0 +fi + +# Lets be safe about this +scratch_d=$(mktemp -d) +trap "rm -rf ${scratch_d}" EXIT + +# Used to identify bits +suite=$(chroot chroot lsb_release -c -s) +version=$(chroot chroot lsb_release --release --short) +distro=$(chroot chroot lsb_release --id --short | tr [:upper:] [:lower:]) + +# Put our vmdk in place for OVA conversion +prefix="${distro}-${suite}-${version}-cloudimg" +vmdk_f="${scratch_d}/${prefix}.vmdk" +cp ${base_vmdk} ${vmdk_f} + +# Get information about the VMDK +vmdk_size=$(du -b "${vmdk_f}" | cut -f1) +vmdk_capacity=$(qemu-img info "${vmdk_f}" | awk '-F[\( ]' '$1 ~ /virtual/ && $NF ~ /bytes.*/ {print$(NF-1)}') + +# Populate the OVF template +ovf="${scratch_d}/${prefix}.ovf" +cp ${my_d}/ovf/ubuntu-ova-v1-vmdk.tmpl ${ovf} +serial_stamp=$(date +%Y%m%d) +sed -i "${ovf}" \ + -e "s/@@NAME@@/${prefix}-${serial_stamp}/g" \ + -e "s/@@FILENAME@@/${vmdk_f##*/}/g" \ + -e "s/@@VMDK_FILE_SIZE@@/${vmdk_size}/g" \ + -e "s/@@VMDK_CAPACITY@@/${vmdk_capacity}/g" \ + -e "s/@@NUM_CPUS@@/2/g" \ + -e "s/@@VERSION@@/${version}/g" \ + -e "s/@@DATE@@/${serial_stamp}/g" \ + -e "s/@@MEM_SIZE@@/1024/g" + +# Get the checksums +vmdk_sha256=$(sha256sum ${vmdk_f} | cut -d' ' -f1) +ovf_sha256=$(sha256sum ${ovf} | cut -d' ' -f1) + +# Generate the manifest +manifest="${scratch_d}/${prefix}.mf" +cat > "${manifest}" < Date: Thu, 12 Nov 2015 14:51:48 -0700 Subject: [PATCH 07/15] ubuntu-cpc: generate Vagrant images --- .../ubuntu-cpc/hooks/042-vagrant.binary | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100755 live-build/ubuntu-cpc/hooks/042-vagrant.binary diff --git a/live-build/ubuntu-cpc/hooks/042-vagrant.binary b/live-build/ubuntu-cpc/hooks/042-vagrant.binary new file mode 100755 index 00000000..6c185881 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/042-vagrant.binary @@ -0,0 +1,185 @@ +#!/bin/bash -eux +# vi: ts=4 noexpandtab +# +# Generate a generic Vagrant Box. +# +# Vagrant images are essentially nothing more than OVA's with extra-metadata. +# +# We can't use the OVA's for Vagrant since Vagrant uses SSH to modify the instance. +# This build step creates a cloud-config ISO so that Cloud-Init will configure +# the initial user, creates meta-data that tells Vagrant how to interact with +# the cloud-init created users, and finally create the OVA. +# +# For this step, we re-use the VMDK's made in 040-vmdk-image.binary + +cur_d=${PWD} +my_d=$(dirname $(readlink -f ${0})) + +architecture=$(chroot chroot dpkg --print-architecture) +base_vmdk="livecd.ubuntu-cpc.disk1.vmdk" +case ${architecture} in + amd64) base_vmdk="livecd.ubuntu-cpc.uefi.vmdk";; + *) echo "OVA images are not supported for ${architecture} yet."; + exit 0;; +esac + +if [ ! -e ${base_vmdk} ]; then + find . | grep vmdk + exit 0 +fi + +# Virtualbox is needed for making a small VMDK +apt-get -qqy install genisoimage qemu-utils + +# Lets be safe about this +box_d=$(mktemp -d) +seed_d=$(mktemp -d) +trap "rm -rf ${box_d} ${seed_d}" EXIT + +# Used to identify bits +suite=$(chroot chroot lsb_release -c -s) +version=$(chroot chroot lsb_release --release --short) +distro=$(chroot chroot lsb_release --id --short | tr [:upper:] [:lower:]) + +# Get the VMDK in place +prefix="${distro}-${suite}-${version}-cloudimg" +vmdk_f="${box_d}/${prefix}.vmdk" +cp ${base_vmdk} ${vmdk_f} + +# Vagrant needs a base user. We either inject the well-known SSH key +# or use password authentication. Both are ugly. So we'll use a password +# and make it random. This obviously is insecure...but at least its +# better than the alternatives. +ubuntu_user_pass=$(openssl rand -hex 12) + +#################################### +# Create the ConfigDrive +# This is a cloud-init piece that instructs cloud-init to configure +# a default user at first boot. + +cdrom_vmdk_f="${box_d}/${prefix}-configdrive.vmdk" + +# Create the user-data. This is totally insecure, but so is Vagrant. To +# mitigate this insecurity, the vagrant instance is not accessible +# except via local host. +cat > ${seed_d}/user-data < ${seed_d}/meta-data < ${box_d}/Vagrantfile < ${box_d}/metadata.json < "${manifest}" < Date: Thu, 12 Nov 2015 14:58:28 -0700 Subject: [PATCH 08/15] ubuntu-cpc: add ability add arbitrary hooks for extending builds --- live-build/ubuntu-cpc/hooks/999-extras.binary | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 live-build/ubuntu-cpc/hooks/999-extras.binary diff --git a/live-build/ubuntu-cpc/hooks/999-extras.binary b/live-build/ubuntu-cpc/hooks/999-extras.binary new file mode 100755 index 00000000..4e6c7e81 --- /dev/null +++ b/live-build/ubuntu-cpc/hooks/999-extras.binary @@ -0,0 +1,15 @@ +#!/bin/bash +# Execute extra binary hooks. + +my_dir=$(dirname $(readlink -f ${0})) +extra_d=${my_dir}/extra + +if [ ! -d ${my_dir}/extra ]; then + exit 0 +fi + +# Execute extra binary hooks +for recipe in $(find ${extra_d} -type f -executable); +do + bash ${recipe} +done From dc9da068dce3f33d34b6f562f56a423664baf04b Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 15:04:18 -0700 Subject: [PATCH 09/15] ubuntu-cpc: extended hooks to build additional cloud image targets including VMDKs, OVA, and Vagrant images. --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index fed3b79d..0a618b11 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +livecd-rootfs (2.354) xenial; urgency=medium + + * ubuntu-cpc: extended hooks to build additional cloud image targets + including VMDKs, OVA, and Vagrant images. + + -- Ben Howard Thu, 12 Nov 2015 15:00:36 -0700 + livecd-rootfs (2.353) xenial; urgency=medium * drop input group from snappy images, adjust md5 sums in From e931fbeaa5ac32ef5f73ae833f7649cd44aee845 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 17:21:33 -0700 Subject: [PATCH 10/15] ubuntu-cpc: move vmdk creation code into common funtion to support Vagrant Box builds --- live-build/ubuntu-cpc/functions | 63 +++++++++++++++++++ .../ubuntu-cpc/hooks/040-vmdk-image.binary | 62 +----------------- .../ubuntu-cpc/hooks/042-vagrant.binary | 5 +- 3 files changed, 70 insertions(+), 60 deletions(-) diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 3c082ff2..513cc9e8 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -59,3 +59,66 @@ umount_partition() { umount "$mountpoint/dev" umount "$mountpoint" } + +modify_vmdk_header() { + # Modify the VMDK headers so that both VirtualBox _and_ VMware can + # read the vmdk and import them. The vodoo here is _not_ documented + # anywhere....so this will have to do. + + vmdk_name="${1}" + descriptor=$(mktemp) + newdescriptor=$(mktemp) + + # Extract the vmdk header for manipulation + dd if="${vmdk_name}" of="${descriptor}" bs=1 skip=512 count=1024 + + # The sed lines below is where the magic is. Specifically: + # ddb.toolsVersion: sets the open-vm-tools so that VMware shows + # the tooling as current + # ddb.virtualHWVersion: set the version to 7, which covers most + # current versions of VMware + # createType: make sure its set to stream Optimized + # remove the vmdk-stream-converter comment and replace with + # # Disk DescriptorFile. This is needed for Virtualbox + # remove the comments from vmdk-stream-converter which causes + # VirtualBox and others to fail VMDK validation + + sed -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ + -e 's|ddb.virtualHWVersion.*|ddb.virtualHWVersion = "7"|' \ + -e 's|createType.*|createType="streamOptimized"|' \ + -e 's|# Description file.*|# Disk DescriptorFile|' \ + -e '/# Believe.*/d' \ + -e '/# Indicates no parent/d' \ + "${descriptor}" > "${new_descriptor}" + + # The header is cannot be bigger than 1024 + expr $(stat --format=%s ${new_descriptor}) \< 1024 || { + echo "descriptor is too large, VMDK will be invalid!"; exit 1; } + + # Overwrite the vmdk header with our new, modified one + dd conv=notrunc,nocreat \ + if="${new_descriptor}" of="${vmdk_name}" \ + bs=1 seek=512 count=1024 + + rm ${descriptor} ${new_descriptor} +} + +create_vmdk() { + src="$1" + destination="$2" + size="${3:-102400}" + + apt-get install -qqy qemu-utils vmdk-stream-converter + streamconverter="/usr/share/pyshared/VMDKstream.py" + scratch_d=$(mktemp -d) + + cp ${src} ${scratch_d}/resize.img + qemu-img resize ${scratch_d}/resize.img ${size}M + python ${streamconverter} ${scratch_d}/resize.img "${destination}" + modify_vmdk_header "${destination}" + qemu-img info "${destination}" + + rm -rf ${scratch_d} +} + + diff --git a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary index 51d91768..2fc48065 100755 --- a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary +++ b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary @@ -18,66 +18,10 @@ case ${architecture} in exit 0;; esac -apt-get install -qqy qemu-utils vmdk-stream-converter +. /build/config/functions -streamconverter="/usr/share/pyshared/VMDKstream.py" - -# Lets be safe about this -scratch_d=$(mktemp -d) -trap "rm -rf ${scratch_d}" EXIT - -modify_vmdk_header() { - # Modify the VMDK headers so that both VirtualBox _and_ VMware can - # read the vmdk and import them. The vodoo here is _not_ documented - # anywhere....so this will have to do. - - vmdk_name="${1}" - - # Extract the vmdk header for manipulation - dd if="${vmdk_name}" of=descriptor.txt bs=1 skip=512 count=1024 - - # The sed lines below is where the magic is. Specifically: - # ddb.toolsVersion: sets the open-vm-tools so that VMware shows - # the tooling as current - # ddb.virtualHWVersion: set the version to 7, which covers most - # current versions of VMware - # createType: make sure its set to stream Optimized - # remove the vmdk-stream-converter comment and replace with - # # Disk DescriptorFile. This is needed for Virtualbox - # remove the comments from vmdk-stream-converter which causes - # VirtualBox and others to fail VMDK validation - - sed -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ - -e 's|ddb.virtualHWVersion.*|ddb.virtualHWVersion = "7"|' \ - -e 's|createType.*|createType="streamOptimized"|' \ - -e 's|# Description file.*|# Disk DescriptorFile|' \ - -e '/# Believe.*/d' \ - -e '/# Indicates no parent/d' \ - descriptor.txt > new_descriptor.txt - - # The header is cannot be bigger than 1024 - expr $(stat --format=%s new_descriptor.txt) \< 1024 || { - echo "descriptor is too large, VMDK will be invalid!"; exit 1; } - - # Overwrite the vmdk header with our new, modified one - dd conv=notrunc,nocreat \ - if=new_descriptor.txt of="${vmdk_name}" \ - bs=1 seek=512 count=1024 -} - -convert_image() { - src="$1" - destination="$2" - - cp ${src} ${scratch_d}/resize.img - qemu-img resize ${scratch_d}/resize.img 10G - python ${streamconverter} ${scratch_d}/resize.img "${destination}" - modify_vmdk_header "${destination}" - qemu-img info "${destination}" -} - -convert_image binary/boot/disk.ext4 livecd.ubuntu-cpc.disk1.vmdk +create_vmdk binary/boot/disk.ext4 livecd.ubuntu-cpc.disk1.vmdk if [ -e binary/boot/disk-uefi.ext4 ]; then - convert_image binary/boot/disk-uefi.ext4 livecd.ubuntu-cpc.uefi.vmdk + create_vmdk binary/boot/disk-uefi.ext4 livecd.ubuntu-cpc.uefi.vmdk fi diff --git a/live-build/ubuntu-cpc/hooks/042-vagrant.binary b/live-build/ubuntu-cpc/hooks/042-vagrant.binary index 6c185881..fa53778a 100755 --- a/live-build/ubuntu-cpc/hooks/042-vagrant.binary +++ b/live-build/ubuntu-cpc/hooks/042-vagrant.binary @@ -28,6 +28,8 @@ if [ ! -e ${base_vmdk} ]; then exit 0 fi +. /build/config/functions + # Virtualbox is needed for making a small VMDK apt-get -qqy install genisoimage qemu-utils @@ -83,12 +85,13 @@ genisoimage \ -output ${seed_d}/seed.iso \ -volid cidata \ -joliet -rock \ + -input-charset utf-8 \ ${seed_d}/user-data \ ${seed_d}/meta-data \ ${seed_d}/bloat_file # Make a VMDK out of the seed file. -qemu-img convert -O vmdk ${seed_d}/seed.iso ${cdrom_vmdk_f} +create_vmdk ${seed_d}/seed.iso ${cdrom_vmdk_f} ### END Create ConfigDrive ########################## From 9a87b066cfbf9e787697cdc9e1e748af41f179fa Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 17:33:47 -0700 Subject: [PATCH 11/15] Comment fixup --- debian/changelog | 12 ++++++++++++ live-build/ubuntu-cpc/functions | 8 +++++++- live-build/ubuntu-cpc/hooks/040-vmdk-image.binary | 5 ----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0a618b11..c055287c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +livecd-rootfs (2.354.0~utlemming31) xenial; urgency=medium + + * Test build + + -- Ben Howard Thu, 12 Nov 2015 17:22:18 -0700 + +livecd-rootfs (2.354.0~utlemming30) xenial; urgency=medium + + * Test build + + -- Ben Howard Thu, 12 Nov 2015 17:21:42 -0700 + livecd-rootfs (2.354) xenial; urgency=medium * ubuntu-cpc: extended hooks to build additional cloud image targets diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 513cc9e8..474e854d 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -63,7 +63,8 @@ umount_partition() { modify_vmdk_header() { # Modify the VMDK headers so that both VirtualBox _and_ VMware can # read the vmdk and import them. The vodoo here is _not_ documented - # anywhere....so this will have to do. + # anywhere....so this will have to do. This is undocumented vodoo + # that has been learned by the Cloud Image team. vmdk_name="${1}" descriptor=$(mktemp) @@ -104,6 +105,11 @@ modify_vmdk_header() { } create_vmdk() { + # There is no real good way to create a _compressed_ VMDK using open source + # tooling that works across multiple VMDK-capable platforms. Thie functions + # uses vmdk-stream-converter and then call modify_vmdk_header to produce a + # compatiable VMDK. + src="$1" destination="$2" size="${3:-102400}" diff --git a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary index 2fc48065..11c0ed7f 100755 --- a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary +++ b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary @@ -2,11 +2,6 @@ # vi: ts=4 noexpandtab # # Generate VMDK files -# -# There is no real good way to create a _compressed_ VMDK using open source -# tooling that works across multiple VMDK-capable platforms. This hook uses -# vmdk-stream-converter and then modifies the header manually to produce a -# compatiable VMDK. architecture=$(chroot chroot dpkg --print-architecture) From 4a752e14c0fc289bd25cc68324961e0566034609 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 17:52:05 -0700 Subject: [PATCH 12/15] Use standard tools for resize --- live-build/ubuntu-cpc/functions | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 474e854d..4847867f 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -112,16 +112,17 @@ create_vmdk() { src="$1" destination="$2" - size="${3:-102400}" + size="${3:-10240}" apt-get install -qqy qemu-utils vmdk-stream-converter streamconverter="/usr/share/pyshared/VMDKstream.py" scratch_d=$(mktemp -d) - cp ${src} ${scratch_d}/resize.img - qemu-img resize ${scratch_d}/resize.img ${size}M + + truncate --size=${size}M ${scratch_d}/resize.img python ${streamconverter} ${scratch_d}/resize.img "${destination}" modify_vmdk_header "${destination}" + qemu-img info "${destination}" rm -rf ${scratch_d} From 2b818e8475f95027c05066e0bbad9896df1192e4 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 17:56:02 -0700 Subject: [PATCH 13/15] Restrict Vagrant size --- live-build/ubuntu-cpc/hooks/042-vagrant.binary | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/live-build/ubuntu-cpc/hooks/042-vagrant.binary b/live-build/ubuntu-cpc/hooks/042-vagrant.binary index fa53778a..83441598 100755 --- a/live-build/ubuntu-cpc/hooks/042-vagrant.binary +++ b/live-build/ubuntu-cpc/hooks/042-vagrant.binary @@ -87,11 +87,10 @@ genisoimage \ -joliet -rock \ -input-charset utf-8 \ ${seed_d}/user-data \ - ${seed_d}/meta-data \ - ${seed_d}/bloat_file + ${seed_d}/meta-data # Make a VMDK out of the seed file. -create_vmdk ${seed_d}/seed.iso ${cdrom_vmdk_f} +create_vmdk ${seed_d}/seed.iso ${cdrom_vmdk_f} 10 ### END Create ConfigDrive ########################## From 48a08042edd1df4082129a790a1677e1f267b391 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 12 Nov 2015 18:52:33 -0700 Subject: [PATCH 14/15] Fix --- debian/changelog | 12 ++++++++++++ live-build/ubuntu-cpc/functions | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index c055287c..3963ff92 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +livecd-rootfs (2.354.0~utlemming33) xenial; urgency=medium + + * Test build + + -- Ben Howard Thu, 12 Nov 2015 17:56:31 -0700 + +livecd-rootfs (2.354.0~utlemming32) xenial; urgency=medium + + * Test build + + -- Ben Howard Thu, 12 Nov 2015 17:52:26 -0700 + livecd-rootfs (2.354.0~utlemming31) xenial; urgency=medium * Test build diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 4847867f..31645633 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -90,18 +90,18 @@ modify_vmdk_header() { -e 's|# Description file.*|# Disk DescriptorFile|' \ -e '/# Believe.*/d' \ -e '/# Indicates no parent/d' \ - "${descriptor}" > "${new_descriptor}" + "${descriptor}" > "${newdescriptor}" # The header is cannot be bigger than 1024 - expr $(stat --format=%s ${new_descriptor}) \< 1024 || { + expr $(stat --format=%s ${newdescriptor}) \< 1024 || { echo "descriptor is too large, VMDK will be invalid!"; exit 1; } # Overwrite the vmdk header with our new, modified one dd conv=notrunc,nocreat \ - if="${new_descriptor}" of="${vmdk_name}" \ + if="${newdescriptor}" of="${vmdk_name}" \ bs=1 seek=512 count=1024 - rm ${descriptor} ${new_descriptor} + rm ${descriptor} ${newdescriptor} } create_vmdk() { From 1855130ecc9a6cf192a843ca3f733d4eca614d45 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 13 Nov 2015 08:09:26 -0700 Subject: [PATCH 15/15] Pre-merge commit --- debian/changelog | 31 ------------------- live-build/ubuntu-cpc/functions | 28 ++++++++--------- .../ubuntu-cpc/hooks/040-vmdk-image.binary | 4 +-- .../hooks/041-vmdk-ova-image.binary | 2 +- .../ubuntu-cpc/hooks/042-vagrant.binary | 14 ++++----- 5 files changed, 22 insertions(+), 57 deletions(-) diff --git a/debian/changelog b/debian/changelog index 3963ff92..fed3b79d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,34 +1,3 @@ -livecd-rootfs (2.354.0~utlemming33) xenial; urgency=medium - - * Test build - - -- Ben Howard Thu, 12 Nov 2015 17:56:31 -0700 - -livecd-rootfs (2.354.0~utlemming32) xenial; urgency=medium - - * Test build - - -- Ben Howard Thu, 12 Nov 2015 17:52:26 -0700 - -livecd-rootfs (2.354.0~utlemming31) xenial; urgency=medium - - * Test build - - -- Ben Howard Thu, 12 Nov 2015 17:22:18 -0700 - -livecd-rootfs (2.354.0~utlemming30) xenial; urgency=medium - - * Test build - - -- Ben Howard Thu, 12 Nov 2015 17:21:42 -0700 - -livecd-rootfs (2.354) xenial; urgency=medium - - * ubuntu-cpc: extended hooks to build additional cloud image targets - including VMDKs, OVA, and Vagrant images. - - -- Ben Howard Thu, 12 Nov 2015 15:00:36 -0700 - livecd-rootfs (2.353) xenial; urgency=medium * drop input group from snappy images, adjust md5 sums in diff --git a/live-build/ubuntu-cpc/functions b/live-build/ubuntu-cpc/functions index 31645633..b5500e70 100644 --- a/live-build/ubuntu-cpc/functions +++ b/live-build/ubuntu-cpc/functions @@ -1,3 +1,5 @@ +# vi: ts=4 expandtab syntax=sh + CLOUD_IMG_STR="# CLOUD_IMG: This file was created/modified by the Cloud Image build process" IMAGE_SIZE=$((2252*1024**2)) # 2.2G (the current size we ship) @@ -84,16 +86,15 @@ modify_vmdk_header() { # remove the comments from vmdk-stream-converter which causes # VirtualBox and others to fail VMDK validation - sed -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ - -e 's|ddb.virtualHWVersion.*|ddb.virtualHWVersion = "7"|' \ - -e 's|createType.*|createType="streamOptimized"|' \ - -e 's|# Description file.*|# Disk DescriptorFile|' \ - -e '/# Believe.*/d' \ + sed -e 's|# Description file.*|# Disk DescriptorFile|' \ + -e '/# Believe this is random*/d' \ -e '/# Indicates no parent/d' \ + -e '/# The Disk Data Base/d' \ + -e 's|ddb.comment.*|ddb.toolsVersion = "2147483647"|' \ "${descriptor}" > "${newdescriptor}" # The header is cannot be bigger than 1024 - expr $(stat --format=%s ${newdescriptor}) \< 1024 || { + expr $(stat --format=%s ${newdescriptor}) \< 1024 > /dev/null 2>&1 || { echo "descriptor is too large, VMDK will be invalid!"; exit 1; } # Overwrite the vmdk header with our new, modified one @@ -106,9 +107,9 @@ modify_vmdk_header() { create_vmdk() { # There is no real good way to create a _compressed_ VMDK using open source - # tooling that works across multiple VMDK-capable platforms. Thie functions - # uses vmdk-stream-converter and then call modify_vmdk_header to produce a - # compatiable VMDK. + # tooling that works across multiple VMDK-capable platforms. This functions + # uses vmdk-stream-converter and then calls modify_vmdk_header to produce a + # compatible VMDK. src="$1" destination="$2" @@ -120,12 +121,9 @@ create_vmdk() { cp ${src} ${scratch_d}/resize.img truncate --size=${size}M ${scratch_d}/resize.img - python ${streamconverter} ${scratch_d}/resize.img "${destination}" - modify_vmdk_header "${destination}" - - qemu-img info "${destination}" + python ${streamconverter} ${scratch_d}/resize.img ${destination} + modify_vmdk_header ${destination} + qemu-img info ${destination} rm -rf ${scratch_d} } - - diff --git a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary index 11c0ed7f..44f876dc 100755 --- a/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary +++ b/live-build/ubuntu-cpc/hooks/040-vmdk-image.binary @@ -1,5 +1,5 @@ #!/bin/bash -eux -# vi: ts=4 noexpandtab +# vi: ts=4 expandtab # # Generate VMDK files @@ -15,7 +15,7 @@ esac . /build/config/functions -create_vmdk binary/boot/disk.ext4 livecd.ubuntu-cpc.disk1.vmdk +create_vmdk binary/boot/disk.ext4 livecd.ubuntu-cpc.disk1.vmdk if [ -e binary/boot/disk-uefi.ext4 ]; then create_vmdk binary/boot/disk-uefi.ext4 livecd.ubuntu-cpc.uefi.vmdk diff --git a/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary b/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary index 895d3b27..84134c42 100755 --- a/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary +++ b/live-build/ubuntu-cpc/hooks/041-vmdk-ova-image.binary @@ -1,5 +1,5 @@ #!/bin/bash -eux -# vi: ts=4 noexpandtab +# vi: ts=4 expandtab # # Generate OVA images # diff --git a/live-build/ubuntu-cpc/hooks/042-vagrant.binary b/live-build/ubuntu-cpc/hooks/042-vagrant.binary index 83441598..ee3a03d6 100755 --- a/live-build/ubuntu-cpc/hooks/042-vagrant.binary +++ b/live-build/ubuntu-cpc/hooks/042-vagrant.binary @@ -17,14 +17,12 @@ my_d=$(dirname $(readlink -f ${0})) architecture=$(chroot chroot dpkg --print-architecture) base_vmdk="livecd.ubuntu-cpc.disk1.vmdk" -case ${architecture} in - amd64) base_vmdk="livecd.ubuntu-cpc.uefi.vmdk";; - *) echo "OVA images are not supported for ${architecture} yet."; - exit 0;; -esac - -if [ ! -e ${base_vmdk} ]; then - find . | grep vmdk + +if [[ "${architecture}" =~ (amd64|i386) ]]; then + echo "Vagrant images are not supported for ${architecutre}" + exit 0 +elif [ ! -e ${base_vmdk} ]; then + echo "Did not find VMDK to produce Vagrant images." exit 0 fi