From b9cdfdb1e2f985ac8bd7647e58e024dec5430d4e Mon Sep 17 00:00:00 2001 From: CloudBuilder Date: Mon, 8 Nov 2021 10:18:33 +0000 Subject: [PATCH] Imported 2.664.33 No reason for CPC update specified. --- debian/changelog | 38 ++++ live-build/auto/build | 18 +- live-build/auto/config | 5 +- .../099-ubuntu-image-customization.chroot | 122 ++++++++++++ magic-proxy | 173 +++++++++++++----- 5 files changed, 304 insertions(+), 52 deletions(-) create mode 100644 live-build/ubuntu/hooks/099-ubuntu-image-customization.chroot diff --git a/debian/changelog b/debian/changelog index e6f80f42..59477607 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,41 @@ +livecd-rootfs (2.664.33) focal; urgency=medium + + * Install cloud-initramfs-growroot to actually enable rootfs resize. + * Fix a grub error by making sure the unicode.pf2 font is installed in the + right path for preinstalled amd64 desktop images. + + -- Łukasz 'sil2100' Zemczak Fri, 29 Oct 2021 15:33:34 +0200 + +livecd-rootfs (2.664.32) focal; urgency=medium + + * 099-ubuntu-image-customization.chroot: fix a typo in it. + + -- Brian Murray Thu, 28 Oct 2021 11:12:32 -0700 + +livecd-rootfs (2.664.31) focal; urgency=medium + + [ Łukasz 'sil2100' Zemczak ] + * Add the 099-ubuntu-image-customization.chroot for + desktop-preinstalled images similar to what we have in groovy+ (for the pi + desktop), but improved for amd64 platforms. We need it to generate a valid + grub.cfg on the rootfs (similar to ubuntu-cpc) and then use that instead + of a static configuration locked on the boot partition (LP: #1949102). + + [ Brian Murray ] + * Properly check ARCH when setting the intel-iot model. + + -- Łukasz 'sil2100' Zemczak Thu, 28 Oct 2021 17:35:12 +0200 + +livecd-rootfs (2.664.30) focal; urgency=medium + + [ Thomas Bechtold ] + * magic-proxy: Replace http.client with urllib calls. live-build/auto/build: + change iptables calls to query rules and quickly check that connectivity + works after transparent proxy has been installed. (LP: #1917920) + * magic-proxy: fix TypeError when trying to call get_uri() (LP: #1944906) + + -- Brian Murray Thu, 21 Oct 2021 11:55:24 -0700 + livecd-rootfs (2.664.29) focal; urgency=medium * Generate manifest for HyperV desktop image (LP: #1940136) diff --git a/live-build/auto/build b/live-build/auto/build index fc456919..0ed25a59 100755 --- a/live-build/auto/build +++ b/live-build/auto/build @@ -35,6 +35,18 @@ run_iptables () { kver="${kver#*.}" kver_minor="${kver%%.*}" + + # LP: #1917920 + # I'm seeing issues after iptables got upgraded from 1.8.5 to + # 1.8.7 Somehow installing our nat rule doesn't get activated, and + # no networking is happening at all. + + # But somehow calling both iptables -S makes things start working. + # Maybe no default chains are installed in our network namespace?! + # Or 1.8.7 is somehow broken? + iptables -v -t nat -S + iptables-legacy -v -t nat -S + if [ "$kver_major" -lt 4 ] || \ ([ "$kver_major" = 4 ] && [ "$kver_minor" -lt 15 ]); then iptables-legacy "$@" @@ -52,10 +64,11 @@ if [ -n "$REPO_SNAPSHOT_STAMP" ]; then apt-get -qyy install iptables # Redirect all outgoing traffic to port 80 to proxy instead. - run_iptables -t nat -A OUTPUT -p tcp --dport 80 \ + run_iptables -v -t nat -A OUTPUT -p tcp --dport 80 \ -m owner ! --uid-owner daemon -j REDIRECT --to 8080 # Run proxy as "daemon" to avoid infinite loop. + LB_PARENT_MIRROR_BOOTSTRAP=$LB_PARENT_MIRROR_BOOTSTRAP \ /usr/share/livecd-rootfs/magic-proxy \ --address="127.0.0.1" \ --port=8080 \ @@ -65,6 +78,9 @@ if [ -n "$REPO_SNAPSHOT_STAMP" ]; then --pid-file=config/magic-proxy.pid \ --background \ --setsid + + # Quick check that magic proxy & iptables chains are working + timeout 3m apt-get update fi # Link output files somewhere launchpad-buildd will be able to find them. diff --git a/live-build/auto/config b/live-build/auto/config index 001dfb75..6d5ad897 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -601,6 +601,9 @@ case $PROJECT in desktop-preinstalled) add_task install minimal standard ubuntu-desktop if [ "$SUBARCH" = "intel-iot" ]; then + # Since for non-pi we don't have any seeds yet but we want to be able to + # grow the rootfs, manually install cloud-initramfs-growroot during build + add_package install cloud-initramfs-growroot KERNEL_FLAVOURS='image-intel' COMPONENTS='main restricted universe' OPTS="${OPTS:+$OPTS }--initramfs=none" @@ -835,7 +838,7 @@ case $PROJECT in arm64) add_package install flash-kernel ;; - amd64) + amd64*) if [ "${SUBARCH:-}" = "intel-iot" ]; then KERNEL_FLAVOURS=image-intel COMPONENTS='main restricted universe' diff --git a/live-build/ubuntu/hooks/099-ubuntu-image-customization.chroot b/live-build/ubuntu/hooks/099-ubuntu-image-customization.chroot new file mode 100644 index 00000000..ccb764b5 --- /dev/null +++ b/live-build/ubuntu/hooks/099-ubuntu-image-customization.chroot @@ -0,0 +1,122 @@ +#!/bin/bash -ex + +. /root/config/chroot + +# Specific ubuntu-image chroot configuration goes here. +if [ "$IMAGEFORMAT" == "none" ]; then + if [ "$SUBPROJECT" == "desktop-preinstalled" ]; then + # Create files/dirs Ubiquity requires + mkdir -p /var/log/installer + touch /var/log/installer/debug + touch /var/log/syslog + chown syslog:adm /var/log/syslog + + # Create the oem user account + if [ -e "/usr/sbin/oem-config-prepare" ]; then + /usr/sbin/useradd -d /home/oem -G adm,sudo -m -N -u 29999 oem + + /usr/sbin/oem-config-prepare --quiet + touch "/var/lib/oem-config/run" + fi + + # Make the writable partition grow + echo "LABEL=writable / ext4 defaults,x-systemd.growfs 0 0" >>/etc/fstab + + # Create a 1GB swapfile + dd if=/dev/zero of=/swapfile bs=1M count=1024 + chmod 0600 /swapfile + mkswap /swapfile + + echo "/swapfile none swap sw 0 0" >>/etc/fstab + + if [ $(dpkg --print-architecture) == "amd64" ]; then + # We need to get a grub.cfg generated for our image + # This is copy-pasted from 999-cpc-fixes.chroot + # TODO: Make this better. One idea would be to have this exported + # in functions or something, and then reused by both the cpc-fixes + # and here. Another possibility is to actually trim down the + # pseudo_grub_probe to only work for a regular preinstalled + # desktop. + # But short term it's safer to use a known code-base. + psuedo_grub_probe() { + cat <<"PSUEDO_GRUB_PROBE" +#!/bin/sh +Usage() { + cat <&2; fail "$@"; } + +short_opts="" +long_opts="device-map:,target:,device" +getopt_out=$(getopt --name "${0##*/}" \ + --options "${short_opts}" --long "${long_opts}" -- "$@") && + eval set -- "${getopt_out}" || + bad_Usage + +device_map="" +target="" +device=0 +arg="" + +while [ $# -ne 0 ]; do + cur=${1}; next=${2}; + case "$cur" in + --device-map) device_map=${next}; shift;; + --device) device=1;; + --target) target=${next}; shift;; + --) shift; break;; + esac + shift; +done +arg=${1} + +case "${target}:${device}:${arg}" in + device:*:/*) echo "/dev/sda1"; exit 0;; + fs:*:*) echo "ext2"; exit 0;; + partmap:*:*) + # older versions of grub (lucid) want 'part_msdos' written + # rather than 'msdos' + legacy_pre="" + grubver=$(dpkg-query --show --showformat '${Version}\n' grub-pc 2>/dev/null) && + dpkg --compare-versions "${grubver}" lt 1.98+20100804-5ubuntu3 && + legacy_pre="part_" + echo "${legacy_pre}msdos"; + exit 0;; + abstraction:*:*) echo ""; exit 0;; + drive:*:/dev/sda) echo "(hd0)";; + drive:*:/dev/sda*) echo "(hd0,1)";; + fs_uuid:*:*) exit 1;; +esac +PSUEDO_GRUB_PROBE + } + + gprobe="/usr/sbin/grub-probe" + moved=0 + if [ -f "${gprobe}" ]; then + mv "${gprobe}" "${gprobe}.dist" + moved=1 + fi + psuedo_grub_probe > "${gprobe}" + chmod 755 "${gprobe}" + + # Generate grub.cfg + /usr/sbin/update-grub2 + + # Somehow grub doesn't copy unicode.pf2 to the right fonts + # directory. + if [ ! -e "/boot/grub/fonts/unicode.pf2" ]; then + mkdir -p /boot/grub/fonts + ln /boot/grub/unicode.pf2 /boot/grub/fonts/unicode.pf2 + fi + + grub2cfg="/boot/grub/grub.cfg" + [ ! -f "${grub2cfg}" ] || + sed -i -e "s,root=/dev/[hs]da1,root=LABEL=writable," "${grub2cfg}" + + [ ${moved} -eq 0 ] || mv "${gprobe}.dist" "${gprobe}" + fi + fi +fi diff --git a/magic-proxy b/magic-proxy index e2d0c28d..10f19c47 100755 --- a/magic-proxy +++ b/magic-proxy @@ -68,6 +68,45 @@ class LPInReleaseCacheError(LPInReleaseBaseError): class LPInReleaseProxyError(LPInReleaseBaseError): pass +IN_LP = "http://ftpmaster.internal/ubuntu" in os.environ.get("LB_PARENT_MIRROR_BOOTSTRAP", "") + +# We cannot proxy & rewrite https requests Thus apt will talk to us +# over http But we must upgrade to https for private-ppas, outside of +# launchpad hence use this helper to re-write urls. +def get_uri(host, path): + if host in ("private-ppa.launchpad.net", "private-ppa.buildd"): + if IN_LP: + return "http://private-ppa.buildd" + path + else: + return "https://private-ppa.launchpad.net" + path + # TODO add split mirror handling for ftpmaster.internal => + # (ports|archive).ubuntu.com + return "http://" + host + path + +def initialize_auth(): + auth_handler = urllib.request.HTTPBasicAuthHandler() + with open('/etc/apt/sources.list') as f: + for line in f.readlines(): + for word in line.split(): + if not word.startswith('http'): + continue + parse=urllib.parse.urlparse(word) + if not parse.username: + continue + if parse.hostname not in ("private-ppa.launchpad.net", "private-ppa.buildd"): + continue + auth_handler.add_password( + "Token Required", "https://private-ppa.launchpad.net" + parse.path, + parse.username, parse.password) + auth_handler.add_password( + "Token Required", "http://private-ppa.buildd" + parse.path, + parse.username, parse.password) + print("add password for", parse.path) + opener = urllib.request.build_opener(auth_handler) + urllib.request.install_opener(opener) + +initialize_auth() + class InRelease: """This class represents an InRelease file.""" @@ -97,7 +136,8 @@ class InRelease: this is set explicitly to correspond to the Last-Modified header spat out by the Web server. """ - self.mirror = mirror + parsed = urllib.parse.urlparse(mirror) + self.mirror = get_uri(parsed.hostname, parsed.path) self.suite = suite self.data = data self.dict = {} @@ -363,7 +403,7 @@ class LPInReleaseCache: suite.""" with self._lock: url_obj = urllib.parse.urlparse(mirror) - address = url_obj.hostname + url_obj.path.rstrip("/") + address = url_obj.scheme + url_obj.hostname + url_obj.path.rstrip("/") inrel_by_hash = self._data\ .get(address, {})\ @@ -403,7 +443,8 @@ class LPInReleaseIndex: which case all look-ups will first go to the cache and only cache misses will result in requests to the Web server. """ - self._mirror = mirror + parsed = urllib.parse.urlparse(mirror) + self._mirror = get_uri(parsed.hostname, parsed.path) self._suite = suite self._cache = cache @@ -528,7 +569,8 @@ class LPInReleaseIndex: return [inrel.hash for inrel in cache_entry] try: - with urllib.request.urlopen(self._base_url) as response: + request=urllib.request.Request(self._base_url) + with urllib.request.urlopen(request) as response: content_encoding = self._guess_content_encoding_for_response( response) @@ -744,6 +786,23 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler): """Process a GET request.""" self.__get_request() + def sanitize_requestline(self): + requestline = [] + for word in self.requestline.split(): + if word.startswith('http'): + parse = urllib.parse.urlparse(word) + parse = urllib.parse.ParseResult( + parse.scheme, + parse.hostname, # not netloc, to sanitize username/password + parse.path, + parse.params, + parse.query, + parse.fragment) + requestline.append(urllib.parse.urlunparse(parse)) + else: + requestline.append(word) + self.requestline = ' '.join(requestline) + def __get_request(self, verb="GET"): """Pass all requests on to the destination server 1:1 except when the target is an InRelease file or a resource listed in an InRelease files. @@ -755,16 +814,25 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler): It is important to understand that there is no status 3xx HTTP redirect happening here, the client does not know that what it receives is not exactly what it requested.""" + host = self.headers.get("host") + + # the host does not start with http(s):// which result in urlparse + # to not detect the host & path correctly (LP:#1944906) + if not host.startswith("http"): + host = "http://{}".format(host) + uri = host + self.path - host, path = self.__get_host_path() + parsed = urllib.parse.urlparse(uri) + + self.sanitize_requestline() m = re.match( r"^(?P.*?)/dists/(?P[^/]+)/(?P.*)$", - path + parsed.path ) if m: - mirror = "http://" + host + m.group("base") + mirror = get_uri(parsed.hostname, m.group("base")) base = m.group("base") suite = m.group("suite") target = m.group("target") @@ -775,50 +843,49 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler): self.server.snapshot_stamp) if inrelease is None: - self.__send_error(404, "No InRelease file found for given " - "mirror, suite and timestamp.") - return - - if target == "InRelease": - # If target is InRelease, send back contents directly. - data = inrelease.data.encode("utf-8") - self.log_message( - "Inject InRelease '{}'".format(inrelease.hash)) - - self.send_response(200) - self.send_header("Content-Length", len(data)) - self.end_headers() + "InRelease not found for {}/{}".format(parsed.hostname, parsed.path)) + self.send_error(404, "No InRelease file found for given " + "mirror, suite and timestamp.") + return - if verb == "GET": - self.wfile.write(data) + hash_ = None - return + if target == "InRelease": + hash_ = inrelease.hash else: - # If target hash is listed, then redirect to by-hash URL. hash_ = inrelease.get_hash_for(target) - if hash_: - self.log_message( - "Inject {} for {}".format(hash_, target)) + if hash_: + self.log_message( + "Inject {} for {}".format(hash_, target)) - target_path = target.rsplit("/", 1)[0] + target_path = target.rsplit("/", 1)[0] - path = "{}/dists/{}/{}/by-hash/SHA256/{}"\ - .format(base, suite, target_path, hash_) + uri = "{}/dists/{}/by-hash/SHA256/{}"\ + .format(mirror, suite, hash_) + else: + uri = get_uri(parsed.hostname, parsed.path) + ## use requests such that authentication via password database happens + ## reuse all the headers that we got asked to provide try: - client = http.client.HTTPConnection(host) - client.request(verb, path) - except Exception as e: - self.log_error("Failed to retrieve http://{}{}: {}" - .format(host, path, str(e))) - return + with urllib.request.urlopen( + urllib.request.Request( + uri, + method=verb, + headers=self.headers)) as response: + self.__send_response(response) + except urllib.error.HTTPError as e: + if e.code not in (304,): + self.log_message( + "urlopen() failed for {} with {}".format(uri, e.reason)) + self.__send_response(e) + except urllib.error.URLError as e: + self.log_message( + "urlopen() failed for {} with {}".format(uri, e.reason)) + self.send_error(501, e.reason) - try: - self.__send_response(client.getresponse()) - except Exception as e: - self.log_error("Error delivering response: {}".format(str(e))) def __get_host_path(self): """Figure out the host to contact and the path of the resource that is @@ -831,20 +898,26 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler): def __send_response(self, response): """Pass on upstream response headers and body to the client.""" - self.send_response(response.status) + if hasattr(response, "status"): + status = response.status + elif hassattr(response, "code"): + status = response.code + elif hasattr(response, "getstatus"): + status = response.getstatus() + + if hasattr(response, "headers"): + headers = response.headers + elif hasattr(response, "info"): + headers = response.info() - for name, value in response.getheaders(): - self.send_header(name, value) + self.send_response(status) - self.end_headers() - shutil.copyfileobj(response, self.wfile) + for name, value in headers.items(): + self.send_header(name, value) - def __send_error(self, status, message): - """Return an HTTP error status and a message in the response body.""" - self.send_response(status) - self.send_header("Content-Type", "text/plain; charset=utf-8") self.end_headers() - self.wfile.write(message.encode("utf-8")) + if hasattr(response, "read"): + shutil.copyfileobj(response, self.wfile) class MagicHTTPProxy(socketserver.ThreadingMixIn, http.server.HTTPServer):