Imported 2.664.33

No reason for CPC update specified.
2.664.33
CloudBuilder 3 years ago
parent 9d01f6c218
commit b9cdfdb1e2

38
debian/changelog vendored

@ -1,3 +1,41 @@
livecd-rootfs (2.664.33) focal; urgency=medium
* Install cloud-initramfs-growroot to actually enable rootfs resize.
* Fix a grub error by making sure the unicode.pf2 font is installed in the
right path for preinstalled amd64 desktop images.
-- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com> Fri, 29 Oct 2021 15:33:34 +0200
livecd-rootfs (2.664.32) focal; urgency=medium
* 099-ubuntu-image-customization.chroot: fix a typo in it.
-- Brian Murray <brian@ubuntu.com> Thu, 28 Oct 2021 11:12:32 -0700
livecd-rootfs (2.664.31) focal; urgency=medium
[ Łukasz 'sil2100' Zemczak ]
* Add the 099-ubuntu-image-customization.chroot for
desktop-preinstalled images similar to what we have in groovy+ (for the pi
desktop), but improved for amd64 platforms. We need it to generate a valid
grub.cfg on the rootfs (similar to ubuntu-cpc) and then use that instead
of a static configuration locked on the boot partition (LP: #1949102).
[ Brian Murray ]
* Properly check ARCH when setting the intel-iot model.
-- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com> Thu, 28 Oct 2021 17:35:12 +0200
livecd-rootfs (2.664.30) focal; urgency=medium
[ Thomas Bechtold ]
* magic-proxy: Replace http.client with urllib calls. live-build/auto/build:
change iptables calls to query rules and quickly check that connectivity
works after transparent proxy has been installed. (LP: #1917920)
* magic-proxy: fix TypeError when trying to call get_uri() (LP: #1944906)
-- Brian Murray <brian@ubuntu.com> Thu, 21 Oct 2021 11:55:24 -0700
livecd-rootfs (2.664.29) focal; urgency=medium livecd-rootfs (2.664.29) focal; urgency=medium
* Generate manifest for HyperV desktop image (LP: #1940136) * Generate manifest for HyperV desktop image (LP: #1940136)

@ -35,6 +35,18 @@ run_iptables () {
kver="${kver#*.}" kver="${kver#*.}"
kver_minor="${kver%%.*}" kver_minor="${kver%%.*}"
# LP: #1917920
# I'm seeing issues after iptables got upgraded from 1.8.5 to
# 1.8.7 Somehow installing our nat rule doesn't get activated, and
# no networking is happening at all.
# But somehow calling both iptables -S makes things start working.
# Maybe no default chains are installed in our network namespace?!
# Or 1.8.7 is somehow broken?
iptables -v -t nat -S
iptables-legacy -v -t nat -S
if [ "$kver_major" -lt 4 ] || \ if [ "$kver_major" -lt 4 ] || \
([ "$kver_major" = 4 ] && [ "$kver_minor" -lt 15 ]); then ([ "$kver_major" = 4 ] && [ "$kver_minor" -lt 15 ]); then
iptables-legacy "$@" iptables-legacy "$@"
@ -52,10 +64,11 @@ if [ -n "$REPO_SNAPSHOT_STAMP" ]; then
apt-get -qyy install iptables apt-get -qyy install iptables
# Redirect all outgoing traffic to port 80 to proxy instead. # Redirect all outgoing traffic to port 80 to proxy instead.
run_iptables -t nat -A OUTPUT -p tcp --dport 80 \ run_iptables -v -t nat -A OUTPUT -p tcp --dport 80 \
-m owner ! --uid-owner daemon -j REDIRECT --to 8080 -m owner ! --uid-owner daemon -j REDIRECT --to 8080
# Run proxy as "daemon" to avoid infinite loop. # Run proxy as "daemon" to avoid infinite loop.
LB_PARENT_MIRROR_BOOTSTRAP=$LB_PARENT_MIRROR_BOOTSTRAP \
/usr/share/livecd-rootfs/magic-proxy \ /usr/share/livecd-rootfs/magic-proxy \
--address="127.0.0.1" \ --address="127.0.0.1" \
--port=8080 \ --port=8080 \
@ -65,6 +78,9 @@ if [ -n "$REPO_SNAPSHOT_STAMP" ]; then
--pid-file=config/magic-proxy.pid \ --pid-file=config/magic-proxy.pid \
--background \ --background \
--setsid --setsid
# Quick check that magic proxy & iptables chains are working
timeout 3m apt-get update
fi fi
# Link output files somewhere launchpad-buildd will be able to find them. # Link output files somewhere launchpad-buildd will be able to find them.

@ -601,6 +601,9 @@ case $PROJECT in
desktop-preinstalled) desktop-preinstalled)
add_task install minimal standard ubuntu-desktop add_task install minimal standard ubuntu-desktop
if [ "$SUBARCH" = "intel-iot" ]; then if [ "$SUBARCH" = "intel-iot" ]; then
# Since for non-pi we don't have any seeds yet but we want to be able to
# grow the rootfs, manually install cloud-initramfs-growroot during build
add_package install cloud-initramfs-growroot
KERNEL_FLAVOURS='image-intel' KERNEL_FLAVOURS='image-intel'
COMPONENTS='main restricted universe' COMPONENTS='main restricted universe'
OPTS="${OPTS:+$OPTS }--initramfs=none" OPTS="${OPTS:+$OPTS }--initramfs=none"
@ -835,7 +838,7 @@ case $PROJECT in
arm64) arm64)
add_package install flash-kernel add_package install flash-kernel
;; ;;
amd64) amd64*)
if [ "${SUBARCH:-}" = "intel-iot" ]; then if [ "${SUBARCH:-}" = "intel-iot" ]; then
KERNEL_FLAVOURS=image-intel KERNEL_FLAVOURS=image-intel
COMPONENTS='main restricted universe' COMPONENTS='main restricted universe'

@ -0,0 +1,122 @@
#!/bin/bash -ex
. /root/config/chroot
# Specific ubuntu-image chroot configuration goes here.
if [ "$IMAGEFORMAT" == "none" ]; then
if [ "$SUBPROJECT" == "desktop-preinstalled" ]; then
# Create files/dirs Ubiquity requires
mkdir -p /var/log/installer
touch /var/log/installer/debug
touch /var/log/syslog
chown syslog:adm /var/log/syslog
# Create the oem user account
if [ -e "/usr/sbin/oem-config-prepare" ]; then
/usr/sbin/useradd -d /home/oem -G adm,sudo -m -N -u 29999 oem
/usr/sbin/oem-config-prepare --quiet
touch "/var/lib/oem-config/run"
fi
# Make the writable partition grow
echo "LABEL=writable / ext4 defaults,x-systemd.growfs 0 0" >>/etc/fstab
# Create a 1GB swapfile
dd if=/dev/zero of=/swapfile bs=1M count=1024
chmod 0600 /swapfile
mkswap /swapfile
echo "/swapfile none swap sw 0 0" >>/etc/fstab
if [ $(dpkg --print-architecture) == "amd64" ]; then
# We need to get a grub.cfg generated for our image
# This is copy-pasted from 999-cpc-fixes.chroot
# TODO: Make this better. One idea would be to have this exported
# in functions or something, and then reused by both the cpc-fixes
# and here. Another possibility is to actually trim down the
# pseudo_grub_probe to only work for a regular preinstalled
# desktop.
# But short term it's safer to use a known code-base.
psuedo_grub_probe() {
cat <<"PSUEDO_GRUB_PROBE"
#!/bin/sh
Usage() {
cat <<EOF
Usage: euca-psuedo-grub-probe
this is a wrapper around grub-probe to provide the answers for an ec2 guest
EOF
}
bad_Usage() { Usage 1>&2; fail "$@"; }
short_opts=""
long_opts="device-map:,target:,device"
getopt_out=$(getopt --name "${0##*/}" \
--options "${short_opts}" --long "${long_opts}" -- "$@") &&
eval set -- "${getopt_out}" ||
bad_Usage
device_map=""
target=""
device=0
arg=""
while [ $# -ne 0 ]; do
cur=${1}; next=${2};
case "$cur" in
--device-map) device_map=${next}; shift;;
--device) device=1;;
--target) target=${next}; shift;;
--) shift; break;;
esac
shift;
done
arg=${1}
case "${target}:${device}:${arg}" in
device:*:/*) echo "/dev/sda1"; exit 0;;
fs:*:*) echo "ext2"; exit 0;;
partmap:*:*)
# older versions of grub (lucid) want 'part_msdos' written
# rather than 'msdos'
legacy_pre=""
grubver=$(dpkg-query --show --showformat '${Version}\n' grub-pc 2>/dev/null) &&
dpkg --compare-versions "${grubver}" lt 1.98+20100804-5ubuntu3 &&
legacy_pre="part_"
echo "${legacy_pre}msdos";
exit 0;;
abstraction:*:*) echo ""; exit 0;;
drive:*:/dev/sda) echo "(hd0)";;
drive:*:/dev/sda*) echo "(hd0,1)";;
fs_uuid:*:*) exit 1;;
esac
PSUEDO_GRUB_PROBE
}
gprobe="/usr/sbin/grub-probe"
moved=0
if [ -f "${gprobe}" ]; then
mv "${gprobe}" "${gprobe}.dist"
moved=1
fi
psuedo_grub_probe > "${gprobe}"
chmod 755 "${gprobe}"
# Generate grub.cfg
/usr/sbin/update-grub2
# Somehow grub doesn't copy unicode.pf2 to the right fonts
# directory.
if [ ! -e "/boot/grub/fonts/unicode.pf2" ]; then
mkdir -p /boot/grub/fonts
ln /boot/grub/unicode.pf2 /boot/grub/fonts/unicode.pf2
fi
grub2cfg="/boot/grub/grub.cfg"
[ ! -f "${grub2cfg}" ] ||
sed -i -e "s,root=/dev/[hs]da1,root=LABEL=writable," "${grub2cfg}"
[ ${moved} -eq 0 ] || mv "${gprobe}.dist" "${gprobe}"
fi
fi
fi

@ -68,6 +68,45 @@ class LPInReleaseCacheError(LPInReleaseBaseError):
class LPInReleaseProxyError(LPInReleaseBaseError): class LPInReleaseProxyError(LPInReleaseBaseError):
pass pass
IN_LP = "http://ftpmaster.internal/ubuntu" in os.environ.get("LB_PARENT_MIRROR_BOOTSTRAP", "")
# We cannot proxy & rewrite https requests Thus apt will talk to us
# over http But we must upgrade to https for private-ppas, outside of
# launchpad hence use this helper to re-write urls.
def get_uri(host, path):
if host in ("private-ppa.launchpad.net", "private-ppa.buildd"):
if IN_LP:
return "http://private-ppa.buildd" + path
else:
return "https://private-ppa.launchpad.net" + path
# TODO add split mirror handling for ftpmaster.internal =>
# (ports|archive).ubuntu.com
return "http://" + host + path
def initialize_auth():
auth_handler = urllib.request.HTTPBasicAuthHandler()
with open('/etc/apt/sources.list') as f:
for line in f.readlines():
for word in line.split():
if not word.startswith('http'):
continue
parse=urllib.parse.urlparse(word)
if not parse.username:
continue
if parse.hostname not in ("private-ppa.launchpad.net", "private-ppa.buildd"):
continue
auth_handler.add_password(
"Token Required", "https://private-ppa.launchpad.net" + parse.path,
parse.username, parse.password)
auth_handler.add_password(
"Token Required", "http://private-ppa.buildd" + parse.path,
parse.username, parse.password)
print("add password for", parse.path)
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
initialize_auth()
class InRelease: class InRelease:
"""This class represents an InRelease file.""" """This class represents an InRelease file."""
@ -97,7 +136,8 @@ class InRelease:
this is set explicitly to correspond to the Last-Modified header spat this is set explicitly to correspond to the Last-Modified header spat
out by the Web server. out by the Web server.
""" """
self.mirror = mirror parsed = urllib.parse.urlparse(mirror)
self.mirror = get_uri(parsed.hostname, parsed.path)
self.suite = suite self.suite = suite
self.data = data self.data = data
self.dict = {} self.dict = {}
@ -363,7 +403,7 @@ class LPInReleaseCache:
suite.""" suite."""
with self._lock: with self._lock:
url_obj = urllib.parse.urlparse(mirror) url_obj = urllib.parse.urlparse(mirror)
address = url_obj.hostname + url_obj.path.rstrip("/") address = url_obj.scheme + url_obj.hostname + url_obj.path.rstrip("/")
inrel_by_hash = self._data\ inrel_by_hash = self._data\
.get(address, {})\ .get(address, {})\
@ -403,7 +443,8 @@ class LPInReleaseIndex:
which case all look-ups will first go to the cache and only cache which case all look-ups will first go to the cache and only cache
misses will result in requests to the Web server. misses will result in requests to the Web server.
""" """
self._mirror = mirror parsed = urllib.parse.urlparse(mirror)
self._mirror = get_uri(parsed.hostname, parsed.path)
self._suite = suite self._suite = suite
self._cache = cache self._cache = cache
@ -528,7 +569,8 @@ class LPInReleaseIndex:
return [inrel.hash for inrel in cache_entry] return [inrel.hash for inrel in cache_entry]
try: try:
with urllib.request.urlopen(self._base_url) as response: request=urllib.request.Request(self._base_url)
with urllib.request.urlopen(request) as response:
content_encoding = self._guess_content_encoding_for_response( content_encoding = self._guess_content_encoding_for_response(
response) response)
@ -744,6 +786,23 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
"""Process a GET request.""" """Process a GET request."""
self.__get_request() self.__get_request()
def sanitize_requestline(self):
requestline = []
for word in self.requestline.split():
if word.startswith('http'):
parse = urllib.parse.urlparse(word)
parse = urllib.parse.ParseResult(
parse.scheme,
parse.hostname, # not netloc, to sanitize username/password
parse.path,
parse.params,
parse.query,
parse.fragment)
requestline.append(urllib.parse.urlunparse(parse))
else:
requestline.append(word)
self.requestline = ' '.join(requestline)
def __get_request(self, verb="GET"): def __get_request(self, verb="GET"):
"""Pass all requests on to the destination server 1:1 except when the """Pass all requests on to the destination server 1:1 except when the
target is an InRelease file or a resource listed in an InRelease files. target is an InRelease file or a resource listed in an InRelease files.
@ -755,16 +814,25 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
It is important to understand that there is no status 3xx HTTP redirect It is important to understand that there is no status 3xx HTTP redirect
happening here, the client does not know that what it receives is not happening here, the client does not know that what it receives is not
exactly what it requested.""" exactly what it requested."""
host = self.headers.get("host")
# the host does not start with http(s):// which result in urlparse
# to not detect the host & path correctly (LP:#1944906)
if not host.startswith("http"):
host = "http://{}".format(host)
uri = host + self.path
host, path = self.__get_host_path() parsed = urllib.parse.urlparse(uri)
self.sanitize_requestline()
m = re.match( m = re.match(
r"^(?P<base>.*?)/dists/(?P<suite>[^/]+)/(?P<target>.*)$", r"^(?P<base>.*?)/dists/(?P<suite>[^/]+)/(?P<target>.*)$",
path parsed.path
) )
if m: if m:
mirror = "http://" + host + m.group("base") mirror = get_uri(parsed.hostname, m.group("base"))
base = m.group("base") base = m.group("base")
suite = m.group("suite") suite = m.group("suite")
target = m.group("target") target = m.group("target")
@ -775,27 +843,17 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
self.server.snapshot_stamp) self.server.snapshot_stamp)
if inrelease is None: if inrelease is None:
self.__send_error(404, "No InRelease file found for given " self.log_message(
"InRelease not found for {}/{}".format(parsed.hostname, parsed.path))
self.send_error(404, "No InRelease file found for given "
"mirror, suite and timestamp.") "mirror, suite and timestamp.")
return return
if target == "InRelease": hash_ = None
# If target is InRelease, send back contents directly.
data = inrelease.data.encode("utf-8")
self.log_message(
"Inject InRelease '{}'".format(inrelease.hash))
self.send_response(200)
self.send_header("Content-Length", len(data))
self.end_headers()
if verb == "GET":
self.wfile.write(data)
return if target == "InRelease":
hash_ = inrelease.hash
else: else:
# If target hash is listed, then redirect to by-hash URL.
hash_ = inrelease.get_hash_for(target) hash_ = inrelease.get_hash_for(target)
if hash_: if hash_:
@ -804,21 +862,30 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
target_path = target.rsplit("/", 1)[0] target_path = target.rsplit("/", 1)[0]
path = "{}/dists/{}/{}/by-hash/SHA256/{}"\ uri = "{}/dists/{}/by-hash/SHA256/{}"\
.format(base, suite, target_path, hash_) .format(mirror, suite, hash_)
else:
uri = get_uri(parsed.hostname, parsed.path)
## use requests such that authentication via password database happens
## reuse all the headers that we got asked to provide
try: try:
client = http.client.HTTPConnection(host) with urllib.request.urlopen(
client.request(verb, path) urllib.request.Request(
except Exception as e: uri,
self.log_error("Failed to retrieve http://{}{}: {}" method=verb,
.format(host, path, str(e))) headers=self.headers)) as response:
return self.__send_response(response)
except urllib.error.HTTPError as e:
if e.code not in (304,):
self.log_message(
"urlopen() failed for {} with {}".format(uri, e.reason))
self.__send_response(e)
except urllib.error.URLError as e:
self.log_message(
"urlopen() failed for {} with {}".format(uri, e.reason))
self.send_error(501, e.reason)
try:
self.__send_response(client.getresponse())
except Exception as e:
self.log_error("Error delivering response: {}".format(str(e)))
def __get_host_path(self): def __get_host_path(self):
"""Figure out the host to contact and the path of the resource that is """Figure out the host to contact and the path of the resource that is
@ -831,21 +898,27 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
def __send_response(self, response): def __send_response(self, response):
"""Pass on upstream response headers and body to the client.""" """Pass on upstream response headers and body to the client."""
self.send_response(response.status) if hasattr(response, "status"):
status = response.status
elif hassattr(response, "code"):
status = response.code
elif hasattr(response, "getstatus"):
status = response.getstatus()
if hasattr(response, "headers"):
headers = response.headers
elif hasattr(response, "info"):
headers = response.info()
self.send_response(status)
for name, value in response.getheaders(): for name, value in headers.items():
self.send_header(name, value) self.send_header(name, value)
self.end_headers() self.end_headers()
if hasattr(response, "read"):
shutil.copyfileobj(response, self.wfile) shutil.copyfileobj(response, self.wfile)
def __send_error(self, status, message):
"""Return an HTTP error status and a message in the response body."""
self.send_response(status)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(message.encode("utf-8"))
class MagicHTTPProxy(socketserver.ThreadingMixIn, http.server.HTTPServer): class MagicHTTPProxy(socketserver.ThreadingMixIn, http.server.HTTPServer):
"""Tiny HTTP server using ProxyingHTTPRequestHandler instances to provide """Tiny HTTP server using ProxyingHTTPRequestHandler instances to provide

Loading…
Cancel
Save