Robert C Jennings 77ae8d704f
ubuntu-cpc: parallel builds
* Replace "snap download" with tool that uses snap store's coherence feature

    This is important for parallel image builds to ensure all pre-seeded snaps
    have the same versions across image variants.

* Inject a proxy into the build providing a snapshot view of the package repo.

    When the REPO_SNAPSHOT_STAMP variable is set, the auto/build script will attempt
    to launch a transparent HTTP proxy on port 8080, and insert an iptables rule to
    redirect all outgoing HTTP requests to this proxy.

    The proxy, contained in the `magic-proxy` Python script, examines each request
    and silently overrides those pointing to InRelease files or files that are
    listed in InRelease files. It will instead provide the contents of the requested
    file as it was at REPO_SNAPSHOT_STAMP, by downloading the corresponding asset
    "by hash".

* Use series files with dependency handling to generate hook symlinks dynamically

    This patch currently only applies to the "ubuntu-cpc" project.

    More and more logic has been going into the hook scripts to decide
    under which conditions they should run or not. As we are moving
    to parallelized builds of image sets, this will get even more
    complicated. Base hooks will have to know which image sets they
    belong to and modification of the dependency chain between scripts
    will become more complicated and prone to errors, as the number of
    image sets grows.

    This patch introduces explicit ordering and dependency handling for
    scripts through the use of `series` files and an explicit syntax
    for dependency specification.
2019-05-21 17:06:59 -05:00

245 lines
5.3 KiB
Perl
Executable File

#!/usr/bin/env perl
# ----------------------------------------------------------------------
# mkknlimg by Phil Elwell for Raspberry Pi
# based on extract-ikconfig by Dick Streefland
#
# (c) 2009,2010 Dick Streefland <dick@streefland.net>
# (c) 2014,2015 Raspberry Pi (Trading) Limited <info@raspberrypi.org>
#
# Licensed under the terms of the GNU General Public License.
# ----------------------------------------------------------------------
use strict;
use warnings;
use integer;
my $trailer_magic = 'RPTL';
my $tmpfile1 = "/tmp/mkknlimg_$$.1";
my $tmpfile2 = "/tmp/mkknlimg_$$.2";
my $dtok = 0;
my $is_283x = 0;
while (@ARGV && ($ARGV[0] =~ /^-/))
{
my $arg = shift(@ARGV);
if ($arg eq '--dtok')
{
$dtok = 1;
}
elsif ($arg eq '--283x')
{
$is_283x = 1;
}
else
{
print ("* Unknown option '$arg'\n");
usage();
}
}
usage() if (@ARGV != 2);
my $kernel_file = $ARGV[0];
my $out_file = $ARGV[1];
if (! -r $kernel_file)
{
print ("* File '$kernel_file' not found\n");
usage();
}
my @wanted_strings =
(
'bcm2708_fb',
'brcm,bcm2835-mmc',
'brcm,bcm2835-sdhost',
'brcm,bcm2708-pinctrl',
'brcm,bcm2835-gpio',
'brcm,bcm2835',
'brcm,bcm2836'
);
my $res = try_extract($kernel_file, $tmpfile1);
$res = try_decompress('\037\213\010', 'xy', 'gunzip', 0,
$kernel_file, $tmpfile1, $tmpfile2) if (!$res);
$res = try_decompress('\3757zXZ\000', 'abcde', 'unxz --single-stream', -1,
$kernel_file, $tmpfile1, $tmpfile2) if (!$res);
$res = try_decompress('BZh', 'xy', 'bunzip2', 0,
$kernel_file, $tmpfile1, $tmpfile2) if (!$res);
$res = try_decompress('\135\0\0\0', 'xxx', 'unlzma', 0,
$kernel_file, $tmpfile1, $tmpfile2) if (!$res);
$res = try_decompress('\211\114\132', 'xy', 'lzop -d', 0,
$kernel_file, $tmpfile1, $tmpfile2) if (!$res);
$res = try_decompress('\002\041\114\030', 'xy', 'lz4 -d', 1,
$kernel_file, $tmpfile1, $tmpfile2) if (!$res);
my $append_trailer;
my $trailer;
my $kver = '?';
$append_trailer = $dtok;
if ($res)
{
$kver = $res->{''} || '?';
print("Version: $kver\n");
$append_trailer = $dtok;
if (!$dtok)
{
if (config_bool($res, 'bcm2708_fb') ||
config_bool($res, 'brcm,bcm2835-mmc') ||
config_bool($res, 'brcm,bcm2835-sdhost'))
{
$dtok ||= config_bool($res, 'brcm,bcm2708-pinctrl');
$dtok ||= config_bool($res, 'brcm,bcm2835-gpio');
$is_283x ||= config_bool($res, 'brcm,bcm2835');
$is_283x ||= config_bool($res, 'brcm,bcm2836');
$dtok ||= $is_283x;
$append_trailer = 1;
}
else
{
print ("* This doesn't look like a Raspberry Pi kernel. In pass-through mode.\n");
}
}
}
elsif (!$dtok)
{
print ("* Is this a valid kernel? In pass-through mode.\n");
}
if ($append_trailer)
{
printf("DT: %s\n", $dtok ? "y" : "n");
printf("283x: %s\n", $is_283x ? "y" : "n");
my @atoms;
push @atoms, [ $trailer_magic, pack('V', 0) ];
push @atoms, [ 'KVer', $kver ];
push @atoms, [ 'DTOK', pack('V', $dtok) ];
push @atoms, [ '283x', pack('V', $is_283x) ];
$trailer = pack_trailer(\@atoms);
$atoms[0]->[1] = pack('V', length($trailer));
$trailer = pack_trailer(\@atoms);
}
my $ofh;
my $total_len = 0;
if ($out_file eq $kernel_file)
{
die "* Failed to open '$out_file' for append\n"
if (!open($ofh, '>>', $out_file));
$total_len = tell($ofh);
}
else
{
die "* Failed to open '$kernel_file'\n"
if (!open(my $ifh, '<', $kernel_file));
die "* Failed to create '$out_file'\n"
if (!open($ofh, '>', $out_file));
my $copybuf;
while (1)
{
my $bytes = sysread($ifh, $copybuf, 64*1024);
last if (!$bytes);
syswrite($ofh, $copybuf, $bytes);
$total_len += $bytes;
}
close($ifh);
}
if ($trailer)
{
# Pad to word-alignment
syswrite($ofh, "\x000\x000\x000", (-$total_len & 0x3));
syswrite($ofh, $trailer);
}
close($ofh);
exit($trailer ? 0 : 1);
END {
unlink($tmpfile1) if ($tmpfile1);
unlink($tmpfile2) if ($tmpfile2);
}
sub usage
{
print ("Usage: mkknlimg [--dtok] [--283x] <vmlinux|zImage|bzImage> <outfile>\n");
exit(1);
}
sub try_extract
{
my ($knl, $tmp) = @_;
my $ver = `strings "$knl" | grep -a -E "^Linux version [1-9]"`;
return undef if (!$ver);
chomp($ver);
my $res = { ''=>$ver };
my $string_pattern = '^('.join('|', @wanted_strings).')$';
my @matches = `strings \"$knl\" | grep -E \"$string_pattern\"`;
foreach my $match (@matches)
{
chomp($match);
$res->{$match} = 1;
}
return $res;
}
sub try_decompress
{
my ($magic, $subst, $zcat, $idx, $knl, $tmp1, $tmp2) = @_;
my $pos = `tr "$magic\n$subst" "\n$subst=" < "$knl" | grep -abo "^$subst"`;
if ($pos)
{
chomp($pos);
$pos = (split(/[\r\n]+/, $pos))[$idx];
return undef if (!defined($pos));
$pos =~ s/:.*[\r\n]*$//s;
my $cmd = "tail -c+$pos \"$knl\" | $zcat > $tmp2 2> /dev/null";
my $err = (system($cmd) >> 8);
return undef if (($err != 0) && ($err != 2));
return try_extract($tmp2, $tmp1);
}
return undef;
}
sub pack_trailer
{
my ($atoms) = @_;
my $trailer = pack('VV', 0, 0);
for (my $i = $#$atoms; $i>=0; $i--)
{
my $atom = $atoms->[$i];
$trailer .= pack('a*x!4Va4', $atom->[1], length($atom->[1]), $atom->[0]);
}
return $trailer;
}
sub config_bool
{
my ($configs, $wanted) = @_;
my $val = $configs->{$wanted} || 'n';
return (($val eq 'y') || ($val eq '1'));
}