From 0cb4fc3643d3d9b22026d93905dc9ca192cc7b0e Mon Sep 17 00:00:00 2001 From: "Zak B. Elep" Date: Sat, 4 Aug 2018 19:50:13 +0800 Subject: [PATCH] generate.pl: overhaul to support building slim images Do a long-needed overhaul of this script allow building of `slim` variants: - Make the base image configurable through a new `builds` setting. - Rename the `64bit` image to `main`, as bit-ness no longer applies as we already build on multiple architectures. - Rename `Releases.yaml` to the more general `config.yml`, allowing it to contain the supported image variants and Configure options. - Run through perltidy :lipstick: The `slim` variants installs the minimum build-deps needed to build Perl, and removes them after (save for make and netbase, to retain being able to install CPAN modules through cpanm.) --- Releases.yaml => config.yml | 21 ++-- generate.pl | 187 ++++++++++++++++++++++++------------ 2 files changed, 143 insertions(+), 65 deletions(-) rename Releases.yaml => config.yml (83%) diff --git a/Releases.yaml b/config.yml similarity index 83% rename from Releases.yaml rename to config.yml index d2c3843..4f6128d 100644 --- a/Releases.yaml +++ b/config.yml @@ -1,39 +1,48 @@ +--- +builds: + - main + - slim + +options: + common: "-Duseshrplib -Dvendorprefix=/usr/local" + threaded: "-Dusethreads" + releases: - version: 5.8.9 sha256: 1097fbcd48ceccb2bc735d119c9db399a02a8ab9f7dc53e29e47e6a8d0d72e79 extra_flags: "-A ccflags=-fwrapv" test_parallel: no - buildpack_deps: jessie + debian_release: jessie - version: 5.10.1 sha256: 9385f2c8c2ca8b1dc4a7c31903f1f8dc8f2ba867dc2a9e5c93012ed6b564e826 extra_flags: "-A ccflags=-fwrapv" test_parallel: no - buildpack_deps: jessie + debian_release: jessie - version: 5.12.5 sha256: 10749417fd3010aae320a34181ad4cd6a4855c1fc63403b87fa4d630b18e966c extra_flags: "-A ccflags=-fwrapv" test_parallel: no - buildpack_deps: jessie + debian_release: jessie - version: 5.14.4 sha256: eece8c2b0d491bf6f746bd1f4f1bb7ce26f6b98e91c54690c617d7af38964745 extra_flags: "-A ccflags=-fwrapv" test_parallel: no - buildpack_deps: jessie + debian_release: jessie - version: 5.16.3 sha256: bb7bc735e6813b177dcfccd480defcde7eddefa173b5967eac11babd1bfa98e8 extra_flags: "-A ccflags=-fwrapv" test_parallel: no - buildpack_deps: jessie + debian_release: jessie - version: 5.18.4 sha256: 1fb4d27b75cd244e849f253320260efe1750641aaff4a18ce0d67556ff1b96a5 extra_flags: "-A ccflags=-fwrapv" test_parallel: no - buildpack_deps: jessie + debian_release: jessie - version: 5.20.3 sha256: 1b40068166c242e34a536836286e70b78410602a80615143301e52aa2901493b diff --git a/generate.pl b/generate.pl index a14add4..d867f9d 100755 --- a/generate.pl +++ b/generate.pl @@ -9,23 +9,68 @@ use LWP::Simple; sub die_with_sample { die < /dev/null \ + && apt-mark manual $savedPackages \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && rm -fr /var/cache/apt/* /var/lib/apt/lists/* +EOF +chomp $docker_slim_run_purge; + +my $config = do { + open my $fh, '<', 'config.yml' or die "Couldn't open config"; local $/; Load <$fh>; }; @@ -35,43 +80,41 @@ my $template = do { ; }; -my $common = join " ", qw{ --Duseshrplib --Dvendorprefix=/usr/local -}; - -my %builds = ( - "64bit" => "$common", - "64bit,threaded" => "-Dusethreads $common", -); +my %builds; # sha256 taken from http://www.cpan.org/authors/id/M/MI/MIYAGAWA/CHECKSUMS my %cpanm = ( - name => "App-cpanminus-1.7044", - url => "http://www.cpan.org/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.7044.tar.gz", - sha256 => "9b60767fe40752ef7a9d3f13f19060a63389a5c23acc3e9827e19b75500f81f3", + name => "App-cpanminus-1.7044", + url => "http://www.cpan.org/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.7044.tar.gz", + sha256 => "9b60767fe40752ef7a9d3f13f19060a63389a5c23acc3e9827e19b75500f81f3", ); -die_with_sample unless defined $yaml->{releases}; -die_with_sample unless ref $yaml->{releases} eq "ARRAY"; +die_with_sample unless defined $config->{releases}; +die_with_sample unless ref $config->{releases} eq "ARRAY"; -if (! -d "downloads") { +if (!-d "downloads") { mkdir "downloads" or die "Couldn't create a downloads directory"; } -for my $release (@{$yaml->{releases}}) { - do { die_with_sample unless $release->{$_}} for (qw(version sha256)); +for my $build (@{$config->{builds}}) { + $builds{$build} = $config->{options}{common}; + $builds{"$build,threaded"} = "@{$config->{options}}{qw/threaded common/}"; +} + +for my $release (@{$config->{releases}}) { + do { die_with_sample unless $release->{$_} } + for (qw(version sha256)); die "Bad version: $release->{version}" unless $release->{version} =~ /\A5\.\d+\.\d+\Z/; my $patch; $release->{type} ||= 'bz2'; my $file = "perl-$release->{version}.tar.$release->{type}"; - my $url = "https://www.cpan.org/src/5.0/$file"; - if (-f "downloads/$file" && - `sha256sum downloads/$file` =~ /^\Q$release->{sha256}\E\s+\Qdownloads\/$file\E/) { - print "Skipping download of $file, already current\n"; - } else { + my $url = "https://www.cpan.org/src/5.0/$file"; + if (-f "downloads/$file" && `sha256sum downloads/$file` =~ /^\Q$release->{sha256}\E\s+\Qdownloads\/$file\E/) { + print "Skipping download of $file, already current\n"; + } + else { print "Downloading $url\n"; getstore($url, "downloads/$file"); } @@ -95,20 +138,37 @@ for my $release (@{$yaml->{releases}}) { die "Couldn't create a Devel::PatchPerl patch for $release->{version}" if $? != 0; } - $release->{url} = $url; - $release->{extra_flags} = "" unless defined $release->{extra_flags}; - $release->{_tag} = $release->{buildpack_deps} || "stretch"; - $release->{"cpanm_dist_$_"} = $cpanm{$_} for keys %cpanm; + for my $build (keys %builds) { + $release->{url} = $url; + $release->{"cpanm_dist_$_"} = $cpanm{$_} for keys %cpanm; + + $release->{extra_flags} ||= ''; + $release->{debian_release} ||= 'stretch'; + + if ($build =~ /main/) { + $release->{image} = 'buildpack-deps'; + $release->{tag} = $release->{debian_release}; + } + else { + $release->{image} = 'debian'; + $release->{tag} = "@{[ $release->{debian_release} ]}-slim"; + } - for my $config (keys %builds) { my $output = $template; $output =~ s/\{\{$_\}\}/$release->{$_}/mg - for (qw(version pause extra_flags sha256 type url _tag cpanm_dist_name cpanm_dist_url cpanm_dist_sha256)); - $output =~ s/\{\{args\}\}/$builds{$config}/mg; + for (qw(version pause extra_flags sha256 type url image tag cpanm_dist_name cpanm_dist_url cpanm_dist_sha256)); + $output =~ s/\{\{args\}\}/$builds{$build}/mg; - my $dir = sprintf "%i.%03i.%03i-%s", - ($release->{version} =~ /(\d+)\.(\d+)\.(\d+)/), - $config; + if ($build =~ /slim/) { + $output =~ s/\{\{docker_slim_run_install\}\}/$docker_slim_run_install/mg; + $output =~ s/\{\{docker_slim_run_purge\}\}/$docker_slim_run_purge/mg; + } + else { + $output =~ s/\{\{docker_slim_run_install\}\}/true/mg; + $output =~ s/\{\{docker_slim_run_purge\}\}/true/mg; + } + + my $dir = sprintf "%i.%03i.%03i-%s", ($release->{version} =~ /(\d+)\.(\d+)\.(\d+)/), $build; mkdir $dir unless -d $dir; @@ -119,11 +179,13 @@ for my $release (@{$yaml->{releases}}) { } if (defined $release->{test_parallel} && $release->{test_parallel} eq "no") { - $output =~ s/\{\{test\}\}/make test_harness/; - } elsif (!defined $release->{test_parallel} || $release->{test_parallel} eq "yes") { - $output =~ s/\{\{test\}\}/TEST_JOBS=\$(nproc) make test_harness/; - } else { - die "test_parallel was provided for $release->{version} but is invalid; should be 'yes' or 'no'\n"; + $output =~ s/\{\{test\}\}/make test_harness/; + } + elsif (!defined $release->{test_parallel} || $release->{test_parallel} eq "yes") { + $output =~ s/\{\{test\}\}/TEST_JOBS=\$(nproc) make test_harness/; + } + else { + die "test_parallel was provided for $release->{version} but is invalid; should be 'yes' or 'no'\n"; } open my $dockerfile, ">", "$dir/Dockerfile" or die "Couldn't open $dir/Dockerfile for writing"; @@ -136,17 +198,18 @@ for my $release (@{$yaml->{releases}}) { =head1 NAME -generate.pl +generate.pl - generate Dockerfiles for Perl =head1 SYNOPSIS -generate.pl is a little helper script to reinitalize the Dockerfiles from a YAML file. + cd /path/to/docker-perl + ./generate.pl =head1 DESCRIPTION -generate.pl is meant to be run from the actual repo directory, with a Releases.yaml file -correctly configured. It starts with a 'releases' key, which contains a list of releases, -each with the following keys: +generate.pl is meant to be run from the actual repo directory, with a +config.yml file correctly configured. It contains with a 'releases' +key, which contains a list of releases, each with the following keys: =over 4 @@ -160,23 +223,27 @@ The actual perl version, such as B<5.20.1>. =item sha256 -The SHA-256 of the C<.tar.bz2> file for that release. +The SHA-256 of the tarball for that release. + +=back =item OPTIONAL =over 4 -=item buildpack_deps +=item debian_release -The Docker L -image tag which this Perl would build on. +The Docker image tag which this Perl would build on, common to both the +L and +L Docker images. -Defaults: C +Defaults: C for C
builds, C for C +builds. =item extra_flags -Additional text to pass to C. At the moment, this is necessary for -5.18.x so that it can get the C<-fwrapv> flag. +Additional text to pass to C. At the moment, this is +necessary for 5.18.x so that it can get the C<-fwrapv> flag. Default: C<""> @@ -195,13 +262,14 @@ Default: C =cut __DATA__ -FROM buildpack-deps:{{_tag}} +FROM {{image}}:{{tag}} LABEL maintainer="Peter Martini , Zak B. Elep " COPY *.patch /usr/src/perl/ WORKDIR /usr/src/perl -RUN curl -SL {{url}} -o perl-{{version}}.tar.{{type}} \ +RUN {{docker_slim_run_install}} \ + && curl -SL {{url}} -o perl-{{version}}.tar.{{type}} \ && echo '{{sha256}} *perl-{{version}}.tar.{{type}}' | sha256sum -c - \ && tar --strip-components=1 -xaf perl-{{version}}.tar.{{type}} -C /usr/src/perl \ && rm perl-{{version}}.tar.{{type}} \ @@ -217,6 +285,7 @@ RUN curl -SL {{url}} -o perl-{{version}}.tar.{{type}} \ && curl -LO {{cpanm_dist_url}} \ && echo '{{cpanm_dist_sha256}} *{{cpanm_dist_name}}.tar.gz' | sha256sum -c - \ && tar -xzf {{cpanm_dist_name}}.tar.gz && cd {{cpanm_dist_name}} && perl bin/cpanm . && cd /root \ + && {{docker_slim_run_purge}} \ && rm -fr ./cpanm /root/.cpanm /usr/src/perl /usr/src/{{cpanm_dist_name}}* /tmp/* WORKDIR /root