From b86caa134680b84d10343d596f9505bf61eb087e Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:18:19 -0700 Subject: [PATCH 1/8] Make the directory if it doesn't already exist --- generate.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generate.pl b/generate.pl index f679dc1..a3fdbd1 100755 --- a/generate.pl +++ b/generate.pl @@ -56,6 +56,8 @@ for my $release (@{$yaml->{releases}}) { ($release->{version} =~ /(\d+)\.(\d+)\.(\d+)/), $config; + mkdir $dir unless -d $dir; + open my $dockerfile, ">$dir/Dockerfile" or die "Couldn't open $dir/Dockerfile for writing"; print $dockerfile $output; close $dockerfile; From 808a689179b34cce47284569f838e6fc669572a1 Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:19:33 -0700 Subject: [PATCH 2/8] Add patch support This is not yet complete; what remains is the ability to generate the final patch, the one that patches patchlevel.h (and thus lists the locally applied patches in perl -V / Config.pm). This requires downloading the proper patchlevel.h for each version and generating a patch in generate.pl. --- generate.pl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/generate.pl b/generate.pl index a3fdbd1..78f0183 100755 --- a/generate.pl +++ b/generate.pl @@ -58,6 +58,16 @@ for my $release (@{$yaml->{releases}}) { mkdir $dir unless -d $dir; + # glob switches behavior in scalar context, so force an intermediate + # list context so we can get the count + if (() = glob "$dir/*.patch") { + $output =~ s#{{copy_patches}}#COPY *.patch /usr/src/perl/#; + $output =~ s#{{apply_patches}}#cat *.patch | patch -p1#; + } else { + $output =~ s/{{copy_patches}}\n//mg; + $output =~ s/.*{{apply_patches}}.*\n//mg; + } + open my $dockerfile, ">$dir/Dockerfile" or die "Couldn't open $dir/Dockerfile for writing"; print $dockerfile $output; close $dockerfile; @@ -112,12 +122,14 @@ RUN apt-get update \ && rm -fr /var/lib/apt/lists/* RUN mkdir /usr/src/perl +{{copy_patches}} WORKDIR /usr/src/perl RUN curl -SL https://cpan.metacpan.org/authors/id/{{pause}}/perl-{{version}}.tar.bz2 -o perl-{{version}}.tar.bz2 \ && echo '{{sha1}} *perl-{{version}}.tar.bz2' | sha1sum -c - \ && tar --strip-components=1 -xjf perl-{{version}}.tar.bz2 -C /usr/src/perl \ && rm perl-{{version}}.tar.bz2 \ + && {{apply_patches}} \ && ./Configure {{args}} {{extra_flags}} -des \ && make -j$(nproc) \ && TEST_JOBS=$(nproc) make test_harness \ From ec0e1348273eccf3e1dd567dccca214ca2a1fec7 Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:25:21 -0700 Subject: [PATCH 3/8] Allow Releases.yaml to disable parallel testing Perl 5.16.3 is sporadically failing the dist/IO/t/io_unix.t on my mention when run with TEST_JOBS > 1, but consistently passes without that. Adding a flag to Releases.yaml to allow this to be coordinated properly. --- generate.pl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/generate.pl b/generate.pl index 78f0183..cfc0c67 100755 --- a/generate.pl +++ b/generate.pl @@ -68,6 +68,14 @@ for my $release (@{$yaml->{releases}}) { $output =~ s/.*{{apply_patches}}.*\n//mg; } + if (defined $release->{test_parallel} && $release->{test_parallel} eq "no") { + $output =~ s/{{test}}/make test_harness/; + } elsif (!defined $release->{test_parallel} || $release->{test_parallel} eq "yes") { + $output =~ s/{{test}}/TEST_JOBS=\$(nproc) make test_harness/; + } else { + die "test_parallel was provided for $release->{version} but is invalid; should be 'yes' or 'no'\n"; + } + open my $dockerfile, ">$dir/Dockerfile" or die "Couldn't open $dir/Dockerfile for writing"; print $dockerfile $output; close $dockerfile; @@ -132,7 +140,7 @@ RUN curl -SL https://cpan.metacpan.org/authors/id/{{pause}}/perl-{{version}}.tar && {{apply_patches}} \ && ./Configure {{args}} {{extra_flags}} -des \ && make -j$(nproc) \ - && TEST_JOBS=$(nproc) make test_harness \ + && {{test}} \ && make install \ && cd /usr/src \ && curl -LO https://raw.githubusercontent.com/miyagawa/cpanminus/master/cpanm \ From 4cc1d62d058ec6fefb7bef271d362af47d7b89c7 Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:29:23 -0700 Subject: [PATCH 4/8] generate.pl pod update; add desc of test_parallel --- generate.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/generate.pl b/generate.pl index cfc0c67..719af03 100755 --- a/generate.pl +++ b/generate.pl @@ -117,6 +117,12 @@ The PAUSE (CPAN user) account that the release was uploaded to. Additional text to pass to C. At the moment, this is necessary for 5.18.x so that it can get the C<-fwrapv> flag. +=item (optionally) test_parallel + +This can be either 'no', 'yes', or unspecified (equivalent to 'yes'). +Added due to dist/IO/t/io_unix.t failing when TEST_JOBS > 1, but should +only be used in case of a documented issue. + =back =cut From 2fddedd2d3e70f12a91bfd784223fa035915bacf Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:32:49 -0700 Subject: [PATCH 5/8] generate.pl POD changes; split keys into sections --- generate.pl | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/generate.pl b/generate.pl index 719af03..c2dab3e 100755 --- a/generate.pl +++ b/generate.pl @@ -100,6 +100,10 @@ each with the following keys: =over 4 +=item REQUIRED + +=over 4 + =item version The actual perl version, such as B<5.20.1>. @@ -112,17 +116,29 @@ The SHA-1 of the C<.tar.bz2> file for that release. The PAUSE (CPAN user) account that the release was uploaded to. -=item (optionally) extra_args +=back + +=item OPTIONAL + +=over 4 + +=item extra_args Additional text to pass to C. At the moment, this is necessary for 5.18.x so that it can get the C<-fwrapv> flag. -=item (optionally) test_parallel +Default: C<""> + +=item test_parallel This can be either 'no', 'yes', or unspecified (equivalent to 'yes'). Added due to dist/IO/t/io_unix.t failing when TEST_JOBS > 1, but should only be used in case of a documented issue. +Default: C + +=back + =back =cut From 6500519dcd938e37e61d5ca0e27b6a2149b0a4a4 Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:54:24 -0700 Subject: [PATCH 6/8] Add note to README.md re locally applied patches --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 2b65279..2eb7ea2 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,9 @@ The 64bit builds specify use64bitall despite this being largely redundant The individual Dockerfiles are generated via 'generate.pl', which uses Releases.yaml to populate the individual files. + +For older versions of Perl, some patches may be necessary to build properly on +a current base OS. In those cases, perl -V will show the locally applied patches. +These changes should be limited to Configure rather than to code itself, and +will be a cherry pick or back port of a patch from the mainline perl branch +whenever possible. From 94595e689aea6181eae7827a83a4b4696a7ebe7e Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:55:43 -0700 Subject: [PATCH 7/8] Add 5.16.3 to list of releases Note that this requires parallel testing disabled due to intermittent failures (really, mostly failures with intermittent successes) in dist/IO/t/io_unix.t, and a small patch for Configure (to be added in the next commit). --- 5.016.003-64bit,threaded/Dockerfile | 29 +++++++++++++++++++++++++++++ 5.016.003-64bit/Dockerfile | 29 +++++++++++++++++++++++++++++ Releases.yaml | 6 ++++++ 3 files changed, 64 insertions(+) create mode 100644 5.016.003-64bit,threaded/Dockerfile create mode 100644 5.016.003-64bit/Dockerfile diff --git a/5.016.003-64bit,threaded/Dockerfile b/5.016.003-64bit,threaded/Dockerfile new file mode 100644 index 0000000..25cfa0e --- /dev/null +++ b/5.016.003-64bit,threaded/Dockerfile @@ -0,0 +1,29 @@ +FROM buildpack-deps +MAINTAINER Peter Martini + +RUN apt-get update \ + && apt-get install -y curl procps \ + && rm -fr /var/lib/apt/lists/* + +RUN mkdir /usr/src/perl +COPY *.patch /usr/src/perl/ +WORKDIR /usr/src/perl + +RUN curl -SL https://cpan.metacpan.org/authors/id/R/RJ/RJBS/perl-5.16.3.tar.bz2 -o perl-5.16.3.tar.bz2 \ + && echo '060bc17cf9f142d043f9bf7b861422ec624875ea *perl-5.16.3.tar.bz2' | sha1sum -c - \ + && tar --strip-components=1 -xjf perl-5.16.3.tar.bz2 -C /usr/src/perl \ + && rm perl-5.16.3.tar.bz2 \ + && cat *.patch | patch -p1 \ + && ./Configure -Dusethreads -Duse64bitall -A ccflags=-fwrapv -des \ + && make -j$(nproc) \ + && make test_harness \ + && make install \ + && cd /usr/src \ + && curl -LO https://raw.githubusercontent.com/miyagawa/cpanminus/master/cpanm \ + && chmod +x cpanm \ + && ./cpanm App::cpanminus \ + && rm -fr ./cpanm /root/.cpanm /usr/src/perl + +WORKDIR /root + +CMD ["perl5.16.3","-de0"] diff --git a/5.016.003-64bit/Dockerfile b/5.016.003-64bit/Dockerfile new file mode 100644 index 0000000..9b41573 --- /dev/null +++ b/5.016.003-64bit/Dockerfile @@ -0,0 +1,29 @@ +FROM buildpack-deps +MAINTAINER Peter Martini + +RUN apt-get update \ + && apt-get install -y curl procps \ + && rm -fr /var/lib/apt/lists/* + +RUN mkdir /usr/src/perl +COPY *.patch /usr/src/perl/ +WORKDIR /usr/src/perl + +RUN curl -SL https://cpan.metacpan.org/authors/id/R/RJ/RJBS/perl-5.16.3.tar.bz2 -o perl-5.16.3.tar.bz2 \ + && echo '060bc17cf9f142d043f9bf7b861422ec624875ea *perl-5.16.3.tar.bz2' | sha1sum -c - \ + && tar --strip-components=1 -xjf perl-5.16.3.tar.bz2 -C /usr/src/perl \ + && rm perl-5.16.3.tar.bz2 \ + && cat *.patch | patch -p1 \ + && ./Configure -Duse64bitall -A ccflags=-fwrapv -des \ + && make -j$(nproc) \ + && make test_harness \ + && make install \ + && cd /usr/src \ + && curl -LO https://raw.githubusercontent.com/miyagawa/cpanminus/master/cpanm \ + && chmod +x cpanm \ + && ./cpanm App::cpanminus \ + && rm -fr ./cpanm /root/.cpanm /usr/src/perl + +WORKDIR /root + +CMD ["perl5.16.3","-de0"] diff --git a/Releases.yaml b/Releases.yaml index ccb7f7a..ef3922e 100644 --- a/Releases.yaml +++ b/Releases.yaml @@ -1,4 +1,10 @@ releases: + - version: 5.16.3 + sha1: 060bc17cf9f142d043f9bf7b861422ec624875ea + pause: RJBS + extra_flags: "-A ccflags=-fwrapv" + test_parallel: no + - version: 5.18.4 sha1: 69c34558a0a939a7adbbc1de48c06ea418d81e27 pause: RJBS From a12be2a2f6bcb3b7dbd0ab791938d4c38494ee67 Mon Sep 17 00:00:00 2001 From: Peter Martini Date: Sat, 11 Apr 2015 14:56:56 -0700 Subject: [PATCH 8/8] Backport of Configure patch for 5.16.3 This is necessary for h2ph to work. The patch comes directly from the main perl branch without any modifications. --- ...-Cppsym-warnings-for-extra-tokens-pe.patch | 43 +++++++++++++++++++ .../0002-UpdatePatchLevel.patch | 10 +++++ ...-Cppsym-warnings-for-extra-tokens-pe.patch | 43 +++++++++++++++++++ 5.016.003-64bit/0002-UpdatePatchLevel.patch | 10 +++++ 4 files changed, 106 insertions(+) create mode 100644 5.016.003-64bit,threaded/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch create mode 100644 5.016.003-64bit,threaded/0002-UpdatePatchLevel.patch create mode 100644 5.016.003-64bit/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch create mode 100644 5.016.003-64bit/0002-UpdatePatchLevel.patch diff --git a/5.016.003-64bit,threaded/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch b/5.016.003-64bit,threaded/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch new file mode 100644 index 0000000..082887b --- /dev/null +++ b/5.016.003-64bit,threaded/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch @@ -0,0 +1,43 @@ +From 6f87f404fa51739971a4068da1f11443024f3fc4 Mon Sep 17 00:00:00 2001 +From: Andy Dougherty +Date: Wed, 6 Jun 2012 11:12:58 -0400 +Subject: [perl #113024] Configure: Avoid Cppsym warnings for extra tokens + [perl #113024] + +The cppsymbols can include macros such as __INT16_C(c), which can't +be tested with a simple #ifdef. This patch strips off the opening +parenthesis and everything following it. These macros were generated +by cpp -dM. + +Also ensure Cppsym.true list is sorted for later input to comm. +(I noticed this while testing this change on Solaris.) +--- + Configure | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/Configure b/Configure +index a780b81..3ae16ca 100755 +--- a/Configure ++++ b/Configure +@@ -21936,15 +21936,16 @@ $cc -o try -Dcpp_stuff=$cpp_stuff $optimize \$ccflags $ldflags try.c $libs && $r + EOSH + chmod +x Cppsym.try + $eunicefix Cppsym.try +-./Cppsym < Cppsym.know > Cppsym.true ++./Cppsym < Cppsym.know | $sort | $uniq > Cppsym.true + : Add in any linux cpp "predefined macros": + case "$osname::$gccversion" in + *linux*::*.*|*gnukfreebsd*::*.*|gnu::*.*) + tHdrH=_tmpHdr + rm -f $tHdrH'.h' $tHdrH + touch $tHdrH'.h' ++ # Filter out macro arguments, such as Linux's __INT8_C(c) + if $cpp -dM $tHdrH'.h' > $tHdrH'_cppsym.h' && [ -s $tHdrH'_cppsym.h' ]; then +- sed 's/#define[\ \ ]*//;s/[\ \ ].*$//' <$tHdrH'_cppsym.h' >$tHdrH'_cppsym.real' ++ sed -e 's/#define[\ \ ]*//;s/[\ \ ].*$//' -e 's/(.*//' <$tHdrH'_cppsym.h' >$tHdrH'_cppsym.real' + if [ -s $tHdrH'_cppsym.real' ]; then + cat $tHdrH'_cppsym.real' Cppsym.know | sort | uniq | ./Cppsym | sort | uniq > Cppsym.true + fi +-- +2.1.4 + diff --git a/5.016.003-64bit,threaded/0002-UpdatePatchLevel.patch b/5.016.003-64bit,threaded/0002-UpdatePatchLevel.patch new file mode 100644 index 0000000..b6d45c4 --- /dev/null +++ b/5.016.003-64bit,threaded/0002-UpdatePatchLevel.patch @@ -0,0 +1,10 @@ +# This should be auto-generated; for now, it's not, +# simply because it should be generated before the build, +# which means having the appropriate version's patchlevel.h +# available, which I don't want to bake into the fetch just +# yet. + +--- a/patchlevel.h ++++ b/patchlevel.h +@@ -135,0 +136 @@ ++ ,"Backport of perl5 git 6f87f404fa51739971a4068da1f11443024f3fc4" diff --git a/5.016.003-64bit/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch b/5.016.003-64bit/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch new file mode 100644 index 0000000..082887b --- /dev/null +++ b/5.016.003-64bit/0001-Configure-Avoid-Cppsym-warnings-for-extra-tokens-pe.patch @@ -0,0 +1,43 @@ +From 6f87f404fa51739971a4068da1f11443024f3fc4 Mon Sep 17 00:00:00 2001 +From: Andy Dougherty +Date: Wed, 6 Jun 2012 11:12:58 -0400 +Subject: [perl #113024] Configure: Avoid Cppsym warnings for extra tokens + [perl #113024] + +The cppsymbols can include macros such as __INT16_C(c), which can't +be tested with a simple #ifdef. This patch strips off the opening +parenthesis and everything following it. These macros were generated +by cpp -dM. + +Also ensure Cppsym.true list is sorted for later input to comm. +(I noticed this while testing this change on Solaris.) +--- + Configure | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/Configure b/Configure +index a780b81..3ae16ca 100755 +--- a/Configure ++++ b/Configure +@@ -21936,15 +21936,16 @@ $cc -o try -Dcpp_stuff=$cpp_stuff $optimize \$ccflags $ldflags try.c $libs && $r + EOSH + chmod +x Cppsym.try + $eunicefix Cppsym.try +-./Cppsym < Cppsym.know > Cppsym.true ++./Cppsym < Cppsym.know | $sort | $uniq > Cppsym.true + : Add in any linux cpp "predefined macros": + case "$osname::$gccversion" in + *linux*::*.*|*gnukfreebsd*::*.*|gnu::*.*) + tHdrH=_tmpHdr + rm -f $tHdrH'.h' $tHdrH + touch $tHdrH'.h' ++ # Filter out macro arguments, such as Linux's __INT8_C(c) + if $cpp -dM $tHdrH'.h' > $tHdrH'_cppsym.h' && [ -s $tHdrH'_cppsym.h' ]; then +- sed 's/#define[\ \ ]*//;s/[\ \ ].*$//' <$tHdrH'_cppsym.h' >$tHdrH'_cppsym.real' ++ sed -e 's/#define[\ \ ]*//;s/[\ \ ].*$//' -e 's/(.*//' <$tHdrH'_cppsym.h' >$tHdrH'_cppsym.real' + if [ -s $tHdrH'_cppsym.real' ]; then + cat $tHdrH'_cppsym.real' Cppsym.know | sort | uniq | ./Cppsym | sort | uniq > Cppsym.true + fi +-- +2.1.4 + diff --git a/5.016.003-64bit/0002-UpdatePatchLevel.patch b/5.016.003-64bit/0002-UpdatePatchLevel.patch new file mode 100644 index 0000000..b6d45c4 --- /dev/null +++ b/5.016.003-64bit/0002-UpdatePatchLevel.patch @@ -0,0 +1,10 @@ +# This should be auto-generated; for now, it's not, +# simply because it should be generated before the build, +# which means having the appropriate version's patchlevel.h +# available, which I don't want to bake into the fetch just +# yet. + +--- a/patchlevel.h ++++ b/patchlevel.h +@@ -135,0 +136 @@ ++ ,"Backport of perl5 git 6f87f404fa51739971a4068da1f11443024f3fc4"