From 53f91a4caa9c1c1f4570a65f34c77f6aa329842d Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 16:32:47 -0700 Subject: [PATCH 1/8] removed z_const --- .dev/CRAN_Release.cmd | 3 +++ src/fwrite.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.dev/CRAN_Release.cmd b/.dev/CRAN_Release.cmd index 863b023b5c..9ae5d18cf0 100644 --- a/.dev/CRAN_Release.cmd +++ b/.dev/CRAN_Release.cmd @@ -48,6 +48,9 @@ grep "Rprintf" ./src/init.c grep "nearest *=" ./src/*.c # none grep "class *=" ./src/*.c # quite a few but none global +# ensure no use of z_const from zconf.h; #3939 +grep "z_const" ./src/* # none other than the comment + # No undefined type punning of the form: *(long long *)&REAL(column)[i] # Failed clang 3.9.1 -O3 due to this, I think. grep "&REAL" ./src/*.c diff --git a/src/fwrite.c b/src/fwrite.c index 0ed9f2184a..387cca38ff 100644 --- a/src/fwrite.c +++ b/src/fwrite.c @@ -566,7 +566,7 @@ int compressbuff(z_stream *stream, void* dest, size_t *destLen, const void* sour stream->next_out = dest; stream->avail_out = *destLen; - stream->next_in = (z_const Bytef *)source; + stream->next_in = (const Bytef *)source; // don't use z_const anywhere; #3939 stream->avail_in = sourceLen; err = deflate(stream, Z_FINISH); From cc4fa28e84fdf66fe7f9afd7acae46c9135b4fb6 Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 20:28:49 -0700 Subject: [PATCH 2/8] added src/configure, and changed "zlib.h" to --- src/configure | 32 ++++++++++++++++++++++++++++++++ src/fwrite.c | 4 ++-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100755 src/configure diff --git a/src/configure b/src/configure new file mode 100755 index 0000000000..6075186d38 --- /dev/null +++ b/src/configure @@ -0,0 +1,32 @@ +#! /bin/sh +# Let's keep this simple. If pkg-config is available, use it. Otherwise print +# the helpful message to aid user if compilation does fail. Note 25 of R-exts: +# "[pkg-config] is available on the machines used to produce the CRAN binary packages" + +if ! hash pkg-config 2>/dev/null; then + echo "*** pkg-config is not installed. It is used to check that zlib is installed. Compilation" + echo "*** will now be attempted and if it works you can ignore this message. If it fails and" + echo "*** and you cannot install pkg-config, try: locate zlib.h zconf.h. If they are not found" + echo "*** then try installing zlib1g-dev (Debian/Ubuntu), zlib-devel (Fedora) or zlib (OSX)" + return 0 # now that the advice has been printed, exit with success and continue +fi + +if ! `pkg-config --exists zlib`; then + echo "pkg-config did not detect zlib is installed. Try installing:" + echo "* deb: zlib1g-dev (Debian, Ubuntu, ...)" + echo "* rpm: zlib-devel (Fedora, EPEL, ...)" + echo "* brew: zlib (OSX)" + return 1 # nothing more to do; zlib is required currently +fi + +version=`pkg-config --modversion zlib` +echo "zlib ${version} is available ok" + +lib=`pkg-config --libs zlib` +if test $lib != "-lz"; then + echo "zlib is linked using ${lib} not the expected standard -lz. Please report to data.table issue tracker on GitHub." + return 1 +fi + +return 0 + diff --git a/src/fwrite.c b/src/fwrite.c index 387cca38ff..e58b96bb71 100644 --- a/src/fwrite.c +++ b/src/fwrite.c @@ -1,4 +1,3 @@ -#include "fwriteLookups.h" #include #include // for access() #include @@ -7,6 +6,7 @@ #include // isfinite, isnan #include // abs #include // strlen, strerror +#include // for compression to .gz #ifdef WIN32 #include @@ -19,8 +19,8 @@ #define CLOSE close #endif -#include "zlib.h" // for writing gzip file #include "myomp.h" +#include "fwriteLookups.h" #include "fwrite.h" #define NUM_SF 15 From d1896a6057c61ba954d07dab9b29852b8d238a59 Mon Sep 17 00:00:00 2001 From: Michael Chirico Date: Wed, 9 Oct 2019 11:40:32 +0800 Subject: [PATCH 3/8] only search c files --- .dev/CRAN_Release.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dev/CRAN_Release.cmd b/.dev/CRAN_Release.cmd index 9ae5d18cf0..1665edde7c 100644 --- a/.dev/CRAN_Release.cmd +++ b/.dev/CRAN_Release.cmd @@ -49,7 +49,7 @@ grep "nearest *=" ./src/*.c # none grep "class *=" ./src/*.c # quite a few but none global # ensure no use of z_const from zconf.h; #3939 -grep "z_const" ./src/* # none other than the comment +grep "z_const" ./src/*.c # none other than the comment # No undefined type punning of the form: *(long long *)&REAL(column)[i] # Failed clang 3.9.1 -O3 due to this, I think. From 5c12ce66878c6fa9c79ecc16f5f8d812108578ba Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 21:06:48 -0700 Subject: [PATCH 4/8] const results in 'discards const' warning, so removed --- .appveyor.yml | 2 +- .dev/CRAN_Release.cmd | 1 + src/configure => configure | 0 src/fwrite.c | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) rename src/configure => configure (100%) diff --git a/.appveyor.yml b/.appveyor.yml index 19ed1dd6e4..ea8711a2b6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -30,7 +30,7 @@ environment: - R_VERSION: release # the single Windows.zip binary (both 32bit/64bit) that users following dev version of installation instructions should click -# - R_VERSION: devel # temporarily off pending R-devel fix of slowdown from 5min to 20-30min (slowing down data.table dev) in July 2019 possibly due to r76734. + - R_VERSION: devel # temporarily off pending R-devel fix of slowdown from 5min to 20-30min (slowing down data.table dev) in July 2019 possibly due to r76734. before_build: - cmd: ECHO no Revision metadata added to DESCRIPTION diff --git a/.dev/CRAN_Release.cmd b/.dev/CRAN_Release.cmd index 9ae5d18cf0..ede21bfd56 100644 --- a/.dev/CRAN_Release.cmd +++ b/.dev/CRAN_Release.cmd @@ -152,6 +152,7 @@ system.time(test.data.table(script="*.Rraw")) # apx 8h = froll 3h + nafill 1m + # Upload to win-builder: release, dev & old-release # Turn on Travis OSX; it's off in dev until it's added to GLCI (#3326) as it adds 17min after 11min Linux. +# Turn on r-devel in Appveyor; it may be off in dev for similar dev cycle speed reasons ############################################### diff --git a/src/configure b/configure similarity index 100% rename from src/configure rename to configure diff --git a/src/fwrite.c b/src/fwrite.c index e58b96bb71..e5a7b68190 100644 --- a/src/fwrite.c +++ b/src/fwrite.c @@ -566,7 +566,7 @@ int compressbuff(z_stream *stream, void* dest, size_t *destLen, const void* sour stream->next_out = dest; stream->avail_out = *destLen; - stream->next_in = (const Bytef *)source; // don't use z_const anywhere; #3939 + stream->next_in = (Bytef *)source; // don't use z_const anywhere; #3939 stream->avail_in = sourceLen; err = deflate(stream, Z_FINISH); From 6e1a5be39536e88df88d33b84dea75fd9c0c32f1 Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 21:33:42 -0700 Subject: [PATCH 5/8] exit not return for osx --- configure | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configure b/configure index 6075186d38..dc663810b9 100755 --- a/configure +++ b/configure @@ -8,7 +8,7 @@ if ! hash pkg-config 2>/dev/null; then echo "*** will now be attempted and if it works you can ignore this message. If it fails and" echo "*** and you cannot install pkg-config, try: locate zlib.h zconf.h. If they are not found" echo "*** then try installing zlib1g-dev (Debian/Ubuntu), zlib-devel (Fedora) or zlib (OSX)" - return 0 # now that the advice has been printed, exit with success and continue + exit 0 # now that the advice has been printed, exit with success and continue fi if ! `pkg-config --exists zlib`; then @@ -16,7 +16,7 @@ if ! `pkg-config --exists zlib`; then echo "* deb: zlib1g-dev (Debian, Ubuntu, ...)" echo "* rpm: zlib-devel (Fedora, EPEL, ...)" echo "* brew: zlib (OSX)" - return 1 # nothing more to do; zlib is required currently + exit 1 # nothing more to do; zlib is required currently fi version=`pkg-config --modversion zlib` @@ -25,8 +25,8 @@ echo "zlib ${version} is available ok" lib=`pkg-config --libs zlib` if test $lib != "-lz"; then echo "zlib is linked using ${lib} not the expected standard -lz. Please report to data.table issue tracker on GitHub." - return 1 + exit 1 fi -return 0 +exit 0 From 554587cd37c4a217392b0a2e038cfedbe808fde1 Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 21:52:27 -0700 Subject: [PATCH 6/8] remove a space --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index dc663810b9..76024fcfe4 100755 --- a/configure +++ b/configure @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh # Let's keep this simple. If pkg-config is available, use it. Otherwise print # the helpful message to aid user if compilation does fail. Note 25 of R-exts: # "[pkg-config] is available on the machines used to produce the CRAN binary packages" From c627c146733cd82236556b6bbbe9613b2b452aaf Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 22:11:01 -0700 Subject: [PATCH 7/8] search .c and .h files --- .dev/CRAN_Release.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dev/CRAN_Release.cmd b/.dev/CRAN_Release.cmd index 35c24e7c5d..543ea1c339 100644 --- a/.dev/CRAN_Release.cmd +++ b/.dev/CRAN_Release.cmd @@ -49,7 +49,7 @@ grep "nearest *=" ./src/*.c # none grep "class *=" ./src/*.c # quite a few but none global # ensure no use of z_const from zconf.h; #3939 -grep "z_const" ./src/*.c # none other than the comment +grep "z_const" ./src/*.[hc] # none other than the comment # No undefined type punning of the form: *(long long *)&REAL(column)[i] # Failed clang 3.9.1 -O3 due to this, I think. From 06f5953eb82c43f8441c6e87b5e3c6a85146a694 Mon Sep 17 00:00:00 2001 From: mattdowle Date: Tue, 8 Oct 2019 22:27:22 -0700 Subject: [PATCH 8/8] news item --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index db035bd941..fb10f4f81a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,8 @@ 1. `shift()` on a `nanotime` with the default `fill=NA` now fills a `nanotime` missing value correctly, [#3945](https://github.com/Rdatatable/data.table/issues/3945). Thanks to @mschubmehl for reporting and fixing in PR [#3942](https://github.com/Rdatatable/data.table/pull/3942). +2. Compilation failed on CRAN's MacOS due to an older version of `zlib.h/zconf.h` which did not have `z_const` defined, [#3939](https://github.com/Rdatatable/data.table/issues/3939). Other open-source projects unrelated to R have experienced this problem on MacOS too. We have followed the common practice of removing `z_const` to support the older `zlib` versions, and data.table's release procedures have gained a `grep` to ensure `z_const` isn't used again by accident in future. The library `zlib` is used for `fwrite`'s new feature of multithreaded compression on-the-fly; see item 3 of 1.12.4 below. + ## NOTES