From 6934ab45ed2eac3539b5da0095bd51f23cab8799 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 6 Aug 2025 09:16:53 -0700 Subject: [PATCH 1/2] Update to zlib-ng 2.2.5 --- src/native/external/zlib-ng-version.txt | 6 +- src/native/external/zlib-ng/.gitignore | 4 + src/native/external/zlib-ng/CMakeLists.txt | 62 ++++++++-- src/native/external/zlib-ng/Makefile.in | 2 +- .../external/zlib-ng/arch/riscv/Makefile.in | 63 ++++++++++ .../zlib-ng/arch/riscv/chunkset_rvv.c | 2 +- .../zlib-ng/arch/s390/s390_functions.h | 2 +- .../zlib-ng/arch/x86/chunkset_avx512.c | 7 ++ .../external/zlib-ng/arch/x86/x86_features.c | 9 +- .../external/zlib-ng/arch/x86/x86_functions.h | 30 +++-- .../external/zlib-ng/cmake/detect-arch.cmake | 1 - .../zlib-ng/cmake/detect-intrinsics.cmake | 14 ++- .../zlib-ng/cmake/toolchain-riscv-clang.cmake | 16 +++ .../zlib-ng/cmake/toolchain-riscv.cmake | 41 +++---- src/native/external/zlib-ng/configure | 112 ++++++++++++++++-- src/native/external/zlib-ng/deflate.h | 2 +- src/native/external/zlib-ng/deflate_medium.c | 45 ++++--- src/native/external/zlib-ng/zlib-ng.h.in | 6 +- src/native/external/zlib-ng/zlib.h.in | 6 +- src/native/external/zlib-ng/zlib.pc.cmakein | 1 + src/native/external/zlib-ng/zlib.pc.in | 1 + src/native/external/zlib-ng/zutil.c | 2 +- 22 files changed, 336 insertions(+), 98 deletions(-) create mode 100644 src/native/external/zlib-ng/arch/riscv/Makefile.in create mode 100644 src/native/external/zlib-ng/cmake/toolchain-riscv-clang.cmake diff --git a/src/native/external/zlib-ng-version.txt b/src/native/external/zlib-ng-version.txt index 198e16578e1f7d..8a1e4932e7d8af 100644 --- a/src/native/external/zlib-ng-version.txt +++ b/src/native/external/zlib-ng-version.txt @@ -1,7 +1,7 @@ -v2.2.4 -860e4cff7917d93f54f5d7f0bc1d0e8b1a3cb988 +v2.2.5 +5f0a27330f90da761864e02004acb3cbdc2c84a5 -https://github.com/zlib-ng/zlib-ng/releases/tag/2.2.4 +https://github.com/zlib-ng/zlib-ng/pull/1943 We have removed the following folders from our local copy as these files are not needed for our compilation: diff --git a/src/native/external/zlib-ng/.gitignore b/src/native/external/zlib-ng/.gitignore index 71b9e0747f88a8..0a81b3ade206da 100644 --- a/src/native/external/zlib-ng/.gitignore +++ b/src/native/external/zlib-ng/.gitignore @@ -103,3 +103,7 @@ gzread.c MinSizeRel RelWithDebInfo /_deps/googletest* + +# Test datasets +test/data/corpora/ +test/data/local/ diff --git a/src/native/external/zlib-ng/CMakeLists.txt b/src/native/external/zlib-ng/CMakeLists.txt index 367413062fc3da..571979eb4d6987 100644 --- a/src/native/external/zlib-ng/CMakeLists.txt +++ b/src/native/external/zlib-ng/CMakeLists.txt @@ -47,6 +47,15 @@ include(CMakeDependentOption) include(CMakePackageConfigHelpers) include(FeatureSummary) +# We need to enable C++ before trying to check for coverage +option(WITH_GTEST "Build gtest_zlib" ON) +option(WITH_FUZZERS "Build test/fuzz" OFF) +option(WITH_BENCHMARKS "Build test/benchmarks" OFF) + +if(WITH_GTEST OR WITH_FUZZERS OR WITH_BENCHMARKS) + enable_language(CXX) +endif() + include(cmake/detect-arch.cmake) include(cmake/detect-install-dirs.cmake) include(cmake/detect-coverage.cmake) @@ -79,9 +88,6 @@ option(WITH_GZFILEOP "Compile with support for gzFile related functions" ON) option(ZLIB_COMPAT "Compile with zlib compatible API" OFF) option(ZLIB_ENABLE_TESTS "Build test binaries" ON) option(ZLIBNG_ENABLE_TESTS "Test zlib-ng specific API" ON) -option(WITH_GTEST "Build gtest_zlib" ON) -option(WITH_FUZZERS "Build test/fuzz" OFF) -option(WITH_BENCHMARKS "Build test/benchmarks" OFF) option(WITH_BENCHMARK_APPS "Build application benchmarks" OFF) option(WITH_OPTIM "Build with optimisation" ON) option(WITH_REDUCED_MEM "Reduced memory usage for special cases (reduces performance)" OFF) @@ -362,6 +368,21 @@ if(MSVC) endif() endif() +# +# Additional flags for features checking +# +set(ADDITIONAL_CHECK_FLAGS ) +if(APPLE) + check_c_compiler_flag(-Werror=unguarded-availability HAVE_W_ERROR_UNGUARDED_AVAILABILITY) + if(HAVE_W_ERROR_UNGUARDED_AVAILABILITY) + set(ADDITIONAL_CHECK_FLAGS "${ADDITIONAL_CHECK_FLAGS} -Werror=unguarded-availability") + endif() + check_c_compiler_flag(-Werror=unguarded-availability-new HAVE_W_ERROR_UNGUARDED_AVAILABILITY_NEW) + if(HAVE_W_ERROR_UNGUARDED_AVAILABILITY_NEW) + set(ADDITIONAL_CHECK_FLAGS "${ADDITIONAL_CHECK_FLAGS} -Werror=unguarded-availability-new") + endif() +endif() + # # Check for standard/system includes # @@ -407,28 +428,42 @@ set(CMAKE_REQUIRED_DEFINITIONS) # clear variable # # Check for fseeko and other optional functions # +set(CMAKE_REQUIRED_FLAGS "${ADDITIONAL_CHECK_FLAGS}") check_function_exists(fseeko HAVE_FSEEKO) if(NOT HAVE_FSEEKO) add_definitions(-DNO_FSEEKO) endif() +set(CMAKE_REQUIRED_FLAGS) +set(CMAKE_REQUIRED_FLAGS "${ADDITIONAL_CHECK_FLAGS}") check_function_exists(strerror HAVE_STRERROR) if(NOT HAVE_STRERROR) add_definitions(-DNO_STRERROR) endif() +set(CMAKE_REQUIRED_FLAGS) -set(CMAKE_REQUIRED_DEFINITIONS -D_POSIX_C_SOURCE=200112L) +# +# Check for aligned memory allocation support: POSIX +# +set(CMAKE_REQUIRED_DEFINITIONS -D_POSIX_C_SOURCE=200112L -D_ISOC11_SOURCE=1) +set(CMAKE_REQUIRED_FLAGS "${ADDITIONAL_CHECK_FLAGS}") check_symbol_exists(posix_memalign stdlib.h HAVE_POSIX_MEMALIGN) if(HAVE_POSIX_MEMALIGN) add_definitions(-DHAVE_POSIX_MEMALIGN) endif() +set(CMAKE_REQUIRED_FLAGS) set(CMAKE_REQUIRED_DEFINITIONS) -set(CMAKE_REQUIRED_DEFINITIONS -D_ISOC11_SOURCE=1) +# +# Check for aligned memory allocation support: C11 +# +set(CMAKE_REQUIRED_DEFINITIONS -D_POSIX_C_SOURCE=200112L -D_ISOC11_SOURCE=1) +set(CMAKE_REQUIRED_FLAGS "${ADDITIONAL_CHECK_FLAGS}") check_symbol_exists(aligned_alloc stdlib.h HAVE_ALIGNED_ALLOC) if(HAVE_ALIGNED_ALLOC) add_definitions(-DHAVE_ALIGNED_ALLOC) endif() +set(CMAKE_REQUIRED_FLAGS) set(CMAKE_REQUIRED_DEFINITIONS) if(WITH_SANITIZER STREQUAL "Address") @@ -577,13 +612,6 @@ if(MSVC) add_definitions(-D_CRT_NONSTDC_NO_DEPRECATE) endif() -if(BASEARCH_X86_FOUND) - # FORCE_SSE2 option will only be shown if HAVE_SSE2_INTRIN is true - if("${ARCH}" MATCHES "i[3-6]86") - cmake_dependent_option(FORCE_SSE2 "Always assume CPU is SSE2 capable" OFF "HAVE_SSE2_INTRIN" OFF) - endif() -endif() - # # Enable deflate_quick at level 1 # @@ -902,6 +930,10 @@ if(WITH_OPTIM) endif() if(WITH_SSE2) check_sse2_intrinsics() + # FORCE_SSE2 option will only be shown if HAVE_SSE2_INTRIN is true + if("${ARCH}" MATCHES "i[3-6]86") + cmake_dependent_option(FORCE_SSE2 "Always assume CPU is SSE2 capable" OFF "HAVE_SSE2_INTRIN" OFF) + endif() if(HAVE_SSE2_INTRIN) add_definitions(-DX86_SSE2) set(SSE2_SRCS ${ARCHDIR}/chunkset_sse2.c ${ARCHDIR}/compare256_sse2.c ${ARCHDIR}/slide_hash_sse2.c) @@ -1064,6 +1096,12 @@ else() set(PC_INC_INSTALL_DIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") endif() +if(IS_ABSOLUTE "${CMAKE_INSTALL_BINDIR}") + set(PC_BIN_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}") +else() + set(PC_BIN_INSTALL_DIR "\${exec_prefix}/${CMAKE_INSTALL_BINDIR}") +endif() + if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}") else() diff --git a/src/native/external/zlib-ng/Makefile.in b/src/native/external/zlib-ng/Makefile.in index 1493124bbf4267..2f89c56b060c7d 100644 --- a/src/native/external/zlib-ng/Makefile.in +++ b/src/native/external/zlib-ng/Makefile.in @@ -30,7 +30,7 @@ LDSHARED=$(CC) LDSHAREDFLAGS=-shared LDVERSIONSCRIPT= -VER=2.2.4 +VER=2.2.5 VER1=2 STATICLIB=$(LIBNAME1).a diff --git a/src/native/external/zlib-ng/arch/riscv/Makefile.in b/src/native/external/zlib-ng/arch/riscv/Makefile.in new file mode 100644 index 00000000000000..8f253066861414 --- /dev/null +++ b/src/native/external/zlib-ng/arch/riscv/Makefile.in @@ -0,0 +1,63 @@ +# Makefile for zlib-ng +# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler +# Copyright (C) 2024 Hans Kristian Rosbach +# Copyright (C) 2025 Yin Tong , ByteDance +# For conditions of distribution and use, see copyright notice in zlib.h + +CC= +CFLAGS= +SFLAGS= +INCLUDES= +SUFFIX= + +SRCDIR=. +SRCTOP=../.. +TOPDIR=$(SRCTOP) + +RVVFLAG= + +all: \ + riscv_features.o riscv_features.lo \ + adler32_rvv.o adler32_rvv.lo \ + chunkset_rvv.o chunkset_rvv.lo \ + compare256_rvv.o compare256_rvv.lo \ + slide_hash_rvv.o slide_hash_rvv.lo + +riscv_features.o: $(SRCDIR)/riscv_features.c + $(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/riscv_features.c + +riscv_features.lo: $(SRCDIR)/riscv_features.c + $(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/riscv_features.c + +adler32_rvv.o: $(SRCDIR)/adler32_rvv.c + $(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_rvv.c + +adler32_rvv.lo: $(SRCDIR)/adler32_rvv.c + $(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/adler32_rvv.c + +chunkset_rvv.o: $(SRCDIR)/chunkset_rvv.c + $(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_rvv.c + +chunkset_rvv.lo: $(SRCDIR)/chunkset_rvv.c + $(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/chunkset_rvv.c + +compare256_rvv.o: $(SRCDIR)/compare256_rvv.c + $(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_rvv.c + +compare256_rvv.lo: $(SRCDIR)/compare256_rvv.c + $(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_rvv.c + +slide_hash_rvv.o: $(SRCDIR)/slide_hash_rvv.c + $(CC) $(CFLAGS) $(RVVFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_rvv.c + +slide_hash_rvv.lo: $(SRCDIR)/slide_hash_rvv.c + $(CC) $(SFLAGS) $(RVVFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/slide_hash_rvv.c + +mostlyclean: clean +clean: + rm -f *.o *.lo *~ + rm -rf objs + rm -f *.gcda *.gcno *.gcov + +distclean: clean + rm -f Makefile diff --git a/src/native/external/zlib-ng/arch/riscv/chunkset_rvv.c b/src/native/external/zlib-ng/arch/riscv/chunkset_rvv.c index ee43bde2f71d81..e0915dfc9666e5 100644 --- a/src/native/external/zlib-ng/arch/riscv/chunkset_rvv.c +++ b/src/native/external/zlib-ng/arch/riscv/chunkset_rvv.c @@ -92,7 +92,7 @@ static inline uint8_t* CHUNKCOPY(uint8_t *out, uint8_t const *from, unsigned len from += align; len -= align; ptrdiff_t dist = out - from; - if (dist >= len) { + if (dist < 0 || dist >= len) { memcpy(out, from, len); out += len; from += len; diff --git a/src/native/external/zlib-ng/arch/s390/s390_functions.h b/src/native/external/zlib-ng/arch/s390/s390_functions.h index e6855cd7011cf4..2c43a50627fa1d 100644 --- a/src/native/external/zlib-ng/arch/s390/s390_functions.h +++ b/src/native/external/zlib-ng/arch/s390/s390_functions.h @@ -20,7 +20,7 @@ uint32_t crc32_s390_vx(uint32_t crc, const uint8_t *buf, size_t len); #ifdef DISABLE_RUNTIME_CPU_DETECTION # if defined(S390_CRC32_VX) && defined(__zarch__) && __ARCH__ >= 11 && defined(__VX__) # undef native_crc32 -# define native_crc32 = crc32_s390_vx +# define native_crc32 crc32_s390_vx # endif #endif diff --git a/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c b/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c index 9d28d33d5eccec..db8c1eb25e6420 100644 --- a/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c +++ b/src/native/external/zlib-ng/arch/x86/chunkset_avx512.c @@ -98,6 +98,10 @@ static inline uint8_t* CHUNKCOPY(uint8_t *out, uint8_t const *from, unsigned len return out; } +/* MSVC compiler decompression bug when optimizing for size */ +#if defined(_MSC_VER) && _MSC_VER < 1943 +# pragma optimize("", off) +#endif static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t dist) { lut_rem_pair lut_rem = perm_idx_lut[dist - 3]; __m256i ret_vec; @@ -128,6 +132,9 @@ static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t return ret_vec; } +#if defined(_MSC_VER) && _MSC_VER < 1943 +# pragma optimize("", on) +#endif static inline void storehalfchunk(uint8_t *out, halfchunk_t *chunk) { _mm_storeu_si128((__m128i *)out, *chunk); diff --git a/src/native/external/zlib-ng/arch/x86/x86_features.c b/src/native/external/zlib-ng/arch/x86/x86_features.c index 9491a00730831f..5efb1787563e1a 100644 --- a/src/native/external/zlib-ng/arch/x86/x86_features.c +++ b/src/native/external/zlib-ng/arch/x86/x86_features.c @@ -86,19 +86,17 @@ void Z_INTERNAL x86_check_features(struct x86_cpu_features *features) { } if (maxbasic >= 7) { + // Reference: https://software.intel.com/sites/default/files/article/405250/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family.pdf cpuidex(7, 0, &eax, &ebx, &ecx, &edx); - // check BMI1 bit - // Reference: https://software.intel.com/sites/default/files/article/405250/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family.pdf - features->has_vpclmulqdq = ecx & 0x400; + // check BMI2 bit + features->has_bmi2 = ebx & 0x8; // check AVX2 bit if the OS supports saving YMM registers if (features->has_os_save_ymm) { features->has_avx2 = ebx & 0x20; } - features->has_bmi2 = ebx & 0x8; - // check AVX512 bits if the OS supports saving ZMM registers if (features->has_os_save_zmm) { features->has_avx512f = ebx & 0x00010000; @@ -112,6 +110,7 @@ void Z_INTERNAL x86_check_features(struct x86_cpu_features *features) { features->has_avx512_common = features->has_avx512f && features->has_avx512dq && features->has_avx512bw \ && features->has_avx512vl && features->has_bmi2; features->has_avx512vnni = ecx & 0x800; + features->has_vpclmulqdq = ecx & 0x400; } } } diff --git a/src/native/external/zlib-ng/arch/x86/x86_functions.h b/src/native/external/zlib-ng/arch/x86/x86_functions.h index fc62daeae15af6..a5c2d58da7124b 100644 --- a/src/native/external/zlib-ng/arch/x86/x86_functions.h +++ b/src/native/external/zlib-ng/arch/x86/x86_functions.h @@ -90,7 +90,7 @@ uint32_t crc32_vpclmulqdq(uint32_t crc32, const uint8_t *buf, size_t len); # undef native_longest_match_slow # define native_longest_match_slow longest_match_slow_sse2 # endif -#endif +# endif // X86 - SSSE3 # if defined(X86_SSSE3) && defined(__SSSE3__) # undef native_adler32 @@ -105,21 +105,20 @@ uint32_t crc32_vpclmulqdq(uint32_t crc32, const uint8_t *buf, size_t len); # undef native_adler32_fold_copy # define native_adler32_fold_copy adler32_fold_copy_sse42 # endif - // X86 - PCLMUL -#if defined(X86_PCLMULQDQ_CRC) && defined(__PCLMUL__) -# undef native_crc32 -# define native_crc32 crc32_pclmulqdq -# undef native_crc32_fold -# define native_crc32_fold crc32_fold_pclmulqdq -# undef native_crc32_fold_copy -# define native_crc32_fold_copy crc32_fold_pclmulqdq_copy -# undef native_crc32_fold_final -# define native_crc32_fold_final crc32_fold_pclmulqdq_final -# undef native_crc32_fold_reset -# define native_crc32_fold_reset crc32_fold_pclmulqdq_reset -#endif -// X86 - AVX +# if defined(X86_PCLMULQDQ_CRC) && defined(__PCLMUL__) +# undef native_crc32 +# define native_crc32 crc32_pclmulqdq +# undef native_crc32_fold +# define native_crc32_fold crc32_fold_pclmulqdq +# undef native_crc32_fold_copy +# define native_crc32_fold_copy crc32_fold_pclmulqdq_copy +# undef native_crc32_fold_final +# define native_crc32_fold_final crc32_fold_pclmulqdq_final +# undef native_crc32_fold_reset +# define native_crc32_fold_reset crc32_fold_pclmulqdq_reset +# endif +// X86 - AVX2 # if defined(X86_AVX2) && defined(__AVX2__) # undef native_adler32 # define native_adler32 adler32_avx2 @@ -142,7 +141,6 @@ uint32_t crc32_vpclmulqdq(uint32_t crc32, const uint8_t *buf, size_t len); # define native_longest_match_slow longest_match_slow_avx2 # endif # endif - // X86 - AVX512 (F,DQ,BW,Vl) # if defined(X86_AVX512) && defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VL__) # undef native_adler32 diff --git a/src/native/external/zlib-ng/cmake/detect-arch.cmake b/src/native/external/zlib-ng/cmake/detect-arch.cmake index dfdc6013cea78b..f80ee4a05a72cf 100644 --- a/src/native/external/zlib-ng/cmake/detect-arch.cmake +++ b/src/native/external/zlib-ng/cmake/detect-arch.cmake @@ -22,7 +22,6 @@ elseif(CMAKE_CROSSCOMPILING) set(ARCH ${CMAKE_C_COMPILER_TARGET}) else() # Let preprocessor parse archdetect.c and raise an error containing the arch identifier - enable_language(C) try_run( run_result_unused compile_result_unused diff --git a/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake b/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake index b96ac0a4428a9f..e953346f3fb4b8 100644 --- a/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake +++ b/src/native/external/zlib-ng/cmake/detect-intrinsics.cmake @@ -137,11 +137,15 @@ macro(check_avx512vnni_intrinsics) set(CMAKE_REQUIRED_FLAGS "${AVX512VNNIFLAG} ${NATIVEFLAG} ${ZNOLTOFLAG}") check_c_source_compiles( "#include - __m512i f(__m512i x, __m512i y) { - __m512i z = _mm512_setzero_epi32(); - return _mm512_dpbusd_epi32(z, x, y); - } - int main(void) { return 0; }" + int main(void) { + const __m512i z512 = _mm512_setzero_si512(); + const __m256i z256 = _mm256_setzero_si256(); + volatile __m512i r512 = _mm512_dpbusd_epi32(z512, z512, z512); + volatile __m256i r256 = _mm256_dpbusd_epi32(z256, z256, z256); + (void)r512; + (void)r256; + return 0; + }" HAVE_AVX512VNNI_INTRIN ) set(CMAKE_REQUIRED_FLAGS) diff --git a/src/native/external/zlib-ng/cmake/toolchain-riscv-clang.cmake b/src/native/external/zlib-ng/cmake/toolchain-riscv-clang.cmake new file mode 100644 index 00000000000000..71fa7f4cf30ac7 --- /dev/null +++ b/src/native/external/zlib-ng/cmake/toolchain-riscv-clang.cmake @@ -0,0 +1,16 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR riscv64) +set(CMAKE_SYSTEM_VERSION 1) + +set(CMAKE_C_COMPILER clang) +set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu) +set(CMAKE_CXX_COMPILER clang++) +set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu) + +set(CMAKE_CROSSCOMPILING TRUE) +set(CMAKE_CROSSCOMPILING_EMULATOR qemu-riscv64 -cpu rv64,zba=true,zbb=true,zbc=true,zbs=true,v=true,vlen=512,elen=64,vext_spec=v1.0 -L /usr/${CMAKE_C_COMPILER_TARGET}/) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/src/native/external/zlib-ng/cmake/toolchain-riscv.cmake b/src/native/external/zlib-ng/cmake/toolchain-riscv.cmake index 9cf8fdb7fec2fd..241f2a7d711947 100644 --- a/src/native/external/zlib-ng/cmake/toolchain-riscv.cmake +++ b/src/native/external/zlib-ng/cmake/toolchain-riscv.cmake @@ -1,28 +1,25 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR riscv64) +set(CMAKE_SYSTEM_VERSION 1) + +set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu) +set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu) + set(CMAKE_CROSSCOMPILING TRUE) -set(CMAKE_SYSTEM_NAME "Linux") -set(CMAKE_SYSTEM_PROCESSOR "riscv64") +set(CMAKE_CROSSCOMPILING_EMULATOR qemu-riscv64 -cpu rv64,zba=true,zbb=true,zbc=true,zbs=true,v=true,vlen=512,elen=64,vext_spec=v1.0 -L /usr/${CMAKE_C_COMPILER_TARGET}/) -# Avoid to use system path for cross-compile -set(CMAKE_FIND_USE_CMAKE_SYSTEM_PATH FALSE) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) -set(TOOLCHAIN_PATH "" CACHE STRING "The toolchain path.") -if(NOT TOOLCHAIN_PATH) - set(TOOLCHAIN_PATH ${CMAKE_SOURCE_DIR}/prebuilt-riscv-toolchain-qemu/riscv-clang) +find_program(C_COMPILER_FULL_PATH NAMES ${CMAKE_C_COMPILER_TARGET}-gcc) +if(NOT C_COMPILER_FULL_PATH) + message(FATAL_ERROR "Cross-compiler for ${CMAKE_C_COMPILER_TARGET} not found") endif() +set(CMAKE_C_COMPILER ${C_COMPILER_FULL_PATH}) -set(TOOLCHAIN_PREFIX "riscv64-unknown-linux-gnu-" CACHE STRING "The toolchain prefix.") -set(QEMU_PATH "" CACHE STRING "The qemu path.") -if(NOT QEMU_PATH) - set(QEMU_PATH ${CMAKE_SOURCE_DIR}/prebuilt-riscv-toolchain-qemu/riscv-qemu/bin/qemu-riscv64) +find_program(CXX_COMPILER_FULL_PATH NAMES g++-${CMAKE_CXX_COMPILER_TARGET} ${CMAKE_CXX_COMPILER_TARGET}-g++) +if(CXX_COMPILER_FULL_PATH) + set(CMAKE_CXX_COMPILER ${CXX_COMPILER_FULL_PATH}) endif() - -# toolchain setting -set(CMAKE_C_COMPILER "${TOOLCHAIN_PATH}/bin/${TOOLCHAIN_PREFIX}clang") -set(CMAKE_CXX_COMPILER "${TOOLCHAIN_PATH}/bin/${TOOLCHAIN_PREFIX}clang++") - -# disable auto-vectorizer -add_compile_options(-fno-vectorize -fno-slp-vectorize) - -# emulator setting -set(QEMU_CPU_OPTION "rv64,zba=true,zbb=true,zbc=true,zbs=true,v=true,vlen=512,elen=64,vext_spec=v1.0") -set(CMAKE_CROSSCOMPILING_EMULATOR ${QEMU_PATH} -cpu ${QEMU_CPU_OPTION} -L ${TOOLCHAIN_PATH}/sysroot/) diff --git a/src/native/external/zlib-ng/configure b/src/native/external/zlib-ng/configure index 3487d092bc9dfe..832b42ac102835 100644 --- a/src/native/external/zlib-ng/configure +++ b/src/native/external/zlib-ng/configure @@ -98,6 +98,7 @@ buildaltivec=1 buildpower8=1 buildpower9=1 buildneon=1 +buildrvv=1 builddfltccdeflate=0 builddfltccinflate=0 buildcrc32vx=1 @@ -116,6 +117,7 @@ vpclmulflag="-mvpclmulqdq -mavx512f" xsaveflag="-mxsave" acleflag= neonflag= +rvvflag= armv6flag= noltoflag="-fno-lto" vgfmaflag="-march=z13" @@ -169,6 +171,7 @@ case "$1" in echo ' [--without-acle] Compiles without ARM C Language Extensions' | tee -a configure.log echo ' [--without-neon] Compiles without ARM Neon SIMD instruction set' | tee -a configure.log echo ' [--without-armv6] Compiles without ARMv6 SIMD instruction set' | tee -a configure.log + echo ' [--without-rvv] Compiles without RVV instruction set' | tee -a configure.log echo ' [--without-altivec] Compiles without PPC AltiVec support' | tee -a configure.log echo ' [--without-power8] Compiles without Power8 instruction set' | tee -a configure.log echo ' [--with-dfltcc-deflate] Use DEFLATE CONVERSION CALL instruction for compression on IBM Z' | tee -a configure.log @@ -202,6 +205,7 @@ case "$1" in --without-neon) buildneon=0; shift ;; --without-armv6) buildarmv6=0; shift ;; --without-altivec) buildaltivec=0 ; shift ;; + --without-rvv) buildrvv=0 ; shift ;; --without-power8) buildpower8=0 ; shift ;; --without-power9) buildpower9=0 ; shift ;; --with-dfltcc-deflate) builddfltccdeflate=1; shift ;; @@ -339,6 +343,8 @@ if test "$gcc" -eq 1 && ($cc $CFLAGS -c $test.c) >> configure.log 2>&1; then ARCH=powerpc64 ;; powerpc64le | ppc64le) ARCH=powerpc64le ;; + riscv64) + ARCH=riscv64 ;; esac CFLAGS="-O2 ${CFLAGS}" if test -n "${ARCHS}"; then @@ -628,6 +634,32 @@ fi echo >> configure.log +ADDITIONAL_CHECK_FLAGS="" + +# Check for -Werror=unguarded-availability compiler support +echo "" > test.c + cat > $test.c <> configure.log 2>&1; then + echo "Checking for -Werror=unguarded-availability... Yes." | tee -a configure.log + ADDITIONAL_CHECK_FLAGS="$ADDITIONAL_CHECK_FLAGS -Werror=unguarded-availability" +else + echo "Checking for -Werror=unguarded-availability... No." | tee -a configure.log +fi + +# Check for -Werror=unguarded-availability-new compiler support +echo "" > test.c + cat > $test.c <> configure.log 2>&1; then + echo "Checking for -Werror=unguarded-availability-new... Yes." | tee -a configure.log + ADDITIONAL_CHECK_FLAGS="$ADDITIONAL_CHECK_FLAGS -Werror=unguarded-availability-new" +else + echo "Checking for -Werror=unguarded-availability-new... No." | tee -a configure.log +fi + # check for version script support cat > $test.c <> configure.log cat > $test.c < int main(void) { void *ptr = 0; @@ -705,7 +738,7 @@ int main(void) { return ret; } EOF -if try $CC $CFLAGS -o $test $test.c $LDSHAREDLIBC; then +if try $CC $CFLAGS $ADDITIONAL_CHECK_FLAGS -o $test $test.c $LDSHAREDLIBC; then echo "Checking for posix_memalign... Yes." | tee -a configure.log CFLAGS="${CFLAGS} -DHAVE_POSIX_MEMALIGN" SFLAGS="${SFLAGS} -DHAVE_POSIX_MEMALIGN" @@ -715,6 +748,7 @@ fi echo >> configure.log cat > $test.c < int main(void) { @@ -724,7 +758,7 @@ int main(void) { return 0; } EOF -if try $CC $CFLAGS -o $test $test.c $LDSHAREDLIBC; then +if try $CC $CFLAGS $ADDITIONAL_CHECK_FLAGS -o $test $test.c $LDSHAREDLIBC; then echo "Checking for aligned_alloc... Yes." | tee -a configure.log CFLAGS="${CFLAGS} -DHAVE_ALIGNED_ALLOC" SFLAGS="${SFLAGS} -DHAVE_ALIGNED_ALLOC" @@ -739,7 +773,7 @@ cat > $test.c < int main() { return strlen(strerror(errno)); } EOF -if try $CC $CFLAGS -o $test $test.c $LDSHAREDLIBC; then +if try $CC $CFLAGS $ADDITIONAL_CHECK_FLAGS -o $test $test.c $LDSHAREDLIBC; then echo "Checking for strerror... Yes." | tee -a configure.log else CFLAGS="${CFLAGS} -DNO_STRERROR" @@ -1106,11 +1140,15 @@ check_avx512vnni_intrinsics() { # Check whether compiler supports AVX512-VNNI intrinsics cat > $test.c << EOF #include -__m512i f(__m512i x, __m512i y) { - __m512i z = _mm512_setzero_epi32(); - return _mm512_dpbusd_epi32(z, x, y); +int main(void) { + const __m512i z512 = _mm512_setzero_si512(); + const __m256i z256 = _mm256_setzero_si256(); + volatile __m512i r512 = _mm512_dpbusd_epi32(z512, z512, z512); + volatile __m256i r256 = _mm256_dpbusd_epi32(z256, z256, z256); + (void)r512; + (void)r256; + return 0; } -int main(void) { return 0; } EOF if try ${CC} ${CFLAGS} ${avx512vnniflag} $test.c; then echo "Checking for AVX512VNNI intrinsics ... Yes." | tee -a configure.log @@ -1491,6 +1529,40 @@ EOF fi } +check_rvv_compiler_flag() { + cat > $test.c << EOF +int main() { return 0; } +EOF + if try $CC -c $CFLAGS -march=rv64gcv $test.c; then + echo "Check whether -march=rv64gcv works ... Yes." | tee -a configure.log + rvvflag="-march=rv64gcv" + else + echo "Check whether -march=rv64gcv works ... No." | tee -a configure.log + if try $CC -c $CFLAGS -march=rv64gc_zve64x $test.c; then + echo "Check whether -march=rv64gc_zve64x works ... Yes." | tee -a configure.log + rvvflag="-march=rv64gc_zve64x" + else + echo "Check whether -march=rv64gc_zve64x works ... No." | tee -a configure.log + rvvflag="" + fi + fi + + cat > $test.c << EOF +#include +int main(void) { + size_t vl = __riscv_vsetvlmax_e8m1(); + return vl; +} +EOF + if [ -n "$rvvflag" ] && try ${CC} ${CFLAGS} ${rvvflag} $test.c; then + echo "Checking for RISC-V Vector intrinsics ... Yes." | tee -a configure.log + HAVE_RVV_INTRIN=1 + else + echo "Checking for RISC-V Vector intrinsics ... No." | tee -a configure.log + HAVE_RVV_INTRIN=0 + fi +} + # Check whether to disable deflate_medium and deflate_quick if test $without_new_strategies -eq 1; then CFLAGS="${CFLAGS} -DNO_QUICK_STRATEGY -DNO_MEDIUM_STRATEGY" @@ -1847,6 +1919,29 @@ EOF fi fi ;; + riscv64) + [ ! -z $CROSS_PREFIX ] && QEMU_ARCH=riscv64 + ARCHDIR=arch/riscv + + if test $without_optimizations -eq 0; then + if test $buildrvv -eq 1; then + check_rvv_compiler_flag + + if test $HAVE_RVV_INTRIN -eq 1; then + CFLAGS="${CFLAGS} -DRISCV_FEATURES -DRISCV_RVV" + SFLAGS="${SFLAGS} -DRISCV_FEATURES -DRISCV_RVV" + + ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} riscv_features.o" + ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} riscv_features.lo" + + ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_rvv.o chunkset_rvv.o compare256_rvv.o slide_hash_rvv.o" + ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_rvv.lo chunkset_rvv.lo compare256_rvv.lo slide_hash_rvv.lo" + + ARCH="${ARCH}+rvv" + fi + fi + fi + ;; s390x) [ ! -z $CROSS_PREFIX ] && QEMU_ARCH=s390x ARCHDIR=arch/s390 @@ -2096,6 +2191,7 @@ sed < $SRCDIR/$ARCHDIR/Makefile.in " /^NOLTOFLAG *=/s#=.*#=$noltoflag# /^VGFMAFLAG *=/s#=.*#=$vgfmaflag# /^PPCFLAGS *=/s#=.*#=$vmxflag# +/^RVVFLAG *=/s#=.*#=$rvvflag# " > $ARCHDIR/Makefile # Append header files dependencies. diff --git a/src/native/external/zlib-ng/deflate.h b/src/native/external/zlib-ng/deflate.h index 4b79f8f43b2c83..02d68211a318d2 100644 --- a/src/native/external/zlib-ng/deflate.h +++ b/src/native/external/zlib-ng/deflate.h @@ -227,7 +227,7 @@ struct ALIGNED_(64) internal_state { # define max_insert_length max_lazy_match /* Insert new strings in the hash table only if the match length is not * greater than this length. This saves time but degrades compression. - * max_insert_length is used only for compression levels <= 3. + * max_insert_length is used only for compression levels <= 6. */ update_hash_cb update_hash; diff --git a/src/native/external/zlib-ng/deflate_medium.c b/src/native/external/zlib-ng/deflate_medium.c index 2aeebe2026bda8..ae7c737ecb117c 100644 --- a/src/native/external/zlib-ng/deflate_medium.c +++ b/src/native/external/zlib-ng/deflate_medium.c @@ -45,12 +45,10 @@ static void insert_match(deflate_state *s, struct match match) { if (UNLIKELY(s->lookahead <= (unsigned int)(match.match_length + WANT_MIN_MATCH))) return; - /* string at strstart already in table */ - match.strstart++; - match.match_length--; - /* matches that are not long enough we need to emit as literals */ - if (LIKELY(match.match_length < WANT_MIN_MATCH - 1)) { + if (LIKELY(match.match_length < WANT_MIN_MATCH)) { + match.strstart++; + match.match_length--; if (UNLIKELY(match.match_length > 0)) { if (match.strstart >= match.orgstart) { if (match.strstart + match.match_length - 1 >= match.orgstart) { @@ -65,18 +63,35 @@ static void insert_match(deflate_state *s, struct match match) { return; } - /* Insert into hash table. */ - if (LIKELY(match.strstart >= match.orgstart)) { - if (LIKELY(match.strstart + match.match_length - 1 >= match.orgstart)) { - insert_string(s, match.strstart, match.match_length); - } else { - insert_string(s, match.strstart, match.orgstart - match.strstart + 1); + /* Insert new strings in the hash table only if the match length + * is not too large. This saves time but degrades compression. + */ + if (match.match_length <= 16 * s->max_insert_length && s->lookahead >= WANT_MIN_MATCH) { + match.match_length--; /* string at strstart already in table */ + match.strstart++; + + if (LIKELY(match.strstart >= match.orgstart)) { + if (LIKELY(match.strstart + match.match_length - 1 >= match.orgstart)) { + insert_string(s, match.strstart, match.match_length); + } else { + insert_string(s, match.strstart, match.orgstart - match.strstart + 1); + } + } else if (match.orgstart < match.strstart + match.match_length) { + insert_string(s, match.orgstart, match.strstart + match.match_length - match.orgstart); } - } else if (match.orgstart < match.strstart + match.match_length) { - insert_string(s, match.orgstart, match.strstart + match.match_length - match.orgstart); + match.strstart += match.match_length; + match.match_length = 0; + } else { + match.strstart += match.match_length; + match.match_length = 0; + + if (match.strstart >= (STD_MIN_MATCH - 2)) + quick_insert_string(s, match.strstart + 2 - STD_MIN_MATCH); + + /* If lookahead < WANT_MIN_MATCH, ins_h is garbage, but it does not + * matter since it will be recomputed at next deflate call. + */ } - match.strstart += match.match_length; - match.match_length = 0; } static void fizzle_matches(deflate_state *s, struct match *current, struct match *next) { diff --git a/src/native/external/zlib-ng/zlib-ng.h.in b/src/native/external/zlib-ng/zlib-ng.h.in index 93c7c86f26da87..71b41b6996d98a 100644 --- a/src/native/external/zlib-ng/zlib-ng.h.in +++ b/src/native/external/zlib-ng/zlib-ng.h.in @@ -48,11 +48,11 @@ extern "C" { #endif -#define ZLIBNG_VERSION "2.2.4" -#define ZLIBNG_VERNUM 0x020204F0L /* MMNNRRSM: major minor revision status modified */ +#define ZLIBNG_VERSION "2.2.5" +#define ZLIBNG_VERNUM 0x020205F0L /* MMNNRRSM: major minor revision status modified */ #define ZLIBNG_VER_MAJOR 2 #define ZLIBNG_VER_MINOR 2 -#define ZLIBNG_VER_REVISION 4 +#define ZLIBNG_VER_REVISION 5 #define ZLIBNG_VER_STATUS F /* 0=devel, 1-E=beta, F=Release (DEPRECATED) */ #define ZLIBNG_VER_STATUSH 0xF /* Hex values: 0=devel, 1-E=beta, F=Release */ #define ZLIBNG_VER_MODIFIED 0 /* non-zero if modified externally from zlib-ng */ diff --git a/src/native/external/zlib-ng/zlib.h.in b/src/native/external/zlib-ng/zlib.h.in index b8fa106ef1609e..46dce01df25412 100644 --- a/src/native/external/zlib-ng/zlib.h.in +++ b/src/native/external/zlib-ng/zlib.h.in @@ -49,11 +49,11 @@ extern "C" { #endif -#define ZLIBNG_VERSION "2.2.4" -#define ZLIBNG_VERNUM 0x020204F0L /* MMNNRRSM: major minor revision status modified */ +#define ZLIBNG_VERSION "2.2.5" +#define ZLIBNG_VERNUM 0x020205F0L /* MMNNRRSM: major minor revision status modified */ #define ZLIBNG_VER_MAJOR 2 #define ZLIBNG_VER_MINOR 2 -#define ZLIBNG_VER_REVISION 4 +#define ZLIBNG_VER_REVISION 5 #define ZLIBNG_VER_STATUS F /* 0=devel, 1-E=beta, F=Release (DEPRECATED) */ #define ZLIBNG_VER_STATUSH 0xF /* Hex values: 0=devel, 1-E=beta, F=Release */ #define ZLIBNG_VER_MODIFIED 0 /* non-zero if modified externally from zlib-ng */ diff --git a/src/native/external/zlib-ng/zlib.pc.cmakein b/src/native/external/zlib-ng/zlib.pc.cmakein index df8bf9f0514f1c..a5ff63cef46454 100644 --- a/src/native/external/zlib-ng/zlib.pc.cmakein +++ b/src/native/external/zlib-ng/zlib.pc.cmakein @@ -1,6 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} symbol_prefix=@ZLIB_SYMBOL_PREFIX@ +bindir=@PC_BIN_INSTALL_DIR@ libdir=@PC_LIB_INSTALL_DIR@ sharedlibdir=${libdir} includedir=@PC_INC_INSTALL_DIR@ diff --git a/src/native/external/zlib-ng/zlib.pc.in b/src/native/external/zlib-ng/zlib.pc.in index 45b35989b1cb41..25cc923ee16ee4 100644 --- a/src/native/external/zlib-ng/zlib.pc.in +++ b/src/native/external/zlib-ng/zlib.pc.in @@ -1,6 +1,7 @@ prefix=@prefix@ exec_prefix=@exec_prefix@ symbol_prefix=@symbol_prefix@ +bindir=@bindir@ libdir=@libdir@ sharedlibdir=@sharedlibdir@ includedir=@includedir@ diff --git a/src/native/external/zlib-ng/zutil.c b/src/native/external/zlib-ng/zutil.c index 99818c3a7b9e04..5bab7f16bd5df0 100644 --- a/src/native/external/zlib-ng/zutil.c +++ b/src/native/external/zlib-ng/zutil.c @@ -21,7 +21,7 @@ z_const char * const PREFIX(z_errmsg)[10] = { }; const char PREFIX3(vstring)[] = - " zlib-ng 2.2.4"; + " zlib-ng 2.2.5"; #ifdef ZLIB_COMPAT const char * Z_EXPORT zlibVersion(void) { From 47b266a2fb0c720149677e8dd40fe9a5279c8057 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Thu, 7 Aug 2025 13:37:55 -0700 Subject: [PATCH 2/2] Update zlib-ng hash + tag with release --- src/native/external/zlib-ng-version.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/native/external/zlib-ng-version.txt b/src/native/external/zlib-ng-version.txt index 8a1e4932e7d8af..ba420d4726af49 100644 --- a/src/native/external/zlib-ng-version.txt +++ b/src/native/external/zlib-ng-version.txt @@ -1,7 +1,7 @@ v2.2.5 -5f0a27330f90da761864e02004acb3cbdc2c84a5 +425439062b114a0f6cf625022c41d929c7e879f9 -https://github.com/zlib-ng/zlib-ng/pull/1943 +https://github.com/zlib-ng/zlib-ng/releases/tag/2.2.5 We have removed the following folders from our local copy as these files are not needed for our compilation: