From 5a204e3fa8beba898087279a0c2a3f24604bc64d Mon Sep 17 00:00:00 2001 From: Qing Liu Date: Wed, 8 May 2019 16:58:21 -0400 Subject: [PATCH 1/2] porting the blosc related changes. --- include/mgard.h | 11 ++++++-- makefile | 10 +++---- src/mgard.cpp | 75 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 79 insertions(+), 17 deletions(-) diff --git a/include/mgard.h b/include/mgard.h index 49393a858d..80bda2e198 100644 --- a/include/mgard.h +++ b/include/mgard.h @@ -138,11 +138,18 @@ namespace mgard void - compress_memory (void *in_data, size_t in_data_size, + compress_memory_blosc (void *in_data, size_t in_data_size, std::vector &out_data); void - decompress_memory (const void *src, int srcLen, int *dst, int dstLen); + compress_memory_z (void *in_data, size_t in_data_size, + std::vector &out_data); + + void + decompress_memory_blosc (const void *src, int srcLen, int *dst, int dstLen); + + void + decompress_memory_z (const void *src, int srcLen, int *dst, int dstLen); void qread_level_2D (const int nrow, const int ncol, const int nlevel, double *v, diff --git a/makefile b/makefile index e55bd23304..49b3ea3a22 100644 --- a/makefile +++ b/makefile @@ -7,14 +7,14 @@ MKDIR=mkdir RMDIR=rmdir --ignore-fail-on-non-empty -CXXFLAGS= -c -Wall -Wfatal-errors -I$(INC) -O3 -fPIC +CXXFLAGS= -std=c++11 -c -Wall -Wfatal-errors -I$(INC) -O3 -fPIC CFLAGS= -c -Wall -Wfatal-errors -I$(INC) -O3 -LDFLAGS = -lz -lm -lstdc++ +LDFLAGS = -lz -lm -lstdc++ -std=c++11 -L/ccs/proj/e2e/qliu/blosc/lib -lblosc -pthread ARFLAGS = rcs SRC=src -INC=include +INC=include -I/ccs/proj/e2e/qliu/blosc/include OBJ=obj vpath %.o $(OBJ) @@ -39,10 +39,10 @@ LIB=libmgard.a all: $(EXECUTABLE) $(LIB) test test2 test3 $(EXECUTABLE): $(OBJECTS) - $(LINK.o) -o $@ $^ + $(LINK.o) -o $@ $^ $(LDFLAGS) #$(SIRIUS_EXEC): $(OBJECTS_SIRIUS) -# $(LINK.o) -o $@ $^ +# $(LINK.o) -o $@ $^ $(LDFLAGS) $(OBJ)/%.o: %.cpp | $(OBJ) $(COMPILE.cpp) $< -o $@ diff --git a/src/mgard.cpp b/src/mgard.cpp index 854af6b5aa..d5160b101e 100644 --- a/src/mgard.cpp +++ b/src/mgard.cpp @@ -32,7 +32,8 @@ // See accompanying file Copyright.txt for details. // - +#include +#include #include "mgard.h" #include "mgard_nuni.h" @@ -94,7 +95,7 @@ refactor_qz (int nrow, int ncol, int nfib, const double *u, int &outsize, double mgard::quantize_2D_iterleave (nrow, ncol*nfib, v.data(), qv, norm, tol); std::vector out_data; - mgard::compress_memory (qv.data (), sizeof (int) * qv.size (), out_data); + mgard::compress_memory_blosc (qv.data (), sizeof (int) * qv.size (), out_data); outsize = out_data.size (); unsigned char *buffer = (unsigned char *)malloc (outsize); @@ -137,7 +138,7 @@ refactor_qz (int nrow, int ncol, int nfib, const double *u, int &outsize, double int l_target = nlevel-1; - mgard::decompress_memory(data, data_len, out_data.data(), out_data.size()*sizeof(int)); // decompress input buffer + mgard::decompress_memory_blosc(data, data_len, out_data.data(), out_data.size()*sizeof(int)); // decompress input buffer double *v = (double *)malloc (nrow*ncol*nfib*sizeof(double)); @@ -195,7 +196,7 @@ refactor_qz_2D (int nrow, int ncol, const double *u, int &outsize, double tol) mgard::quantize_2D_iterleave (nrow, ncol, v.data(), qv, norm, tol); std::vector out_data; - mgard::compress_memory (qv.data (), sizeof (int) * qv.size (), out_data); + mgard::compress_memory_blosc (qv.data (), sizeof (int) * qv.size (), out_data); outsize = out_data.size (); unsigned char *buffer = (unsigned char *)malloc (outsize); std::copy (out_data.begin (), out_data.end (), buffer); @@ -236,7 +237,7 @@ refactor_qz_2D (int nrow, int ncol, const double *u, int &outsize, double tol) mgard::quantize_2D_iterleave (nrow, ncol, v.data (), qv, norm, tol); std::vector out_data; - mgard::compress_memory (qv.data (), sizeof (int) * qv.size (), out_data); + mgard::compress_memory_blosc (qv.data (), sizeof (int) * qv.size (), out_data); outsize = out_data.size (); unsigned char *buffer = (unsigned char *)malloc (outsize); @@ -262,7 +263,7 @@ refactor_qz_2D (int nrow, int ncol, const double *u, int &outsize, double tol) std::vector out_data(nrow_new*ncol_new + size_ratio); - mgard::decompress_memory(data, data_len, out_data.data(), out_data.size()*sizeof(int)); // decompress input buffer + mgard::decompress_memory_blosc(data, data_len, out_data.data(), out_data.size()*sizeof(int)); // decompress input buffer double *v = (double *)malloc (nrow_new*ncol_new*sizeof(double)); @@ -298,7 +299,7 @@ refactor_qz_2D (int nrow, int ncol, const double *u, int &outsize, double tol) int l_target = 0; std::vector out_data(nrow*ncol + size_ratio); - mgard::decompress_memory(data, data_len, out_data.data(), out_data.size()*sizeof(int)); // decompress input buffer + mgard::decompress_memory_blosc(data, data_len, out_data.data(), out_data.size()*sizeof(int)); // decompress input buffer double *v = (double *)malloc (nrow*ncol*sizeof(double)); @@ -965,8 +966,52 @@ qwrite_2D_interleave (const int nrow, const int ncol, const int nlevel, } void -compress_memory (void *in_data, size_t in_data_size, - std::vector &out_data) +compress_memory_blosc (void *in_data, size_t in_data_size, + std::vector &out_data) +{ + std::cout << "compress_memory_blosc: " << in_data_size< buffer; + + uint8_t * temp_buffer = (uint8_t *) malloc (in_data_size); + assert (temp_buffer); + + /* Initialize the Blosc compressor */ + blosc_init(); + + blosc_set_compressor("zstd"); + //blosc_set_compressor("zlib"); + //blosc_set_compressor("lz4"); + //blosc_set_compressor("lz4hc"); + //blosc_set_compressor("snappy"); + //blosc_set_compressor("blosclz"); + + //blosc_set_nthreads(4); + /* Compress with clevel=5 and shuffle active */ + size_t csize = blosc_compress(9, BLOSC_NOSHUFFLE, sizeof(uint8_t), + in_data_size, in_data, + temp_buffer, in_data_size); + + const char* compressor; + + /* Before any blosc_compress() the compressor must be blosclz */ + compressor = blosc_get_compressor(); + + std::cout << "compressor = " << compressor << " ;csize = " << csize << std::endl; + buffer.insert (buffer.end (), temp_buffer, temp_buffer + csize); + out_data.swap (buffer); + + blosc_destroy(); + + auto t2 = std::chrono::high_resolution_clock::now(); + + std::cout << "compress_memory_blosc done: " << std::chrono::duration_cast(t2-t1).count() + << " milliseconds\n" << std::endl; +} + +void +compress_memory_z (void *in_data, size_t in_data_size, + std::vector &out_data) { std::vector buffer; @@ -1016,7 +1061,17 @@ compress_memory (void *in_data, size_t in_data_size, } void -decompress_memory (const void *src, int srcLen, int *dst, int dstLen) +decompress_memory_blosc (const void *src, int srcLen, int *dst, int dstLen) +{ + blosc_init(); + + blosc_decompress(src, dst, dstLen); + + blosc_destroy(); +} + +void +decompress_memory_z (const void *src, int srcLen, int *dst, int dstLen) { z_stream strm = { 0 }; strm.total_in = strm.avail_in = srcLen; From f0c9a267d063fbe404653f992ccf942fce5897d2 Mon Sep 17 00:00:00 2001 From: Qing Liu Date: Wed, 8 May 2019 17:53:40 -0400 Subject: [PATCH 2/2] 1. added bootstrap.sh, which downloads and build the c-blosc lib. 2. updated makefile to point to the c-blosc installation. --- bootstrap.sh | 13 +++++++++++++ makefile | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100755 bootstrap.sh diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000000..5e64313383 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +rm -rf c-blosc-* v1.16.3 +wget https://github.com/Blosc/c-blosc/archive/v1.16.3.tar.gz +tar -zxvf v1.16.3 +cd c-blosc-* +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX=../../blosc .. +#ccmake .. +cmake --build . --target install + +rm -rf v1.16.3 c-blosc-* v1.16.3 diff --git a/makefile b/makefile index 49b3ea3a22..29346100d5 100644 --- a/makefile +++ b/makefile @@ -10,11 +10,11 @@ RMDIR=rmdir --ignore-fail-on-non-empty CXXFLAGS= -std=c++11 -c -Wall -Wfatal-errors -I$(INC) -O3 -fPIC CFLAGS= -c -Wall -Wfatal-errors -I$(INC) -O3 -LDFLAGS = -lz -lm -lstdc++ -std=c++11 -L/ccs/proj/e2e/qliu/blosc/lib -lblosc -pthread +LDFLAGS = -lz -lm -lstdc++ -std=c++11 -L./blosc/lib -lblosc -pthread ARFLAGS = rcs SRC=src -INC=include -I/ccs/proj/e2e/qliu/blosc/include +INC=include -I./blosc/include OBJ=obj vpath %.o $(OBJ)