From 1bd4f185d767648ff9137efcdc690706f60bb053 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Thu, 30 Dec 2021 19:54:46 -0800 Subject: [PATCH 01/25] Async IO decompression: - Added --[no-]asyncio flag for CLI decompression. - Replaced dstBuffer in decompression with a pool of write jobs. - Added an ability to execute write jobs in a separate thread. - Added an ability to wait (join) on all jobs in a thread pool (queue and running). --- lib/common/pool.c | 20 +++- lib/common/pool.h | 6 + programs/fileio.c | 290 ++++++++++++++++++++++++++++++++++++--------- programs/fileio.h | 1 + programs/zstdcli.c | 11 +- tests/playTests.sh | 16 +++ 6 files changed, 284 insertions(+), 60 deletions(-) diff --git a/lib/common/pool.c b/lib/common/pool.c index 2e37cdd73c8..dce7d1bf2a9 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -96,9 +96,7 @@ static void* POOL_thread(void* opaque) { /* If the intended queue size was 0, signal after finishing job */ ZSTD_pthread_mutex_lock(&ctx->queueMutex); ctx->numThreadsBusy--; - if (ctx->queueSize == 1) { - ZSTD_pthread_cond_signal(&ctx->queuePushCond); - } + ZSTD_pthread_cond_signal(&ctx->queuePushCond); ZSTD_pthread_mutex_unlock(&ctx->queueMutex); } } /* for (;;) */ @@ -190,6 +188,17 @@ void POOL_free(POOL_ctx *ctx) { ZSTD_customFree(ctx, ctx->customMem); } +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { POOL_free (pool); } @@ -330,6 +339,11 @@ void POOL_free(POOL_ctx* ctx) { (void)ctx; } +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +}; + int POOL_resize(POOL_ctx* ctx, size_t numThreads) { (void)ctx; (void)numThreads; return 0; diff --git a/lib/common/pool.h b/lib/common/pool.h index 0ebde1805db..b86a3452e5c 100644 --- a/lib/common/pool.h +++ b/lib/common/pool.h @@ -38,6 +38,12 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, */ void POOL_free(POOL_ctx* ctx); + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + /*! POOL_resize() : * Expands or shrinks pool's number of threads. * This is more efficient than releasing + creating a new context, diff --git a/programs/fileio.c b/programs/fileio.c index 5338fa62955..f6ee7847f16 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -34,6 +34,8 @@ #include /* INT_MAX */ #include #include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ +#include "../lib/common/pool.h" +#include "../lib/common/threading.h" #if defined (_MSC_VER) # include @@ -325,6 +327,7 @@ struct FIO_prefs_s { /* IO preferences */ U32 removeSrcFile; U32 overwrite; + U32 asyncIO; /* Computation resources preferences */ unsigned memLimit; @@ -395,6 +398,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->literalCompressionMode = ZSTD_ps_auto; ret->excludeCompressedFiles = 0; ret->allowBlockDevices = 0; + ret->asyncIO = 0; return ret; } @@ -558,6 +562,10 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) prefs->contentSize = value != 0; } +void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) { + prefs->asyncIO = value; +} + /* FIO_ctx_t functions */ void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { @@ -2000,16 +2008,105 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, /* ************************************************************************** * Decompression ***************************************************************************/ +#define DECOMPRESSION_MAX_WRITE_JOBS (10) + +typedef struct { + POOL_ctx* writerPool; + ZSTD_pthread_mutex_t writeJobsMutex; + void* jobs[DECOMPRESSION_MAX_WRITE_JOBS]; + volatile int availableWriteJobs; + int totalWriteJobs; + FILE* dstFile; + unsigned storedSkips; + FIO_prefs_t* prefs; +} write_pool_ctx_t; + +typedef struct { + write_pool_ctx_t *ctx; + FILE* dstFile; + void *buffer; + size_t bufferSize; + size_t usedBufferSize; +} write_job_t; + typedef struct { void* srcBuffer; size_t srcBufferSize; size_t srcBufferLoaded; - void* dstBuffer; - size_t dstBufferSize; ZSTD_DStream* dctx; - FILE* dstFile; + write_pool_ctx_t *writePoolCtx; } dRess_t; +static write_job_t *FIO_createWriteJob(write_pool_ctx_t *ctx) { + void *buffer; + write_job_t *job; + job = (write_job_t*) malloc(sizeof(write_job_t)); + buffer = malloc(ZSTD_DStreamOutSize()); + if(!job || !buffer) + EXM_THROW(101, "Allocation error : not enough memory"); + job->buffer = buffer; + job->bufferSize = ZSTD_DStreamOutSize(); + job->usedBufferSize = 0; + job->dstFile = NULL; + job->ctx = ctx; + return job; +} + +static void FIO_writePoolCreateThreadPool(write_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { + ctx->writerPool = NULL; + if(prefs->asyncIO) { +#ifdef ZSTD_MULTITHREAD + if (ZSTD_pthread_mutex_init(&ctx->writeJobsMutex, NULL)) + EXM_THROW(102, "Failed creating write jobs mutex"); + // We want DECOMPRESSION_MAX_WRITE_JOBS-2 queue items because we need to always have 1 free buffer to + // decompress into and 1 buffer that's actively written to disk and owned by the writing thread. + assert(DECOMPRESSION_MAX_WRITE_JOBS >= 2); + ctx->writerPool = POOL_create(1, DECOMPRESSION_MAX_WRITE_JOBS - 2); + if (!ctx->writerPool) + EXM_THROW(103, "Failed creating writer thread pool"); +#else + DISPLAYLEVEL(2, "Note : asyncio decompression is disabled (lack of multithreading support) \n"); +#endif + } +} + +static write_pool_ctx_t* FIO_writePoolCreate(FIO_prefs_t* const prefs) { + write_pool_ctx_t *ctx; + int i; + ctx = (write_pool_ctx_t*) malloc(sizeof(write_pool_ctx_t)); + if(!ctx) + EXM_THROW(100, "Allocation error : not enough memory"); + FIO_writePoolCreateThreadPool(ctx, prefs); + ctx->prefs = prefs; + ctx->totalWriteJobs = ctx->writerPool ? DECOMPRESSION_MAX_WRITE_JOBS : 1; + ctx->availableWriteJobs = ctx->totalWriteJobs; + for(i=0; i < ctx->availableWriteJobs; i++) { + ctx->jobs[i] = FIO_createWriteJob(ctx); + } + ctx->storedSkips = 0; + ctx->dstFile = NULL; + return ctx; +} + +static void FIO_writePoolFree(write_pool_ctx_t* ctx) { + int i=0; + if(ctx->writerPool) { + // Make sure we finish all tasks and then free the resources + POOL_joinJobs(ctx->writerPool); + // Make sure we are not leaking jobs + assert(ctx->availableWriteJobs==ctx->totalWriteJobs); + POOL_free(ctx->writerPool); + ZSTD_pthread_mutex_destroy(&ctx->writeJobsMutex); + } + for(i=0; iavailableWriteJobs; i++) { + write_job_t* job = (write_job_t*) ctx->jobs[i]; + free(job->buffer); + free(job); + } + free(ctx); +} + + static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) { dRess_t ress; @@ -2027,9 +2124,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZSTD_DStreamOutSize(); - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) + if (!ress.srcBuffer) EXM_THROW(61, "Allocation error : not enough memory"); /* dictionary */ @@ -2039,6 +2134,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi free(dictBuffer); } + ress.writePoolCtx = FIO_writePoolCreate(prefs); + return ress; } @@ -2046,7 +2143,7 @@ static void FIO_freeDResources(dRess_t ress) { CHECK( ZSTD_freeDStream(ress.dctx) ); free(ress.srcBuffer); - free(ress.dstBuffer); + FIO_writePoolFree(ress.writePoolCtx); } @@ -2148,6 +2245,82 @@ FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedS } } } +static void FIO_writePoolSetDstFile(write_pool_ctx_t *ctx, FILE* dstFile) { + assert(ctx!=NULL); + // We can change the dst file only if we have finished writing + if(ctx->writerPool) + POOL_joinJobs(ctx->writerPool); + assert(ctx->storedSkips == 0); + assert(ctx->availableWriteJobs == ctx->totalWriteJobs); + ctx->dstFile = dstFile; +} + +static int FIO_writePoolCloseDstFile(write_pool_ctx_t *ctx) { + FILE *dstFile = ctx->dstFile; + assert(dstFile!=NULL || ctx->prefs->testMode!=0); + FIO_writePoolSetDstFile(ctx, NULL); + return fclose(dstFile); +} + + +static void FIO_releaseWriteJob(write_job_t *job) { + write_pool_ctx_t *ctx = job->ctx; + if(ctx->writerPool) { + ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); + assert(ctx->availableWriteJobs < DECOMPRESSION_MAX_WRITE_JOBS); + ctx->jobs[ctx->availableWriteJobs++] = job; + ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); + } else { + ctx->availableWriteJobs++; + } +} + +static write_job_t* FIO_writePoolGetAvailableWriteJob(write_pool_ctx_t *ctx) { + write_job_t *job; + assert(ctx->dstFile!=NULL || ctx->prefs->testMode); + if(ctx->writerPool) { + ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); + assert(ctx->availableWriteJobs > 0); + job = (write_job_t*) ctx->jobs[--ctx->availableWriteJobs]; + ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); + } else { + assert(ctx->availableWriteJobs==1); + ctx->availableWriteJobs--; + job = (write_job_t*)ctx->jobs[0]; + } + job->usedBufferSize = 0; + job->dstFile = ctx->dstFile; + return job; +} + +static void FIO_WritePoolWriteJobExecute(void* opaque){ + write_job_t* job = (write_job_t*) opaque; + write_pool_ctx_t* ctx = job->ctx; + ctx->storedSkips = FIO_fwriteSparse(job->dstFile, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); + FIO_releaseWriteJob(job); +} + +static void FIO_writePoolQueueWriteJob(write_job_t *job) { + write_pool_ctx_t* ctx = job->ctx; + if(ctx->writerPool) + POOL_add(ctx->writerPool, FIO_WritePoolWriteJobExecute, job); + else + FIO_WritePoolWriteJobExecute(job); +} + +static void FIO_writePoolQueueWriteJobAndGetNextAvailable(write_job_t **job) { + FIO_writePoolQueueWriteJob(*job); + *job = FIO_writePoolGetAvailableWriteJob((*job)->ctx); +} + +static void FIO_writePoolQueueSparseWriteEnd(write_pool_ctx_t* ctx) { + assert(ctx != NULL); + if(ctx->writerPool) + POOL_joinJobs(ctx->writerPool); + assert(ctx->availableWriteJobs == ctx->totalWriteJobs); + FIO_fwriteSparseEnd(ctx->prefs, ctx->dstFile, ctx->storedSkips); + ctx->storedSkips = 0; +} /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ @@ -2224,7 +2397,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, U64 alreadyDecoded) /* for multi-frames streams */ { U64 frameSize = 0; - U32 storedSkips = 0; + write_job_t *writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); /* display last 20 characters only */ { size_t const srcFileLength = strlen(srcFileName); @@ -2244,7 +2417,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, /* Main decompression Loop */ while (1) { ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; - ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; + ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 }; size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2; UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize); @@ -2256,7 +2429,8 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, } /* Write block */ - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); + writeJob->usedBufferSize = outBuff.pos; + FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); frameSize += outBuff.pos; if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); @@ -2294,7 +2468,8 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ress->srcBufferLoaded += readSize; } } } - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + FIO_releaseWriteJob(writeJob); + FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); return frameSize; } @@ -2302,15 +2477,13 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, #ifdef ZSTD_GZDECOMPRESS static unsigned long long -FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, - const FIO_prefs_t* const prefs, - const char* srcFileName) +FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) { unsigned long long outFileSize = 0; z_stream strm; int flush = Z_NO_FLUSH; int decodingError = 0; - unsigned storedSkips = 0; + write_job_t *writeJob = NULL; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; @@ -2321,8 +2494,9 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return FIO_ERROR_FRAME_DECODING; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; + writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; strm.avail_in = (uInt)ress->srcBufferLoaded; strm.next_in = (z_const unsigned char*)ress->srcBuffer; @@ -2343,12 +2517,13 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); decodingError = 1; break; } - { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); + writeJob->usedBufferSize = decompBytes; + FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); outFileSize += decompBytes; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; } } if (ret == Z_STREAM_END) break; @@ -2362,16 +2537,15 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); decodingError = 1; } - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + FIO_releaseWriteJob(writeJob); + FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif - #ifdef ZSTD_LZMADECOMPRESS static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, - const FIO_prefs_t* const prefs, const char* srcFileName, int plain_lzma) { unsigned long long outFileSize = 0; @@ -2379,7 +2553,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, lzma_action action = LZMA_RUN; lzma_ret initRet; int decodingError = 0; - unsigned storedSkips = 0; + write_job_t *writeJob = NULL; strm.next_in = 0; strm.avail_in = 0; @@ -2396,8 +2570,9 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, return FIO_ERROR_FRAME_DECODING; } - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; + writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = (BYTE const*)ress->srcBuffer; strm.avail_in = ress->srcBufferLoaded; @@ -2420,12 +2595,13 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, srcFileName, ret); decodingError = 1; break; } - { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); + writeJob->usedBufferSize = decompBytes; + FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); outFileSize += decompBytes; - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = writeJob->bufferSize; } } if (ret == LZMA_STREAM_END) break; } @@ -2434,7 +2610,8 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, memmove(ress->srcBuffer, strm.next_in, strm.avail_in); ress->srcBufferLoaded = strm.avail_in; lzma_end(&strm); - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + FIO_releaseWriteJob(writeJob); + FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif @@ -2442,7 +2619,6 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, #ifdef ZSTD_LZ4DECOMPRESS static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, - const FIO_prefs_t* const prefs, const char* srcFileName) { unsigned long long filesize = 0; @@ -2450,7 +2626,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, LZ4F_decompressionContext_t dCtx; LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); int decodingError = 0; - unsigned storedSkips = 0; + write_job_t *writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); @@ -2461,7 +2637,8 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, { size_t inSize = 4; size_t outSize= 0; MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); - nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); + nextToLoad = LZ4F_decompress(dCtx, NULL, &outSize, ress->srcBuffer, &inSize, NULL); + assert(outSize == 0); // We don't expect to output anything here if (LZ4F_isError(nextToLoad)) { DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", srcFileName, LZ4F_getErrorName(nextToLoad)); @@ -2473,29 +2650,32 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, for (;nextToLoad;) { size_t readSize; size_t pos = 0; - size_t decodedBytes = ress->dstBufferSize; + size_t decodedBytes = writeJob->bufferSize; + int fullBufferDecoded = 0; /* Read input */ if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); if (!readSize) break; /* reached end of file or stream */ - while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ + while ((pos < readSize) || fullBufferDecoded) { /* still to read, or still to flush */ /* Decode Input (at least partially) */ size_t remaining = readSize - pos; - decodedBytes = ress->dstBufferSize; - nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); + decodedBytes = writeJob->bufferSize; + nextToLoad = LZ4F_decompress(dCtx, writeJob->buffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); if (LZ4F_isError(nextToLoad)) { DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", srcFileName, LZ4F_getErrorName(nextToLoad)); decodingError = 1; nextToLoad = 0; break; } pos += remaining; + fullBufferDecoded = decodedBytes == writeJob->bufferSize; /* Write Block */ if (decodedBytes) { UTIL_HumanReadableSize_t hrs; - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); + writeJob->usedBufferSize = decodedBytes; + FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); filesize += decodedBytes; hrs = UTIL_makeHumanReadableSize(filesize); DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); @@ -2517,7 +2697,8 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, LZ4F_freeDecompressionContext(dCtx); ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + FIO_releaseWriteJob(writeJob); + FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; } @@ -2566,7 +2747,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ #ifdef ZSTD_GZDECOMPRESS - unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, prefs, srcFileName); + unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2576,7 +2757,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ #ifdef ZSTD_LZMADECOMPRESS - unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, prefs, srcFileName, buf[0] != 0xFD); + unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2585,7 +2766,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, #endif } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { #ifdef ZSTD_LZ4DECOMPRESS - unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, prefs, srcFileName); + unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2594,7 +2775,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, #endif } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ return FIO_passThrough(prefs, - ress.dstFile, srcFile, + ress.writePoolCtx->dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); } else { @@ -2632,7 +2813,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, int releaseDstFile = 0; int transferMTime = 0; - if ((ress.dstFile == NULL) && (prefs->testMode==0)) { + if ((ress.writePoolCtx->dstFile == NULL) && (prefs->testMode==0)) { + FILE *dstFile; int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ && strcmp(dstFileName, stdoutmark) @@ -2644,8 +2826,9 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, releaseDstFile = 1; - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); - if (ress.dstFile==NULL) return 1; + dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); + if (dstFile==NULL) return 1; + FIO_writePoolSetDstFile(ress.writePoolCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -2657,10 +2840,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); if (releaseDstFile) { - FILE* const dstFile = ress.dstFile; clearHandler(); - ress.dstFile = NULL; - if (fclose(dstFile)) { + if (FIO_writePoolCloseDstFile(ress.writePoolCtx)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result = 1; } @@ -2874,15 +3055,16 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, return 1; } if (!prefs->testMode) { - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); - if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); + FILE* dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); + if (dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); + FIO_writePoolSetDstFile(ress.writePoolCtx, dstFile); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); if (!status) fCtx->nbFilesProcessed++; error |= status; } - if ((!prefs->testMode) && (fclose(ress.dstFile))) + if ((!prefs->testMode) && (FIO_writePoolCloseDstFile(ress.writePoolCtx))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { diff --git a/programs/fileio.h b/programs/fileio.h index 61094db83cb..398937a64e8 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -109,6 +109,7 @@ void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); void FIO_displayCompressionParameters(const FIO_prefs_t* prefs); +void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value); /* FIO_ctx_t functions */ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index bfe18c0c1ba..fd563e1c24d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -239,9 +239,12 @@ static void usage_advanced(const char* programName) #ifndef ZSTD_NODECOMPRESS DISPLAYOUT( "\n"); DISPLAYOUT( "Advanced decompression arguments : \n"); - DISPLAYOUT( " -l : print information about zstd compressed files \n"); - DISPLAYOUT( "--test : test compressed file integrity \n"); - DISPLAYOUT( " -M# : Set a memory usage limit for decompression \n"); + DISPLAYOUT( " -l : print information about zstd compressed files \n"); + DISPLAYOUT( "--test : test compressed file integrity \n"); + DISPLAYOUT( " -M# : Set a memory usage limit for decompression \n"); +#ifdef ZSTD_MULTITHREAD + DISPLAYOUT( "--[no-]asyncio : use threaded asynchronous IO for output (default: disabled) \n"); +#endif # if ZSTD_SPARSE_DEFAULT DISPLAYOUT( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout) \n"); # else @@ -912,6 +915,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } + if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;} + if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;} if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } diff --git a/tests/playTests.sh b/tests/playTests.sh index b7a3d88a817..dc29d5bd9d5 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1575,6 +1575,22 @@ elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then exit 1 fi +println "\n===> zstd asyncio decompression tests " +roundTripTest -g8M "3 --asyncio" +roundTripTest -g8M "3 --no-asyncio" +if [ $GZIPMODE -eq 1 ]; then + roundTripTest -g8M "3 --format=gzip --asyncio" + roundTripTest -g8M "3 --format=gzip --no-asyncio" +fi +if [ $LZMAMODE -eq 1 ]; then + roundTripTest -g8M "3 --format=lzma --asyncio" + roundTripTest -g8M "3 --format=lzma --no-asyncio" +fi +if [ $LZ4MODE -eq 1 ]; then + roundTripTest -g8M "3 --format=lz4 --asyncio" + roundTripTest -g8M "3 --format=lz4 --no-asyncio" +fi + if [ "$1" != "--test-large-data" ]; then println "Skipping large data tests" From d735b884e0977599c7a7285e4c6d1de8705af701 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Wed, 5 Jan 2022 16:53:56 -0800 Subject: [PATCH 02/25] Async IO decompression: CR fixes --- programs/fileio.c | 175 +++++++++++++++++++++++++++++----------------- 1 file changed, 109 insertions(+), 66 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f6ee7847f16..97206cb6f8e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1806,7 +1806,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, static const char* checked_index(const char* options[], size_t length, size_t index) { assert(index < length); - // Necessary to avoid warnings since -O3 will omit the above `assert` + /* Necessary to avoid warnings since -O3 will omit the above `assert` */ (void) length; return options[index]; } @@ -2011,21 +2011,31 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, #define DECOMPRESSION_MAX_WRITE_JOBS (10) typedef struct { + /* These struct fields should be set only on creation and not changed afterwards */ POOL_ctx* writerPool; - ZSTD_pthread_mutex_t writeJobsMutex; - void* jobs[DECOMPRESSION_MAX_WRITE_JOBS]; - volatile int availableWriteJobs; int totalWriteJobs; + FIO_prefs_t* prefs; + + /* Controls the file we currently write to, make changes only by using provided utility functions */ FILE* dstFile; unsigned storedSkips; - FIO_prefs_t* prefs; + + /* The jobs and availableWriteJobs fields are access by both the main and writer threads and should + * only be mutated after locking the mutex */ + ZSTD_pthread_mutex_t writeJobsMutex; + void* jobs[DECOMPRESSION_MAX_WRITE_JOBS]; + int availableWriteJobs; } write_pool_ctx_t; typedef struct { + /* These fields are automaically set and shouldn't be changed by non WritePool code. */ write_pool_ctx_t *ctx; FILE* dstFile; void *buffer; size_t bufferSize; + + /* This field should be changed before a job is queued for execution and should contain the number + * of bytes to write from the buffer. */ size_t usedBufferSize; } write_job_t; @@ -2052,14 +2062,17 @@ static write_job_t *FIO_createWriteJob(write_pool_ctx_t *ctx) { return job; } -static void FIO_writePoolCreateThreadPool(write_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { +/* WritePool_createThreadPool: + * Creates a thread pool and a mutex for threaded write pool. + * Displays warning if asyncio is requested but MT isn't available. */ +static void WritePool_createThreadPool(write_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { ctx->writerPool = NULL; if(prefs->asyncIO) { #ifdef ZSTD_MULTITHREAD if (ZSTD_pthread_mutex_init(&ctx->writeJobsMutex, NULL)) EXM_THROW(102, "Failed creating write jobs mutex"); - // We want DECOMPRESSION_MAX_WRITE_JOBS-2 queue items because we need to always have 1 free buffer to - // decompress into and 1 buffer that's actively written to disk and owned by the writing thread. + /* We want DECOMPRESSION_MAX_WRITE_JOBS-2 queue items because we need to always have 1 free buffer to + * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ assert(DECOMPRESSION_MAX_WRITE_JOBS >= 2); ctx->writerPool = POOL_create(1, DECOMPRESSION_MAX_WRITE_JOBS - 2); if (!ctx->writerPool) @@ -2070,13 +2083,15 @@ static void FIO_writePoolCreateThreadPool(write_pool_ctx_t *ctx, const FIO_prefs } } -static write_pool_ctx_t* FIO_writePoolCreate(FIO_prefs_t* const prefs) { +/* WritePool_create: + * Allocates and sets and a new write pool including its included jobs. */ +static write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs) { write_pool_ctx_t *ctx; int i; ctx = (write_pool_ctx_t*) malloc(sizeof(write_pool_ctx_t)); if(!ctx) EXM_THROW(100, "Allocation error : not enough memory"); - FIO_writePoolCreateThreadPool(ctx, prefs); + WritePool_createThreadPool(ctx, prefs); ctx->prefs = prefs; ctx->totalWriteJobs = ctx->writerPool ? DECOMPRESSION_MAX_WRITE_JOBS : 1; ctx->availableWriteJobs = ctx->totalWriteJobs; @@ -2088,16 +2103,20 @@ static write_pool_ctx_t* FIO_writePoolCreate(FIO_prefs_t* const prefs) { return ctx; } -static void FIO_writePoolFree(write_pool_ctx_t* ctx) { +/* WritePool_free: + * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ +static void WritePool_free(write_pool_ctx_t* ctx) { int i=0; if(ctx->writerPool) { - // Make sure we finish all tasks and then free the resources + /* Make sure we finish all tasks and then free the resources */ POOL_joinJobs(ctx->writerPool); - // Make sure we are not leaking jobs + /* Make sure we are not leaking jobs */ assert(ctx->availableWriteJobs==ctx->totalWriteJobs); POOL_free(ctx->writerPool); ZSTD_pthread_mutex_destroy(&ctx->writeJobsMutex); } + assert(ctx->dstFile==NULL); + assert(ctx->storedSkips==0); for(i=0; iavailableWriteJobs; i++) { write_job_t* job = (write_job_t*) ctx->jobs[i]; free(job->buffer); @@ -2134,7 +2153,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi free(dictBuffer); } - ress.writePoolCtx = FIO_writePoolCreate(prefs); + ress.writePoolCtx = WritePool_create(prefs); return ress; } @@ -2143,7 +2162,7 @@ static void FIO_freeDResources(dRess_t ress) { CHECK( ZSTD_freeDStream(ress.dctx) ); free(ress.srcBuffer); - FIO_writePoolFree(ress.writePoolCtx); + WritePool_free(ress.writePoolCtx); } @@ -2245,25 +2264,9 @@ FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedS } } } -static void FIO_writePoolSetDstFile(write_pool_ctx_t *ctx, FILE* dstFile) { - assert(ctx!=NULL); - // We can change the dst file only if we have finished writing - if(ctx->writerPool) - POOL_joinJobs(ctx->writerPool); - assert(ctx->storedSkips == 0); - assert(ctx->availableWriteJobs == ctx->totalWriteJobs); - ctx->dstFile = dstFile; -} - -static int FIO_writePoolCloseDstFile(write_pool_ctx_t *ctx) { - FILE *dstFile = ctx->dstFile; - assert(dstFile!=NULL || ctx->prefs->testMode!=0); - FIO_writePoolSetDstFile(ctx, NULL); - return fclose(dstFile); -} - - -static void FIO_releaseWriteJob(write_job_t *job) { +/* WritePool_releaseWriteJob: + * Releases an acquired job back to the pool. Doesn't execute the job. */ +static void WritePool_releaseWriteJob(write_job_t *job) { write_pool_ctx_t *ctx = job->ctx; if(ctx->writerPool) { ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); @@ -2275,7 +2278,9 @@ static void FIO_releaseWriteJob(write_job_t *job) { } } -static write_job_t* FIO_writePoolGetAvailableWriteJob(write_pool_ctx_t *ctx) { +/* WritePool_acquireWriteJob: + * Returns an available write job to be used for a future write. */ +static write_job_t* WritePool_acquireWriteJob(write_pool_ctx_t *ctx) { write_job_t *job; assert(ctx->dstFile!=NULL || ctx->prefs->testMode); if(ctx->writerPool) { @@ -2293,35 +2298,73 @@ static write_job_t* FIO_writePoolGetAvailableWriteJob(write_pool_ctx_t *ctx) { return job; } -static void FIO_WritePoolWriteJobExecute(void* opaque){ +/* WritePool_executeWriteJob: + * Executes a write job synchronously. Can be used as a function for a thread pool. */ +static void WritePool_executeWriteJob(void* opaque){ write_job_t* job = (write_job_t*) opaque; write_pool_ctx_t* ctx = job->ctx; ctx->storedSkips = FIO_fwriteSparse(job->dstFile, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); - FIO_releaseWriteJob(job); + WritePool_releaseWriteJob(job); } -static void FIO_writePoolQueueWriteJob(write_job_t *job) { +/* WritePool_queueWriteJob: + * Queues a write job for execution. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +static void WritePool_queueWriteJob(write_job_t *job) { write_pool_ctx_t* ctx = job->ctx; if(ctx->writerPool) - POOL_add(ctx->writerPool, FIO_WritePoolWriteJobExecute, job); + POOL_add(ctx->writerPool, WritePool_executeWriteJob, job); else - FIO_WritePoolWriteJobExecute(job); + WritePool_executeWriteJob(job); } -static void FIO_writePoolQueueWriteJobAndGetNextAvailable(write_job_t **job) { - FIO_writePoolQueueWriteJob(*job); - *job = FIO_writePoolGetAvailableWriteJob((*job)->ctx); +/* WritePool_queueAndReacquireWriteJob: + * Queues a write job for execution and acquires a new one. + * After execution `job`'s pointed value would change to the newly acquired job. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +static void WritePool_queueAndReacquireWriteJob(write_job_t **job) { + WritePool_queueWriteJob(*job); + *job = WritePool_acquireWriteJob((*job)->ctx); } -static void FIO_writePoolQueueSparseWriteEnd(write_pool_ctx_t* ctx) { +/* WritePool_sparseWriteEnd: + * Ends sparse writes to the current dstFile. + * Blocks on completion of all current write jobs before executing. */ +static void WritePool_sparseWriteEnd(write_pool_ctx_t* ctx) { assert(ctx != NULL); if(ctx->writerPool) POOL_joinJobs(ctx->writerPool); - assert(ctx->availableWriteJobs == ctx->totalWriteJobs); FIO_fwriteSparseEnd(ctx->prefs, ctx->dstFile, ctx->storedSkips); ctx->storedSkips = 0; } +/* WritePool_setDstFile: + * Sets the destination file for future files in the pool. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. + * Also requires ending of sparse write if a previous file was used in sparse mode. */ +static void WritePool_setDstFile(write_pool_ctx_t *ctx, FILE* dstFile) { + assert(ctx!=NULL); + /* We can change the dst file only if we have finished writing */ + if(ctx->writerPool) + POOL_joinJobs(ctx->writerPool); + assert(ctx->storedSkips == 0); + assert(ctx->availableWriteJobs == ctx->totalWriteJobs); + ctx->dstFile = dstFile; +} + +/* WritePool_closeDstFile: + * Ends sparse write and closes the writePool's current dstFile and sets the dstFile to NULL. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ +static int WritePool_closeDstFile(write_pool_ctx_t *ctx) { + FILE *dstFile = ctx->dstFile; + assert(dstFile!=NULL || ctx->prefs->testMode!=0); + WritePool_sparseWriteEnd(ctx); + WritePool_setDstFile(ctx, NULL); + return fclose(dstFile); +} + /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ static int FIO_passThrough(const FIO_prefs_t* const prefs, @@ -2397,7 +2440,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, U64 alreadyDecoded) /* for multi-frames streams */ { U64 frameSize = 0; - write_job_t *writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); + write_job_t *writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); /* display last 20 characters only */ { size_t const srcFileLength = strlen(srcFileName); @@ -2430,7 +2473,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, /* Write block */ writeJob->usedBufferSize = outBuff.pos; - FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); + WritePool_queueAndReacquireWriteJob(&writeJob); frameSize += outBuff.pos; if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); @@ -2468,8 +2511,8 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ress->srcBufferLoaded += readSize; } } } - FIO_releaseWriteJob(writeJob); - FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); return frameSize; } @@ -2494,7 +2537,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return FIO_ERROR_FRAME_DECODING; - writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); + writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.avail_in = (uInt)ress->srcBufferLoaded; @@ -2520,7 +2563,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -2537,8 +2580,8 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); decodingError = 1; } - FIO_releaseWriteJob(writeJob); - FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif @@ -2570,7 +2613,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, return FIO_ERROR_FRAME_DECODING; } - writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); + writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = (BYTE const*)ress->srcBuffer; @@ -2598,7 +2641,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -2610,8 +2653,8 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, memmove(ress->srcBuffer, strm.next_in, strm.avail_in); ress->srcBufferLoaded = strm.avail_in; lzma_end(&strm); - FIO_releaseWriteJob(writeJob); - FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif @@ -2626,7 +2669,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, LZ4F_decompressionContext_t dCtx; LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); int decodingError = 0; - write_job_t *writeJob = FIO_writePoolGetAvailableWriteJob(ress->writePoolCtx); + write_job_t *writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); @@ -2638,7 +2681,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, size_t outSize= 0; MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); nextToLoad = LZ4F_decompress(dCtx, NULL, &outSize, ress->srcBuffer, &inSize, NULL); - assert(outSize == 0); // We don't expect to output anything here + assert(outSize == 0); /* We don't expect to output anything here */ if (LZ4F_isError(nextToLoad)) { DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", srcFileName, LZ4F_getErrorName(nextToLoad)); @@ -2675,7 +2718,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, if (decodedBytes) { UTIL_HumanReadableSize_t hrs; writeJob->usedBufferSize = decodedBytes; - FIO_writePoolQueueWriteJobAndGetNextAvailable(&writeJob); + WritePool_queueAndReacquireWriteJob(&writeJob); filesize += decodedBytes; hrs = UTIL_makeHumanReadableSize(filesize); DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); @@ -2697,8 +2740,8 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, LZ4F_freeDecompressionContext(dCtx); ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ - FIO_releaseWriteJob(writeJob); - FIO_writePoolQueueSparseWriteEnd(ress->writePoolCtx); + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; } @@ -2828,7 +2871,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; - FIO_writePoolSetDstFile(ress.writePoolCtx, dstFile); + WritePool_setDstFile(ress.writePoolCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -2841,7 +2884,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, if (releaseDstFile) { clearHandler(); - if (FIO_writePoolCloseDstFile(ress.writePoolCtx)) { + if (WritePool_closeDstFile(ress.writePoolCtx)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result = 1; } @@ -3057,14 +3100,14 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, if (!prefs->testMode) { FILE* dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); - FIO_writePoolSetDstFile(ress.writePoolCtx, dstFile); + WritePool_setDstFile(ress.writePoolCtx, dstFile); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); if (!status) fCtx->nbFilesProcessed++; error |= status; } - if ((!prefs->testMode) && (FIO_writePoolCloseDstFile(ress.writePoolCtx))) + if ((!prefs->testMode) && (WritePool_closeDstFile(ress.writePoolCtx))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { From c74b2635d1de1289ea63edf7ffd2090c5af03e63 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Thu, 6 Jan 2022 11:50:43 -0800 Subject: [PATCH 03/25] LZ4 decompression: Fixed multiframe decompression --- programs/fileio.c | 61 +++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 97206cb6f8e..b85e0806bcf 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2165,6 +2165,13 @@ static void FIO_freeDResources(dRess_t ress) WritePool_free(ress.writePoolCtx); } +/* FIO_consumeDSrcBuffer: + * Consumes len bytes from srcBuffer's start and moves the remaining data and srcBufferLoaded accordingly. */ +static void FIO_consumeDSrcBuffer(dRess_t *ress, size_t len) { + assert(ress->srcBufferLoaded >= len); + ress->srcBufferLoaded -= len; + memmove(ress->srcBuffer, (char *)ress->srcBuffer + len, ress->srcBufferLoaded); +} /** FIO_fwriteSparse() : * @return : storedSkips, @@ -2490,10 +2497,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, srcFileName, hrs.precision, hrs.value, hrs.suffix); } - if (inBuff.pos > 0) { - memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); - ress->srcBufferLoaded -= inBuff.pos; - } + FIO_consumeDSrcBuffer(ress, inBuff.pos); if (readSizeHint == 0) break; /* end of frame */ @@ -2572,9 +2576,8 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) if (ret == Z_STREAM_END) break; } - if (strm.avail_in > 0) - memmove(ress->srcBuffer, strm.next_in, strm.avail_in); - ress->srcBufferLoaded = strm.avail_in; + FIO_consumeDSrcBuffer(ress, ress->srcBufferLoaded - strm.avail_in); + if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ && (decodingError==0) ) { DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); @@ -2649,9 +2652,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, if (ret == LZMA_STREAM_END) break; } - if (strm.avail_in > 0) - memmove(ress->srcBuffer, strm.next_in, strm.avail_in); - ress->srcBufferLoaded = strm.avail_in; + FIO_consumeDSrcBuffer(ress, ress->srcBufferLoaded - strm.avail_in); lzma_end(&strm); WritePool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); @@ -2665,7 +2666,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, const char* srcFileName) { unsigned long long filesize = 0; - LZ4F_errorCode_t nextToLoad; + LZ4F_errorCode_t nextToLoad = 4; LZ4F_decompressionContext_t dCtx; LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); int decodingError = 0; @@ -2676,19 +2677,6 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, return FIO_ERROR_FRAME_DECODING; } - /* Init feed with magic number (already consumed from FILE* sFile) */ - { size_t inSize = 4; - size_t outSize= 0; - MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); - nextToLoad = LZ4F_decompress(dCtx, NULL, &outSize, ress->srcBuffer, &inSize, NULL); - assert(outSize == 0); /* We don't expect to output anything here */ - if (LZ4F_isError(nextToLoad)) { - DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", - srcFileName, LZ4F_getErrorName(nextToLoad)); - LZ4F_freeDecompressionContext(dCtx); - return FIO_ERROR_FRAME_DECODING; - } } - /* Main Loop */ for (;nextToLoad;) { size_t readSize; @@ -2697,13 +2685,19 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, int fullBufferDecoded = 0; /* Read input */ - if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; - readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); - if (!readSize) break; /* reached end of file or stream */ + nextToLoad = MIN(nextToLoad, ress->srcBufferSize-ress->srcBufferLoaded); + readSize = fread((char *)ress->srcBuffer + ress->srcBufferLoaded, 1, nextToLoad, srcFile); + if(!readSize && ferror(srcFile)) { + DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); + decodingError=1; + break; + } + if(!readSize && !ress->srcBufferLoaded) break; /* reached end of file */ + ress->srcBufferLoaded += readSize; - while ((pos < readSize) || fullBufferDecoded) { /* still to read, or still to flush */ + while ((pos < ress->srcBufferLoaded) || fullBufferDecoded) { /* still to read, or still to flush */ /* Decode Input (at least partially) */ - size_t remaining = readSize - pos; + size_t remaining = ress->srcBufferLoaded - pos; decodedBytes = writeJob->bufferSize; nextToLoad = LZ4F_decompress(dCtx, writeJob->buffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); if (LZ4F_isError(nextToLoad)) { @@ -2712,6 +2706,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, decodingError = 1; nextToLoad = 0; break; } pos += remaining; + assert(pos <= ress->srcBufferLoaded); fullBufferDecoded = decodedBytes == writeJob->bufferSize; /* Write Block */ @@ -2726,20 +2721,14 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, if (!nextToLoad) break; } + FIO_consumeDSrcBuffer(ress, pos); } - /* can be out because readSize == 0, which could be an fread() error */ - if (ferror(srcFile)) { - DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); - decodingError=1; - } - if (nextToLoad!=0) { DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); decodingError=1; } LZ4F_freeDecompressionContext(dCtx); - ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ WritePool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); From dd5a5401d850c22b86c51429b5f0e1d5afd6feac Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Thu, 6 Jan 2022 11:51:17 -0800 Subject: [PATCH 04/25] Async IO decompression: Added multiframe test for --[no-]asyncio --- tests/playTests.sh | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index dc29d5bd9d5..78d8e742aa3 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1576,21 +1576,43 @@ elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then fi println "\n===> zstd asyncio decompression tests " -roundTripTest -g8M "3 --asyncio" -roundTripTest -g8M "3 --no-asyncio" + +addFrame() { + datagen -g2M -s$2 >> tmp_uncompressed + datagen -g2M -s$2 | zstd --format=$1 >> tmp_compressed.zst +} + +addTwoFrames() { + addFrame $1 1 + addFrame $1 2 +} + +testAsyncIO() { + roundTripTest -g2M "3 --asyncio --format=$1" + roundTripTest -g2M "3 --no-asyncio --format=$1" +} + +rm -f tmp_compressed tmp_uncompressed +testAsyncIO zstd +addTwoFrames zstd if [ $GZIPMODE -eq 1 ]; then - roundTripTest -g8M "3 --format=gzip --asyncio" - roundTripTest -g8M "3 --format=gzip --no-asyncio" + testAsyncIO gzip + addTwoFrames gzip fi if [ $LZMAMODE -eq 1 ]; then - roundTripTest -g8M "3 --format=lzma --asyncio" - roundTripTest -g8M "3 --format=lzma --no-asyncio" + testAsyncIO lzma + addTwoFrames lzma fi if [ $LZ4MODE -eq 1 ]; then - roundTripTest -g8M "3 --format=lz4 --asyncio" - roundTripTest -g8M "3 --format=lz4 --no-asyncio" + testAsyncIO lz4 + addTwoFrames lz4 fi - +cat tmp_uncompressed | $MD5SUM > tmp2 +zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1 +$DIFF -q tmp1 tmp2 +rm tmp1 +zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1 +$DIFF -q tmp1 tmp2 if [ "$1" != "--test-large-data" ]; then println "Skipping large data tests" From 702a9a0d36583c5cca901b2587dedd56fd6f45ba Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Thu, 6 Jan 2022 13:26:48 -0800 Subject: [PATCH 05/25] Async IO decompression: C standard fix --- lib/common/pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/pool.c b/lib/common/pool.c index dce7d1bf2a9..5c1d07d356e 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -342,7 +342,7 @@ void POOL_free(POOL_ctx* ctx) { void POOL_joinJobs(POOL_ctx* ctx){ assert(!ctx || ctx == &g_poolCtx); (void)ctx; -}; +} int POOL_resize(POOL_ctx* ctx, size_t numThreads) { (void)ctx; (void)numThreads; From 05b174a67ee169020e4069434ac6b8ae8a4536f6 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Mon, 10 Jan 2022 18:16:57 -0800 Subject: [PATCH 06/25] Async IO decompression: fix meson compilation --- build/meson/programs/meson.build | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index 4181030c2ee..0ae93fc107c 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -20,14 +20,24 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/dibio.c'), join_paths(zstd_rootdir, 'programs/zstdcli_trace.c'), # needed due to use of private symbol + -fvisibility=hidden - join_paths(zstd_rootdir, 'lib/common/xxhash.c')] + join_paths(zstd_rootdir, 'lib/common/xxhash.c'), + join_paths(zstd_rootdir, 'lib/common/pool.c'), + join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), + join_paths(zstd_rootdir, 'lib/common/error_private.c')] +zstd_deps = [ libzstd_dep ] zstd_c_args = libzstd_debug_cflags + +zstd_frugal_deps = [ libzstd_dep ] +zstd_frugal_c_args = [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ] + if use_multi_thread + zstd_deps += [ thread_dep ] zstd_c_args += [ '-DZSTD_MULTITHREAD' ] + zstd_frugal_deps += [ thread_dep ] + zstd_frugal_c_args += [ '-DZSTD_MULTITHREAD' ] endif -zstd_deps = [ libzstd_dep ] if use_zlib zstd_deps += [ zlib_dep ] zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ] @@ -69,14 +79,17 @@ zstd = executable('zstd', zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/timefn.c'), join_paths(zstd_rootdir, 'programs/util.c'), - join_paths(zstd_rootdir, 'programs/fileio.c')] + join_paths(zstd_rootdir, 'programs/fileio.c'), + join_paths(zstd_rootdir, 'lib/common/pool.c'), + join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), + join_paths(zstd_rootdir, 'lib/common/error_private.c')] # Minimal target, with only zstd compression and decompression. # No bench. No legacy. executable('zstd-frugal', zstd_frugal_sources, - dependencies: libzstd_dep, - c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ], + dependencies: zstd_frugal_deps, + c_args: zstd_frugal_c_args, install: true) install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'), From 4a1b6b14729c0210d59be6c2e807db41b0ac00fd Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Thu, 6 Jan 2022 18:04:23 -0800 Subject: [PATCH 07/25] Async IO compression: Added support for async write in compression --- programs/fileio.c | 814 +++++++++++++++++++++++----------------------- 1 file changed, 414 insertions(+), 400 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b85e0806bcf..e4d8faee1b0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -934,6 +934,321 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* return error; } +/* ********************************************************************** + * AsyncIO functionality + ************************************************************************/ +#define MAX_WRITE_JOBS (10) + +typedef struct { + /* These struct fields should be set only on creation and not changed afterwards */ + POOL_ctx* writerPool; + int totalWriteJobs; + FIO_prefs_t* prefs; + + /* Controls the file we currently write to, make changes only by using provided utility functions */ + FILE* dstFile; + unsigned storedSkips; + + /* The jobs and availableWriteJobs fields are access by both the main and writer threads and should + * only be mutated after locking the mutex */ + ZSTD_pthread_mutex_t writeJobsMutex; + void* jobs[MAX_WRITE_JOBS]; + int availableWriteJobs; +} write_pool_ctx_t; + +typedef struct { + /* These fields are automaically set and shouldn't be changed by non WritePool code. */ + write_pool_ctx_t *ctx; + FILE* dstFile; + void *buffer; + size_t bufferSize; + + /* This field should be changed before a job is queued for execution and should contain the number + * of bytes to write from the buffer. */ + size_t usedBufferSize; +} write_job_t; + + +static write_job_t *FIO_createWriteJob(write_pool_ctx_t *ctx) { + void *buffer; + write_job_t *job; + size_t bufferSize = MAX(ZSTD_DStreamOutSize(), ZSTD_CStreamOutSize()); + job = (write_job_t*) malloc(sizeof(write_job_t)); + buffer = malloc(bufferSize); + if(!job || !buffer) + EXM_THROW(101, "Allocation error : not enough memory"); + job->buffer = buffer; + job->bufferSize = bufferSize; + job->usedBufferSize = 0; + job->dstFile = NULL; + job->ctx = ctx; + return job; +} + + +/* WritePool_createThreadPool: + * Creates a thread pool and a mutex for threaded write pool. + * Displays warning if asyncio is requested but MT isn't available. */ +static void WritePool_createThreadPool(write_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { + ctx->writerPool = NULL; + if(prefs->asyncIO) { +#ifdef ZSTD_MULTITHREAD + if (ZSTD_pthread_mutex_init(&ctx->writeJobsMutex, NULL)) + EXM_THROW(102, "Failed creating write jobs mutex"); + /* We want MAX_WRITE_JOBS-2 queue items because we need to always have 1 free buffer to + * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ + assert(MAX_WRITE_JOBS >= 2); + ctx->writerPool = POOL_create(1, MAX_WRITE_JOBS - 2); + if (!ctx->writerPool) + EXM_THROW(103, "Failed creating writer thread pool"); +#else + DISPLAYLEVEL(2, "Note : asyncio decompression is disabled (lack of multithreading support) \n"); +#endif + } +} + +/* WritePool_create: + * Allocates and sets and a new write pool including its included jobs. */ +static write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs) { + write_pool_ctx_t *ctx; + int i; + ctx = (write_pool_ctx_t*) malloc(sizeof(write_pool_ctx_t)); + if(!ctx) + EXM_THROW(100, "Allocation error : not enough memory"); + WritePool_createThreadPool(ctx, prefs); + ctx->prefs = prefs; + ctx->totalWriteJobs = ctx->writerPool ? MAX_WRITE_JOBS : 1; + ctx->availableWriteJobs = ctx->totalWriteJobs; + for(i=0; i < ctx->availableWriteJobs; i++) { + ctx->jobs[i] = FIO_createWriteJob(ctx); + } + ctx->storedSkips = 0; + ctx->dstFile = NULL; + return ctx; +} + +/* WritePool_free: + * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ +static void WritePool_free(write_pool_ctx_t* ctx) { + int i=0; + if(ctx->writerPool) { + /* Make sure we finish all tasks and then free the resources */ + POOL_joinJobs(ctx->writerPool); + /* Make sure we are not leaking jobs */ + assert(ctx->availableWriteJobs==ctx->totalWriteJobs); + POOL_free(ctx->writerPool); + ZSTD_pthread_mutex_destroy(&ctx->writeJobsMutex); + } + assert(ctx->dstFile==NULL); + assert(ctx->storedSkips==0); + for(i=0; iavailableWriteJobs; i++) { + write_job_t* job = (write_job_t*) ctx->jobs[i]; + free(job->buffer); + free(job); + } + free(ctx); +} + +/** FIO_fwriteSparse() : +* @return : storedSkips, +* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ +static unsigned +FIO_fwriteSparse(FILE* file, + const void* buffer, size_t bufferSize, + const FIO_prefs_t* const prefs, + unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + size_t bufferSizeT = bufferSize / sizeof(size_t); + const size_t* const bufferTEnd = bufferT + bufferSizeT; + const size_t* ptrT = bufferT; + static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ + + if (prefs->testMode) return 0; /* do not output anything in test mode */ + + if (!prefs->sparseFileSupport) { /* normal write */ + size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) + EXM_THROW(70, "Write error : cannot write decoded block : %s", + strerror(errno)); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) { + if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) + EXM_THROW(91, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t nb0T; + + /* adjust last segment if < 32 KB */ + size_t seg0SizeT = segmentSizeT; + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + + /* count leading zeroes */ + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeof(size_t)); + + if (nb0T != seg0SizeT) { /* not all 0s */ + size_t const nbNon0ST = seg0SizeT - nb0T; + /* skip leading zeros */ + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + /* write the rest */ + if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) + EXM_THROW(93, "Write error : cannot write decoded block : %s", + strerror(errno)); + } + ptrT += seg0SizeT; + } + + { static size_t const maskT = sizeof(size_t)-1; + if (bufferSize & maskT) { + /* size not multiple of sizeof(size_t) : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + const char* const restEnd = (const char*)buffer + bufferSize; + assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); + for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + /* not all remaining bytes are 0 */ + size_t const restSize = (size_t)(restEnd - restPtr); + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + if (fwrite(restPtr, 1, restSize, file) != restSize) + EXM_THROW(95, "Write error : cannot write end of decoded block : %s", + strerror(errno)); + storedSkips = 0; + } } } + + return storedSkips; +} + +static void +FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) +{ + if (prefs->testMode) assert(storedSkips == 0); + if (storedSkips>0) { + assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ + (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ + if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) + EXM_THROW(69, "Final skip error (sparse file support)"); + /* last zero must be explicitly written, + * so that skipped ones get implicitly translated as zero by FS */ + { const char lastZeroByte[1] = { 0 }; + if (fwrite(lastZeroByte, 1, 1, file) != 1) + EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); + } } +} + +/* WritePool_releaseWriteJob: + * Releases an acquired job back to the pool. Doesn't execute the job. */ +static void WritePool_releaseWriteJob(write_job_t *job) { + write_pool_ctx_t *ctx = job->ctx; + if(ctx->writerPool) { + ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); + assert(ctx->availableWriteJobs < MAX_WRITE_JOBS); + ctx->jobs[ctx->availableWriteJobs++] = job; + ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); + } else { + ctx->availableWriteJobs++; + } +} + +/* WritePool_acquireWriteJob: + * Returns an available write job to be used for a future write. */ +static write_job_t* WritePool_acquireWriteJob(write_pool_ctx_t *ctx) { + write_job_t *job; + assert(ctx->dstFile!=NULL || ctx->prefs->testMode); + if(ctx->writerPool) { + ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); + assert(ctx->availableWriteJobs > 0); + job = (write_job_t*) ctx->jobs[--ctx->availableWriteJobs]; + ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); + } else { + assert(ctx->availableWriteJobs==1); + ctx->availableWriteJobs--; + job = (write_job_t*)ctx->jobs[0]; + } + job->usedBufferSize = 0; + job->dstFile = ctx->dstFile; + return job; +} + +/* WritePool_executeWriteJob: + * Executes a write job synchronously. Can be used as a function for a thread pool. */ +static void WritePool_executeWriteJob(void* opaque){ + write_job_t* job = (write_job_t*) opaque; + write_pool_ctx_t* ctx = job->ctx; + ctx->storedSkips = FIO_fwriteSparse(job->dstFile, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); + WritePool_releaseWriteJob(job); +} + +/* WritePool_queueWriteJob: + * Queues a write job for execution. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +static void WritePool_queueWriteJob(write_job_t *job) { + write_pool_ctx_t* ctx = job->ctx; + if(ctx->writerPool) + POOL_add(ctx->writerPool, WritePool_executeWriteJob, job); + else + WritePool_executeWriteJob(job); +} + +/* WritePool_queueAndReacquireWriteJob: + * Queues a write job for execution and acquires a new one. + * After execution `job`'s pointed value would change to the newly acquired job. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +static void WritePool_queueAndReacquireWriteJob(write_job_t **job) { + WritePool_queueWriteJob(*job); + *job = WritePool_acquireWriteJob((*job)->ctx); +} + +/* WritePool_sparseWriteEnd: + * Ends sparse writes to the current dstFile. + * Blocks on completion of all current write jobs before executing. */ +static void WritePool_sparseWriteEnd(write_pool_ctx_t* ctx) { + assert(ctx != NULL); + if(ctx->writerPool) + POOL_joinJobs(ctx->writerPool); + FIO_fwriteSparseEnd(ctx->prefs, ctx->dstFile, ctx->storedSkips); + ctx->storedSkips = 0; +} + +/* WritePool_setDstFile: + * Sets the destination file for future files in the pool. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. + * Also requires ending of sparse write if a previous file was used in sparse mode. */ +static void WritePool_setDstFile(write_pool_ctx_t *ctx, FILE* dstFile) { + assert(ctx!=NULL); + /* We can change the dst file only if we have finished writing */ + if(ctx->writerPool) + POOL_joinJobs(ctx->writerPool); + assert(ctx->storedSkips == 0); + assert(ctx->availableWriteJobs == ctx->totalWriteJobs); + ctx->dstFile = dstFile; +} + +/* WritePool_closeDstFile: + * Ends sparse write and closes the writePool's current dstFile and sets the dstFile to NULL. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ +static int WritePool_closeDstFile(write_pool_ctx_t *ctx) { + FILE *dstFile = ctx->dstFile; + assert(dstFile!=NULL || ctx->prefs->testMode!=0); + WritePool_sparseWriteEnd(ctx); + WritePool_setDstFile(ctx, NULL); + return fclose(dstFile); +} + + #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** @@ -941,15 +1256,13 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* ************************************************************************/ typedef struct { FILE* srcFile; - FILE* dstFile; void* srcBuffer; size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; void* dictBuffer; size_t dictBufferSize; const char* dictFileName; ZSTD_CStream* cctx; + write_pool_ctx_t *writePoolCtx; } cRess_t; /** ZSTD_cycleLog() : @@ -1000,7 +1313,6 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, strerror(errno)); ress.srcBufferSize = ZSTD_CStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZSTD_CStreamOutSize(); /* need to update memLimit before calling createDictBuffer * because of memLimit check inside it */ @@ -1008,11 +1320,12 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); } - ress.dstBuffer = malloc(ress.dstBufferSize); ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ - if (!ress.srcBuffer || !ress.dstBuffer) + if (!ress.srcBuffer) EXM_THROW(31, "allocation error : not enough memory"); + ress.writePoolCtx = WritePool_create(prefs); + /* Advanced parameters, including dictionary */ if (dictFileName && (ress.dictBuffer==NULL)) EXM_THROW(32, "allocation error : can't create dictBuffer"); @@ -1075,8 +1388,8 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, static void FIO_freeCResources(const cRess_t* const ress) { free(ress->srcBuffer); - free(ress->dstBuffer); free(ress->dictBuffer); + WritePool_free(ress->writePoolCtx); ZSTD_freeCStream(ress->cctx); /* never fails */ } @@ -1089,6 +1402,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { unsigned long long inFileSize = 0, outFileSize = 0; z_stream strm; + write_job_t *writeJob = NULL; if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION; @@ -1104,10 +1418,11 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); } } + writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); strm.next_in = 0; strm.avail_in = 0; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; while (1) { int ret; @@ -1121,13 +1436,13 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no ret = deflate(&strm, Z_NO_FLUSH); if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); - { size_t const cSize = ress->dstBufferSize - strm.avail_out; + { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { - if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) - EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno)); + writeJob->usedBufferSize = cSize; + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += cSize; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; } } if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", @@ -1141,13 +1456,13 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no while (1) { int const ret = deflate(&strm, Z_FINISH); - { size_t const cSize = ress->dstBufferSize - strm.avail_out; + { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { - if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) - EXM_THROW(75, "Write error : %s ", strerror(errno)); + writeJob->usedBufferSize = cSize; + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += cSize; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; } } if (ret == Z_STREAM_END) break; if (ret != Z_BUF_ERROR) @@ -1159,6 +1474,8 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); } } *readsize = inFileSize; + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; } #endif @@ -1174,6 +1491,7 @@ FIO_compressLzmaFrame(cRess_t* ress, lzma_stream strm = LZMA_STREAM_INIT; lzma_action action = LZMA_RUN; lzma_ret ret; + write_job_t *writeJob = NULL; if (compressionLevel < 0) compressionLevel = 0; if (compressionLevel > 9) compressionLevel = 9; @@ -1191,10 +1509,11 @@ FIO_compressLzmaFrame(cRess_t* ress, EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); } + writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = 0; strm.avail_in = 0; - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; while (1) { if (strm.avail_in == 0) { @@ -1209,13 +1528,13 @@ FIO_compressLzmaFrame(cRess_t* ress, if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); - { size_t const compBytes = ress->dstBufferSize - strm.avail_out; + { size_t const compBytes = writeJob->bufferSize - strm.avail_out; if (compBytes) { - if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) - EXM_THROW(85, "Write error : %s", strerror(errno)); + writeJob->usedBufferSize = compBytes; + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += compBytes; - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; + strm.next_out = (Bytef*)writeJob->buffer; + strm.avail_out = writeJob->bufferSize; } } if (srcFileSize == UTIL_FILESIZE_UNKNOWN) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", @@ -1231,6 +1550,9 @@ FIO_compressLzmaFrame(cRess_t* ress, lzma_end(&strm); *readsize = inFileSize; + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); + return outFileSize; } #endif @@ -1256,6 +1578,8 @@ FIO_compressLz4Frame(cRess_t* ress, LZ4F_preferences_t prefs; LZ4F_compressionContext_t ctx; + write_job_t *writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) EXM_THROW(31, "zstd: failed to create lz4 compression context"); @@ -1272,16 +1596,16 @@ FIO_compressLz4Frame(cRess_t* ress, #if LZ4_VERSION_NUMBER >= 10600 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; #endif - assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); + assert(LZ4F_compressBound(blockSize, &prefs) <= writeJob->bufferSize); { size_t readSize; - size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); + size_t headerSize = LZ4F_compressBegin(ctx, writeJob->buffer, writeJob->bufferSize, &prefs); if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) - EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno)); + writeJob->usedBufferSize = headerSize; + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; /* Read first block */ @@ -1290,8 +1614,7 @@ FIO_compressLz4Frame(cRess_t* ress, /* Main Loop */ while (readSize>0) { - size_t const outSize = LZ4F_compressUpdate(ctx, - ress->dstBuffer, ress->dstBufferSize, + size_t const outSize = LZ4F_compressUpdate(ctx, writeJob->buffer, writeJob->bufferSize, ress->srcBuffer, readSize, NULL); if (LZ4F_isError(outSize)) EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", @@ -1308,10 +1631,8 @@ FIO_compressLz4Frame(cRess_t* ress, } /* Write Block */ - { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); - if (sizeCheck != outSize) - EXM_THROW(36, "Write error : %s", strerror(errno)); - } + writeJob->usedBufferSize = outSize; + WritePool_queueAndReacquireWriteJob(&writeJob); /* Read next block */ readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); @@ -1320,21 +1641,20 @@ FIO_compressLz4Frame(cRess_t* ress, if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); + headerSize = LZ4F_compressEnd(ctx, writeJob->buffer, writeJob->bufferSize, NULL); if (LZ4F_isError(headerSize)) EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", srcFileName, LZ4F_getErrorName(headerSize)); - { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); - if (sizeCheck != headerSize) - EXM_THROW(39, "Write error : %s (cannot write end of stream)", - strerror(errno)); - } + writeJob->usedBufferSize = headerSize; + WritePool_queueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; } *readsize = inFileSize; LZ4F_freeCompressionContext(ctx); + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; } @@ -1350,7 +1670,8 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, { cRess_t const ress = *ressPtr; FILE* const srcFile = ress.srcFile; - FILE* const dstFile = ress.dstFile; + write_job_t *writeJob = WritePool_acquireWriteJob(ressPtr->writePoolCtx); + U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; @@ -1408,7 +1729,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, || (directive == ZSTD_e_end && stillToFlush != 0) ) { size_t const oldIPos = inBuff.pos; - ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; + ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 }; size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); @@ -1421,10 +1742,8 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); if (outBuff.pos) { - size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); - if (sizeCheck != outBuff.pos) - EXM_THROW(25, "Write error : %s (cannot write compressed block)", - strerror(errno)); + writeJob->usedBufferSize = outBuff.pos; + WritePool_queueAndReacquireWriteJob(&writeJob); compressedfilesize += outBuff.pos; } @@ -1564,6 +1883,9 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned long long)*readsize, (unsigned long long)fileSize); } + WritePool_releaseWriteJob(writeJob); + WritePool_sparseWriteEnd(ressPtr->writePoolCtx); + return compressedfilesize; } @@ -1683,8 +2005,9 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, int result; stat_t statbuf; int transferMTime = 0; + FILE *dstFile; assert(ress.srcFile != NULL); - if (ress.dstFile == NULL) { + if (ress.writePoolCtx->dstFile == NULL) { int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) @@ -1696,8 +2019,9 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); - if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ + dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); + if (dstFile==NULL) return 1; /* could not open dstFileName */ + WritePool_setDstFile(ress.writePoolCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, * and the user presses Ctrl-C when asked if they wish to overwrite. @@ -1708,13 +2032,10 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); if (closeDstFile) { - FILE* const dstFile = ress.dstFile; - ress.dstFile = NULL; - clearHandler(); DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); - if (fclose(dstFile)) { /* error closing dstFile */ + if (WritePool_closeDstFile(ress.writePoolCtx)) { /* error closing dstFile */ DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } @@ -1936,23 +2257,24 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, /* init */ assert(outFileName != NULL || suffix != NULL); if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + FILE *dstFile; if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { FIO_freeCResources(&ress); return 1; } - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); - if (ress.dstFile == NULL) { /* could not open outFileName */ + dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); + if (dstFile == NULL) { /* could not open outFileName */ error = 1; } else { + WritePool_setDstFile(ress.writePoolCtx, dstFile); for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); if (!status) fCtx->nbFilesProcessed++; error |= status; } - if (fclose(ress.dstFile)) + if (WritePool_closeDstFile(ress.writePoolCtx)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", strerror(errno), outFileName); - ress.dstFile = NULL; } } else { if (outMirroredRootDirName) @@ -1979,152 +2301,42 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, error |= status; } - if (outDirName) - FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); - } - - if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { - UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); - UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); - - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n", - fCtx->nbFilesProcessed, - (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix); - } - - FIO_freeCResources(&ress); - return error; -} - -#endif /* #ifndef ZSTD_NOCOMPRESS */ - - - -#ifndef ZSTD_NODECOMPRESS - -/* ************************************************************************** - * Decompression - ***************************************************************************/ -#define DECOMPRESSION_MAX_WRITE_JOBS (10) - -typedef struct { - /* These struct fields should be set only on creation and not changed afterwards */ - POOL_ctx* writerPool; - int totalWriteJobs; - FIO_prefs_t* prefs; - - /* Controls the file we currently write to, make changes only by using provided utility functions */ - FILE* dstFile; - unsigned storedSkips; - - /* The jobs and availableWriteJobs fields are access by both the main and writer threads and should - * only be mutated after locking the mutex */ - ZSTD_pthread_mutex_t writeJobsMutex; - void* jobs[DECOMPRESSION_MAX_WRITE_JOBS]; - int availableWriteJobs; -} write_pool_ctx_t; - -typedef struct { - /* These fields are automaically set and shouldn't be changed by non WritePool code. */ - write_pool_ctx_t *ctx; - FILE* dstFile; - void *buffer; - size_t bufferSize; - - /* This field should be changed before a job is queued for execution and should contain the number - * of bytes to write from the buffer. */ - size_t usedBufferSize; -} write_job_t; - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - size_t srcBufferLoaded; - ZSTD_DStream* dctx; - write_pool_ctx_t *writePoolCtx; -} dRess_t; - -static write_job_t *FIO_createWriteJob(write_pool_ctx_t *ctx) { - void *buffer; - write_job_t *job; - job = (write_job_t*) malloc(sizeof(write_job_t)); - buffer = malloc(ZSTD_DStreamOutSize()); - if(!job || !buffer) - EXM_THROW(101, "Allocation error : not enough memory"); - job->buffer = buffer; - job->bufferSize = ZSTD_DStreamOutSize(); - job->usedBufferSize = 0; - job->dstFile = NULL; - job->ctx = ctx; - return job; -} - -/* WritePool_createThreadPool: - * Creates a thread pool and a mutex for threaded write pool. - * Displays warning if asyncio is requested but MT isn't available. */ -static void WritePool_createThreadPool(write_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { - ctx->writerPool = NULL; - if(prefs->asyncIO) { -#ifdef ZSTD_MULTITHREAD - if (ZSTD_pthread_mutex_init(&ctx->writeJobsMutex, NULL)) - EXM_THROW(102, "Failed creating write jobs mutex"); - /* We want DECOMPRESSION_MAX_WRITE_JOBS-2 queue items because we need to always have 1 free buffer to - * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ - assert(DECOMPRESSION_MAX_WRITE_JOBS >= 2); - ctx->writerPool = POOL_create(1, DECOMPRESSION_MAX_WRITE_JOBS - 2); - if (!ctx->writerPool) - EXM_THROW(103, "Failed creating writer thread pool"); -#else - DISPLAYLEVEL(2, "Note : asyncio decompression is disabled (lack of multithreading support) \n"); -#endif - } -} - -/* WritePool_create: - * Allocates and sets and a new write pool including its included jobs. */ -static write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs) { - write_pool_ctx_t *ctx; - int i; - ctx = (write_pool_ctx_t*) malloc(sizeof(write_pool_ctx_t)); - if(!ctx) - EXM_THROW(100, "Allocation error : not enough memory"); - WritePool_createThreadPool(ctx, prefs); - ctx->prefs = prefs; - ctx->totalWriteJobs = ctx->writerPool ? DECOMPRESSION_MAX_WRITE_JOBS : 1; - ctx->availableWriteJobs = ctx->totalWriteJobs; - for(i=0; i < ctx->availableWriteJobs; i++) { - ctx->jobs[i] = FIO_createWriteJob(ctx); + if (outDirName) + FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); } - ctx->storedSkips = 0; - ctx->dstFile = NULL; - return ctx; -} -/* WritePool_free: - * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ -static void WritePool_free(write_pool_ctx_t* ctx) { - int i=0; - if(ctx->writerPool) { - /* Make sure we finish all tasks and then free the resources */ - POOL_joinJobs(ctx->writerPool); - /* Make sure we are not leaking jobs */ - assert(ctx->availableWriteJobs==ctx->totalWriteJobs); - POOL_free(ctx->writerPool); - ZSTD_pthread_mutex_destroy(&ctx->writeJobsMutex); - } - assert(ctx->dstFile==NULL); - assert(ctx->storedSkips==0); - for(i=0; iavailableWriteJobs; i++) { - write_job_t* job = (write_job_t*) ctx->jobs[i]; - free(job->buffer); - free(job); + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { + UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); + UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); + + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n", + fCtx->nbFilesProcessed, + (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix); } - free(ctx); + + FIO_freeCResources(&ress); + return error; } +#endif /* #ifndef ZSTD_NOCOMPRESS */ + + + +#ifndef ZSTD_NODECOMPRESS + +/* ************************************************************************** + * Decompression + ***************************************************************************/ +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + size_t srcBufferLoaded; + ZSTD_DStream* dctx; + write_pool_ctx_t *writePoolCtx; +} dRess_t; static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) { @@ -2173,205 +2385,6 @@ static void FIO_consumeDSrcBuffer(dRess_t *ress, size_t len) { memmove(ress->srcBuffer, (char *)ress->srcBuffer + len, ress->srcBufferLoaded); } -/** FIO_fwriteSparse() : -* @return : storedSkips, -* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ -static unsigned -FIO_fwriteSparse(FILE* file, - const void* buffer, size_t bufferSize, - const FIO_prefs_t* const prefs, - unsigned storedSkips) -{ - const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ - size_t bufferSizeT = bufferSize / sizeof(size_t); - const size_t* const bufferTEnd = bufferT + bufferSizeT; - const size_t* ptrT = bufferT; - static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ - - if (prefs->testMode) return 0; /* do not output anything in test mode */ - - if (!prefs->sparseFileSupport) { /* normal write */ - size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); - if (sizeCheck != bufferSize) - EXM_THROW(70, "Write error : cannot write decoded block : %s", - strerror(errno)); - return 0; - } - - /* avoid int overflow */ - if (storedSkips > 1 GB) { - if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) - EXM_THROW(91, "1 GB skip error (sparse file support)"); - storedSkips -= 1 GB; - } - - while (ptrT < bufferTEnd) { - size_t nb0T; - - /* adjust last segment if < 32 KB */ - size_t seg0SizeT = segmentSizeT; - if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; - bufferSizeT -= seg0SizeT; - - /* count leading zeroes */ - for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeof(size_t)); - - if (nb0T != seg0SizeT) { /* not all 0s */ - size_t const nbNon0ST = seg0SizeT - nb0T; - /* skip leading zeros */ - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - /* write the rest */ - if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) - EXM_THROW(93, "Write error : cannot write decoded block : %s", - strerror(errno)); - } - ptrT += seg0SizeT; - } - - { static size_t const maskT = sizeof(size_t)-1; - if (bufferSize & maskT) { - /* size not multiple of sizeof(size_t) : implies end of block */ - const char* const restStart = (const char*)bufferTEnd; - const char* restPtr = restStart; - const char* const restEnd = (const char*)buffer + bufferSize; - assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); - for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) { - /* not all remaining bytes are 0 */ - size_t const restSize = (size_t)(restEnd - restPtr); - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - if (fwrite(restPtr, 1, restSize, file) != restSize) - EXM_THROW(95, "Write error : cannot write end of decoded block : %s", - strerror(errno)); - storedSkips = 0; - } } } - - return storedSkips; -} - -static void -FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) -{ - if (prefs->testMode) assert(storedSkips == 0); - if (storedSkips>0) { - assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ - (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ - if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) - EXM_THROW(69, "Final skip error (sparse file support)"); - /* last zero must be explicitly written, - * so that skipped ones get implicitly translated as zero by FS */ - { const char lastZeroByte[1] = { 0 }; - if (fwrite(lastZeroByte, 1, 1, file) != 1) - EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); - } } -} - -/* WritePool_releaseWriteJob: - * Releases an acquired job back to the pool. Doesn't execute the job. */ -static void WritePool_releaseWriteJob(write_job_t *job) { - write_pool_ctx_t *ctx = job->ctx; - if(ctx->writerPool) { - ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); - assert(ctx->availableWriteJobs < DECOMPRESSION_MAX_WRITE_JOBS); - ctx->jobs[ctx->availableWriteJobs++] = job; - ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); - } else { - ctx->availableWriteJobs++; - } -} - -/* WritePool_acquireWriteJob: - * Returns an available write job to be used for a future write. */ -static write_job_t* WritePool_acquireWriteJob(write_pool_ctx_t *ctx) { - write_job_t *job; - assert(ctx->dstFile!=NULL || ctx->prefs->testMode); - if(ctx->writerPool) { - ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); - assert(ctx->availableWriteJobs > 0); - job = (write_job_t*) ctx->jobs[--ctx->availableWriteJobs]; - ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); - } else { - assert(ctx->availableWriteJobs==1); - ctx->availableWriteJobs--; - job = (write_job_t*)ctx->jobs[0]; - } - job->usedBufferSize = 0; - job->dstFile = ctx->dstFile; - return job; -} - -/* WritePool_executeWriteJob: - * Executes a write job synchronously. Can be used as a function for a thread pool. */ -static void WritePool_executeWriteJob(void* opaque){ - write_job_t* job = (write_job_t*) opaque; - write_pool_ctx_t* ctx = job->ctx; - ctx->storedSkips = FIO_fwriteSparse(job->dstFile, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); - WritePool_releaseWriteJob(job); -} - -/* WritePool_queueWriteJob: - * Queues a write job for execution. - * Make sure to set `usedBufferSize` to the wanted length before call. - * The queued job shouldn't be used directly after queueing it. */ -static void WritePool_queueWriteJob(write_job_t *job) { - write_pool_ctx_t* ctx = job->ctx; - if(ctx->writerPool) - POOL_add(ctx->writerPool, WritePool_executeWriteJob, job); - else - WritePool_executeWriteJob(job); -} - -/* WritePool_queueAndReacquireWriteJob: - * Queues a write job for execution and acquires a new one. - * After execution `job`'s pointed value would change to the newly acquired job. - * Make sure to set `usedBufferSize` to the wanted length before call. - * The queued job shouldn't be used directly after queueing it. */ -static void WritePool_queueAndReacquireWriteJob(write_job_t **job) { - WritePool_queueWriteJob(*job); - *job = WritePool_acquireWriteJob((*job)->ctx); -} - -/* WritePool_sparseWriteEnd: - * Ends sparse writes to the current dstFile. - * Blocks on completion of all current write jobs before executing. */ -static void WritePool_sparseWriteEnd(write_pool_ctx_t* ctx) { - assert(ctx != NULL); - if(ctx->writerPool) - POOL_joinJobs(ctx->writerPool); - FIO_fwriteSparseEnd(ctx->prefs, ctx->dstFile, ctx->storedSkips); - ctx->storedSkips = 0; -} - -/* WritePool_setDstFile: - * Sets the destination file for future files in the pool. - * Requires completion of all queues write jobs and release of all otherwise acquired jobs. - * Also requires ending of sparse write if a previous file was used in sparse mode. */ -static void WritePool_setDstFile(write_pool_ctx_t *ctx, FILE* dstFile) { - assert(ctx!=NULL); - /* We can change the dst file only if we have finished writing */ - if(ctx->writerPool) - POOL_joinJobs(ctx->writerPool); - assert(ctx->storedSkips == 0); - assert(ctx->availableWriteJobs == ctx->totalWriteJobs); - ctx->dstFile = dstFile; -} - -/* WritePool_closeDstFile: - * Ends sparse write and closes the writePool's current dstFile and sets the dstFile to NULL. - * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -static int WritePool_closeDstFile(write_pool_ctx_t *ctx) { - FILE *dstFile = ctx->dstFile; - assert(dstFile!=NULL || ctx->prefs->testMode!=0); - WritePool_sparseWriteEnd(ctx); - WritePool_setDstFile(ctx, NULL); - return fclose(dstFile); -} - /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ static int FIO_passThrough(const FIO_prefs_t* const prefs, @@ -2589,6 +2602,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) } #endif + #ifdef ZSTD_LZMADECOMPRESS static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, @@ -2830,7 +2844,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, /** FIO_decompressDstFile() : open `dstFileName`, - or path-through if ress.dstFile is already != 0, + or pass-through if ress.writePoolCtx->dstFile is already != 0, then start decompression process (FIO_decompressFrames()). @return : 0 : OK 1 : operation aborted From 309987ddf0faf7054599ac96f68136e94fae7a0a Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 7 Jan 2022 15:35:00 -0800 Subject: [PATCH 08/25] ZSTD CLI decompression: Use buffered writes for improved performance --- programs/fileio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/fileio.c b/programs/fileio.c index e4d8faee1b0..c5a2e80bc99 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -726,6 +726,7 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } + setvbuf(f, NULL, _IOFBF, 1024*1024); return f; } } From 479afaecd7fecc1cf376d46e8f055c051d17102f Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Mon, 10 Jan 2022 19:03:25 -0800 Subject: [PATCH 09/25] Async IO compression: bugfix --- programs/fileio.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index c5a2e80bc99..aa913dfe8e2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -973,7 +973,13 @@ typedef struct { static write_job_t *FIO_createWriteJob(write_pool_ctx_t *ctx) { void *buffer; write_job_t *job; - size_t bufferSize = MAX(ZSTD_DStreamOutSize(), ZSTD_CStreamOutSize()); + size_t bufferSize = 0; +#ifndef ZSTD_NOCOMPRESS + bufferSize = ZSTD_CStreamOutSize(); +#endif +#ifndef ZSTD_NODECOMPRESS + bufferSize = MAX(bufferSize, ZSTD_DStreamOutSize()); +#endif job = (write_job_t*) malloc(sizeof(write_job_t)); buffer = malloc(bufferSize); if(!job || !buffer) From 8fbd1ef7ac30ffcbb719822d82431c5d9cdfffd7 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Mon, 10 Jan 2022 19:15:29 -0800 Subject: [PATCH 10/25] Revert "ZSTD CLI decompression: Use buffered writes for improved performance" This reverts commit 50b22c8801e9e1faee7cf97d0939b32e5f45a043. --- programs/fileio.c | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index aa913dfe8e2..fd580c20ed6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -726,7 +726,6 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } - setvbuf(f, NULL, _IOFBF, 1024*1024); return f; } } From 87cae9b9f920a6f46fce830fa675007df81805b9 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Wed, 12 Jan 2022 13:21:02 -0800 Subject: [PATCH 11/25] Async IO compression: moved FIO_fwriteSparse* functions --- programs/fileio.c | 196 +++++++++++++++++++++++----------------------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index fd580c20ed6..5272c39dbf6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -934,6 +934,104 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* return error; } +/** FIO_fwriteSparse() : +* @return : storedSkips, +* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ +static unsigned +FIO_fwriteSparse(FILE* file, + const void* buffer, size_t bufferSize, + const FIO_prefs_t* const prefs, + unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + size_t bufferSizeT = bufferSize / sizeof(size_t); + const size_t* const bufferTEnd = bufferT + bufferSizeT; + const size_t* ptrT = bufferT; + static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ + + if (prefs->testMode) return 0; /* do not output anything in test mode */ + + if (!prefs->sparseFileSupport) { /* normal write */ + size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) + EXM_THROW(70, "Write error : cannot write decoded block : %s", + strerror(errno)); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) { + if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) + EXM_THROW(91, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t nb0T; + + /* adjust last segment if < 32 KB */ + size_t seg0SizeT = segmentSizeT; + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + + /* count leading zeroes */ + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeof(size_t)); + + if (nb0T != seg0SizeT) { /* not all 0s */ + size_t const nbNon0ST = seg0SizeT - nb0T; + /* skip leading zeros */ + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + /* write the rest */ + if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) + EXM_THROW(93, "Write error : cannot write decoded block : %s", + strerror(errno)); + } + ptrT += seg0SizeT; + } + + { static size_t const maskT = sizeof(size_t)-1; + if (bufferSize & maskT) { + /* size not multiple of sizeof(size_t) : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + const char* const restEnd = (const char*)buffer + bufferSize; + assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); + for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + /* not all remaining bytes are 0 */ + size_t const restSize = (size_t)(restEnd - restPtr); + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + if (fwrite(restPtr, 1, restSize, file) != restSize) + EXM_THROW(95, "Write error : cannot write end of decoded block : %s", + strerror(errno)); + storedSkips = 0; + } } } + + return storedSkips; +} + +static void +FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) +{ + if (prefs->testMode) assert(storedSkips == 0); + if (storedSkips>0) { + assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ + (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ + if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) + EXM_THROW(69, "Final skip error (sparse file support)"); + /* last zero must be explicitly written, + * so that skipped ones get implicitly translated as zero by FS */ + { const char lastZeroByte[1] = { 0 }; + if (fwrite(lastZeroByte, 1, 1, file) != 1) + EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); + } } +} + /* ********************************************************************** * AsyncIO functionality ************************************************************************/ @@ -1055,104 +1153,6 @@ static void WritePool_free(write_pool_ctx_t* ctx) { free(ctx); } -/** FIO_fwriteSparse() : -* @return : storedSkips, -* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ -static unsigned -FIO_fwriteSparse(FILE* file, - const void* buffer, size_t bufferSize, - const FIO_prefs_t* const prefs, - unsigned storedSkips) -{ - const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ - size_t bufferSizeT = bufferSize / sizeof(size_t); - const size_t* const bufferTEnd = bufferT + bufferSizeT; - const size_t* ptrT = bufferT; - static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ - - if (prefs->testMode) return 0; /* do not output anything in test mode */ - - if (!prefs->sparseFileSupport) { /* normal write */ - size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); - if (sizeCheck != bufferSize) - EXM_THROW(70, "Write error : cannot write decoded block : %s", - strerror(errno)); - return 0; - } - - /* avoid int overflow */ - if (storedSkips > 1 GB) { - if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) - EXM_THROW(91, "1 GB skip error (sparse file support)"); - storedSkips -= 1 GB; - } - - while (ptrT < bufferTEnd) { - size_t nb0T; - - /* adjust last segment if < 32 KB */ - size_t seg0SizeT = segmentSizeT; - if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; - bufferSizeT -= seg0SizeT; - - /* count leading zeroes */ - for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeof(size_t)); - - if (nb0T != seg0SizeT) { /* not all 0s */ - size_t const nbNon0ST = seg0SizeT - nb0T; - /* skip leading zeros */ - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - /* write the rest */ - if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) - EXM_THROW(93, "Write error : cannot write decoded block : %s", - strerror(errno)); - } - ptrT += seg0SizeT; - } - - { static size_t const maskT = sizeof(size_t)-1; - if (bufferSize & maskT) { - /* size not multiple of sizeof(size_t) : implies end of block */ - const char* const restStart = (const char*)bufferTEnd; - const char* restPtr = restStart; - const char* const restEnd = (const char*)buffer + bufferSize; - assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); - for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) { - /* not all remaining bytes are 0 */ - size_t const restSize = (size_t)(restEnd - restPtr); - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - if (fwrite(restPtr, 1, restSize, file) != restSize) - EXM_THROW(95, "Write error : cannot write end of decoded block : %s", - strerror(errno)); - storedSkips = 0; - } } } - - return storedSkips; -} - -static void -FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) -{ - if (prefs->testMode) assert(storedSkips == 0); - if (storedSkips>0) { - assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ - (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ - if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) - EXM_THROW(69, "Final skip error (sparse file support)"); - /* last zero must be explicitly written, - * so that skipped ones get implicitly translated as zero by FS */ - { const char lastZeroByte[1] = { 0 }; - if (fwrite(lastZeroByte, 1, 1, file) != 1) - EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); - } } -} - /* WritePool_releaseWriteJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ static void WritePool_releaseWriteJob(write_job_t *job) { From 26266007dc27f0f01f5dbf83846af8ab70c77697 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Wed, 12 Jan 2022 13:52:58 -0800 Subject: [PATCH 12/25] Async IO compression: fixed typose --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5272c39dbf6..289f82619f4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1047,7 +1047,7 @@ typedef struct { FILE* dstFile; unsigned storedSkips; - /* The jobs and availableWriteJobs fields are access by both the main and writer threads and should + /* The jobs and availableWriteJobs fields are accessed by both the main and writer threads and should * only be mutated after locking the mutex */ ZSTD_pthread_mutex_t writeJobsMutex; void* jobs[MAX_WRITE_JOBS]; @@ -1055,7 +1055,7 @@ typedef struct { } write_pool_ctx_t; typedef struct { - /* These fields are automaically set and shouldn't be changed by non WritePool code. */ + /* These fields are automatically set and shouldn't be changed by non WritePool code. */ write_pool_ctx_t *ctx; FILE* dstFile; void *buffer; From 89a78e7895423bc89b7d44a0d5409c7734ad89b8 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Wed, 12 Jan 2022 17:12:30 -0800 Subject: [PATCH 13/25] Async IO compression: Refactored WritePool into IoPool so code can be reused for reading. --- programs/fileio.c | 310 ++++++++++++++++++++++++---------------------- 1 file changed, 165 insertions(+), 145 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 289f82619f4..d81cc436e2b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1035,222 +1035,242 @@ FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedS /* ********************************************************************** * AsyncIO functionality ************************************************************************/ -#define MAX_WRITE_JOBS (10) +#define MAX_IO_JOBS (10) typedef struct { /* These struct fields should be set only on creation and not changed afterwards */ - POOL_ctx* writerPool; - int totalWriteJobs; + POOL_ctx* threadPool; + int totalIoJobs; FIO_prefs_t* prefs; + POOL_function poolFunction; /* Controls the file we currently write to, make changes only by using provided utility functions */ - FILE* dstFile; - unsigned storedSkips; + FILE* file; + unsigned storedSkips; // only used for write io pool + U64 offset; - /* The jobs and availableWriteJobs fields are accessed by both the main and writer threads and should + /* The jobs and availableIoJobs fields are accessed by both the main and writer threads and should * only be mutated after locking the mutex */ - ZSTD_pthread_mutex_t writeJobsMutex; - void* jobs[MAX_WRITE_JOBS]; - int availableWriteJobs; -} write_pool_ctx_t; + ZSTD_pthread_mutex_t ioJobsMutex; + void* jobs[MAX_IO_JOBS]; + int availableIoJobs; +} io_pool_ctx_t; typedef struct { /* These fields are automatically set and shouldn't be changed by non WritePool code. */ - write_pool_ctx_t *ctx; - FILE* dstFile; + io_pool_ctx_t *ctx; + FILE* file; void *buffer; size_t bufferSize; /* This field should be changed before a job is queued for execution and should contain the number * of bytes to write from the buffer. */ size_t usedBufferSize; -} write_job_t; + U64 offset; +} io_job_t; -static write_job_t *FIO_createWriteJob(write_pool_ctx_t *ctx) { +static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { void *buffer; - write_job_t *job; - size_t bufferSize = 0; -#ifndef ZSTD_NOCOMPRESS - bufferSize = ZSTD_CStreamOutSize(); -#endif -#ifndef ZSTD_NODECOMPRESS - bufferSize = MAX(bufferSize, ZSTD_DStreamOutSize()); -#endif - job = (write_job_t*) malloc(sizeof(write_job_t)); + io_job_t *job; + job = (io_job_t*) malloc(sizeof(io_job_t)); buffer = malloc(bufferSize); if(!job || !buffer) EXM_THROW(101, "Allocation error : not enough memory"); job->buffer = buffer; job->bufferSize = bufferSize; job->usedBufferSize = 0; - job->dstFile = NULL; + job->file = NULL; job->ctx = ctx; + job->offset = 0; return job; } +static io_job_t *FIO_createWriteJob(io_pool_ctx_t *ctx) { + size_t bufferSize = 0; +#ifndef ZSTD_NOCOMPRESS + bufferSize = ZSTD_CStreamOutSize(); +#endif +#ifndef ZSTD_NODECOMPRESS + bufferSize = MAX(bufferSize, ZSTD_DStreamOutSize()); +#endif + return FIO_createIoJob(ctx, bufferSize); +} -/* WritePool_createThreadPool: - * Creates a thread pool and a mutex for threaded write pool. +/* IO_createThreadPool: + * Creates a thread pool and a mutex for threaded IO pool. * Displays warning if asyncio is requested but MT isn't available. */ -static void WritePool_createThreadPool(write_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { - ctx->writerPool = NULL; +static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { + ctx->threadPool = NULL; if(prefs->asyncIO) { #ifdef ZSTD_MULTITHREAD - if (ZSTD_pthread_mutex_init(&ctx->writeJobsMutex, NULL)) + if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) EXM_THROW(102, "Failed creating write jobs mutex"); - /* We want MAX_WRITE_JOBS-2 queue items because we need to always have 1 free buffer to + /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ - assert(MAX_WRITE_JOBS >= 2); - ctx->writerPool = POOL_create(1, MAX_WRITE_JOBS - 2); - if (!ctx->writerPool) + assert(MAX_IO_JOBS >= 2); + ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2); + if (!ctx->threadPool) EXM_THROW(103, "Failed creating writer thread pool"); #else + /* TODO: move this error message */ DISPLAYLEVEL(2, "Note : asyncio decompression is disabled (lack of multithreading support) \n"); #endif } } -/* WritePool_create: +/* IoPool_create: * Allocates and sets and a new write pool including its included jobs. */ -static write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs) { - write_pool_ctx_t *ctx; +static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function poolFunction) { + io_pool_ctx_t *ctx; int i; - ctx = (write_pool_ctx_t*) malloc(sizeof(write_pool_ctx_t)); + ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); if(!ctx) EXM_THROW(100, "Allocation error : not enough memory"); - WritePool_createThreadPool(ctx, prefs); + IO_createThreadPool(ctx, prefs); ctx->prefs = prefs; - ctx->totalWriteJobs = ctx->writerPool ? MAX_WRITE_JOBS : 1; - ctx->availableWriteJobs = ctx->totalWriteJobs; - for(i=0; i < ctx->availableWriteJobs; i++) { + ctx->poolFunction = poolFunction; + ctx->totalIoJobs = ctx->threadPool ? MAX_IO_JOBS : 1; + ctx->availableIoJobs = ctx->totalIoJobs; + for(i=0; i < ctx->availableIoJobs; i++) { ctx->jobs[i] = FIO_createWriteJob(ctx); } ctx->storedSkips = 0; - ctx->dstFile = NULL; + ctx->offset = 0; + ctx->file = NULL; return ctx; } -/* WritePool_free: +/* IoPool_free: * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ -static void WritePool_free(write_pool_ctx_t* ctx) { +static void IoPool_free(io_pool_ctx_t* ctx) { int i=0; - if(ctx->writerPool) { + if(ctx->threadPool) { /* Make sure we finish all tasks and then free the resources */ - POOL_joinJobs(ctx->writerPool); + POOL_joinJobs(ctx->threadPool); /* Make sure we are not leaking jobs */ - assert(ctx->availableWriteJobs==ctx->totalWriteJobs); - POOL_free(ctx->writerPool); - ZSTD_pthread_mutex_destroy(&ctx->writeJobsMutex); + assert(ctx->availableIoJobs == ctx->totalIoJobs); + POOL_free(ctx->threadPool); + ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); } - assert(ctx->dstFile==NULL); + assert(ctx->file == NULL); assert(ctx->storedSkips==0); - for(i=0; iavailableWriteJobs; i++) { - write_job_t* job = (write_job_t*) ctx->jobs[i]; + for(i=0; iavailableIoJobs; i++) { + io_job_t* job = (io_job_t*) ctx->jobs[i]; free(job->buffer); free(job); } free(ctx); } -/* WritePool_releaseWriteJob: +/* IoPool_releaseWriteJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ -static void WritePool_releaseWriteJob(write_job_t *job) { - write_pool_ctx_t *ctx = job->ctx; - if(ctx->writerPool) { - ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); - assert(ctx->availableWriteJobs < MAX_WRITE_JOBS); - ctx->jobs[ctx->availableWriteJobs++] = job; - ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); +static void IoPool_releaseWriteJob(io_job_t *job) { + io_pool_ctx_t *ctx = job->ctx; + if(ctx->threadPool) { + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->availableIoJobs < MAX_IO_JOBS); + ctx->jobs[ctx->availableIoJobs++] = job; + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); } else { - ctx->availableWriteJobs++; + ctx->availableIoJobs++; } } -/* WritePool_acquireWriteJob: +/* IoPool_acquireJob: * Returns an available write job to be used for a future write. */ -static write_job_t* WritePool_acquireWriteJob(write_pool_ctx_t *ctx) { - write_job_t *job; - assert(ctx->dstFile!=NULL || ctx->prefs->testMode); - if(ctx->writerPool) { - ZSTD_pthread_mutex_lock(&ctx->writeJobsMutex); - assert(ctx->availableWriteJobs > 0); - job = (write_job_t*) ctx->jobs[--ctx->availableWriteJobs]; - ZSTD_pthread_mutex_unlock(&ctx->writeJobsMutex); +static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { + io_job_t *job; + assert(ctx->file != NULL || ctx->prefs->testMode); + if(ctx->threadPool) { + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->availableIoJobs > 0); + job = (io_job_t*) ctx->jobs[--ctx->availableIoJobs]; + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); } else { - assert(ctx->availableWriteJobs==1); - ctx->availableWriteJobs--; - job = (write_job_t*)ctx->jobs[0]; + assert(ctx->availableIoJobs == 1); + ctx->availableIoJobs--; + job = (io_job_t*)ctx->jobs[0]; } job->usedBufferSize = 0; - job->dstFile = ctx->dstFile; + job->file = ctx->file; + job->offset = ctx->offset; return job; } /* WritePool_executeWriteJob: * Executes a write job synchronously. Can be used as a function for a thread pool. */ static void WritePool_executeWriteJob(void* opaque){ - write_job_t* job = (write_job_t*) opaque; - write_pool_ctx_t* ctx = job->ctx; - ctx->storedSkips = FIO_fwriteSparse(job->dstFile, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); - WritePool_releaseWriteJob(job); + io_job_t* job = (io_job_t*) opaque; + io_pool_ctx_t* ctx = job->ctx; + ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); + IoPool_releaseWriteJob(job); } -/* WritePool_queueWriteJob: +/* IoPool_create: + * Allocates and sets and a new write pool including its included jobs. */ +static io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs) { + return IoPool_create(prefs, WritePool_executeWriteJob); +} + +/* IoPool_queueJob: * Queues a write job for execution. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ -static void WritePool_queueWriteJob(write_job_t *job) { - write_pool_ctx_t* ctx = job->ctx; - if(ctx->writerPool) - POOL_add(ctx->writerPool, WritePool_executeWriteJob, job); +static void IoPool_queueJob(io_job_t *job) { + io_pool_ctx_t* ctx = job->ctx; + assert(job->offset == ctx->offset); + ctx->offset += job->usedBufferSize; + if(ctx->threadPool) + POOL_add(ctx->threadPool, WritePool_executeWriteJob, job); else WritePool_executeWriteJob(job); } -/* WritePool_queueAndReacquireWriteJob: +/* IoPool_queueAndReacquireWriteJob: * Queues a write job for execution and acquires a new one. * After execution `job`'s pointed value would change to the newly acquired job. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ -static void WritePool_queueAndReacquireWriteJob(write_job_t **job) { - WritePool_queueWriteJob(*job); - *job = WritePool_acquireWriteJob((*job)->ctx); +static void IoPool_queueAndReacquireWriteJob(io_job_t **job) { + IoPool_queueJob(*job); + *job = IoPool_acquireJob((*job)->ctx); } /* WritePool_sparseWriteEnd: - * Ends sparse writes to the current dstFile. + * Ends sparse writes to the current file. * Blocks on completion of all current write jobs before executing. */ -static void WritePool_sparseWriteEnd(write_pool_ctx_t* ctx) { +static void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx) { assert(ctx != NULL); - if(ctx->writerPool) - POOL_joinJobs(ctx->writerPool); - FIO_fwriteSparseEnd(ctx->prefs, ctx->dstFile, ctx->storedSkips); + if(ctx->threadPool) + POOL_joinJobs(ctx->threadPool); + FIO_fwriteSparseEnd(ctx->prefs, ctx->file, ctx->storedSkips); ctx->storedSkips = 0; } -/* WritePool_setDstFile: +/* IoPool_setFile: * Sets the destination file for future files in the pool. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. * Also requires ending of sparse write if a previous file was used in sparse mode. */ -static void WritePool_setDstFile(write_pool_ctx_t *ctx, FILE* dstFile) { +static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* dstFile) { assert(ctx!=NULL); /* We can change the dst file only if we have finished writing */ - if(ctx->writerPool) - POOL_joinJobs(ctx->writerPool); + if(ctx->threadPool) + POOL_joinJobs(ctx->threadPool); assert(ctx->storedSkips == 0); - assert(ctx->availableWriteJobs == ctx->totalWriteJobs); - ctx->dstFile = dstFile; + assert(ctx->availableIoJobs == ctx->totalIoJobs); + ctx->file = dstFile; + ctx->offset = 0; } /* WritePool_closeDstFile: - * Ends sparse write and closes the writePool's current dstFile and sets the dstFile to NULL. + * Ends sparse write and closes the writePool's current file and sets the file to NULL. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -static int WritePool_closeDstFile(write_pool_ctx_t *ctx) { - FILE *dstFile = ctx->dstFile; +static int WritePool_closeDstFile(io_pool_ctx_t *ctx) { + FILE *dstFile = ctx->file; assert(dstFile!=NULL || ctx->prefs->testMode!=0); WritePool_sparseWriteEnd(ctx); - WritePool_setDstFile(ctx, NULL); + IoPool_setFile(ctx, NULL); return fclose(dstFile); } @@ -1268,7 +1288,7 @@ typedef struct { size_t dictBufferSize; const char* dictFileName; ZSTD_CStream* cctx; - write_pool_ctx_t *writePoolCtx; + io_pool_ctx_t *writePoolCtx; } cRess_t; /** ZSTD_cycleLog() : @@ -1395,7 +1415,7 @@ static void FIO_freeCResources(const cRess_t* const ress) { free(ress->srcBuffer); free(ress->dictBuffer); - WritePool_free(ress->writePoolCtx); + IoPool_free(ress->writePoolCtx); ZSTD_freeCStream(ress->cctx); /* never fails */ } @@ -1408,7 +1428,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { unsigned long long inFileSize = 0, outFileSize = 0; z_stream strm; - write_job_t *writeJob = NULL; + io_job_t *writeJob = NULL; if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION; @@ -1424,7 +1444,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); } } - writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writePoolCtx); strm.next_in = 0; strm.avail_in = 0; strm.next_out = (Bytef*)writeJob->buffer; @@ -1445,7 +1465,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { writeJob->usedBufferSize = cSize; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); outFileSize += cSize; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -1465,7 +1485,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { writeJob->usedBufferSize = cSize; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); outFileSize += cSize; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -1480,7 +1500,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); } } *readsize = inFileSize; - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; } @@ -1497,7 +1517,7 @@ FIO_compressLzmaFrame(cRess_t* ress, lzma_stream strm = LZMA_STREAM_INIT; lzma_action action = LZMA_RUN; lzma_ret ret; - write_job_t *writeJob = NULL; + io_job_t *writeJob = NULL; if (compressionLevel < 0) compressionLevel = 0; if (compressionLevel > 9) compressionLevel = 9; @@ -1515,7 +1535,7 @@ FIO_compressLzmaFrame(cRess_t* ress, EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); } - writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writePoolCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = 0; @@ -1537,7 +1557,7 @@ FIO_compressLzmaFrame(cRess_t* ress, { size_t const compBytes = writeJob->bufferSize - strm.avail_out; if (compBytes) { writeJob->usedBufferSize = compBytes; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); outFileSize += compBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -1556,7 +1576,7 @@ FIO_compressLzmaFrame(cRess_t* ress, lzma_end(&strm); *readsize = inFileSize; - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; @@ -1584,7 +1604,7 @@ FIO_compressLz4Frame(cRess_t* ress, LZ4F_preferences_t prefs; LZ4F_compressionContext_t ctx; - write_job_t *writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ress->writePoolCtx); LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) @@ -1653,13 +1673,13 @@ FIO_compressLz4Frame(cRess_t* ress, srcFileName, LZ4F_getErrorName(headerSize)); writeJob->usedBufferSize = headerSize; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; } *readsize = inFileSize; LZ4F_freeCompressionContext(ctx); - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; @@ -1676,7 +1696,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, { cRess_t const ress = *ressPtr; FILE* const srcFile = ress.srcFile; - write_job_t *writeJob = WritePool_acquireWriteJob(ressPtr->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ressPtr->writePoolCtx); U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; @@ -1749,7 +1769,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); if (outBuff.pos) { writeJob->usedBufferSize = outBuff.pos; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); compressedfilesize += outBuff.pos; } @@ -1889,7 +1909,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned long long)*readsize, (unsigned long long)fileSize); } - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ressPtr->writePoolCtx); return compressedfilesize; @@ -1992,7 +2012,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, /*! FIO_compressFilename_dstFile() : - * open dstFileName, or pass-through if ress.dstFile != NULL, + * open dstFileName, or pass-through if ress.file != NULL, * then start compression with FIO_compressFilename_internal(). * Manages source removal (--rm) and file permissions transfer. * note : ress.srcFile must be != NULL, @@ -2013,7 +2033,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, int transferMTime = 0; FILE *dstFile; assert(ress.srcFile != NULL); - if (ress.writePoolCtx->dstFile == NULL) { + if (ress.writePoolCtx->file == NULL) { int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) @@ -2027,7 +2047,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; /* could not open dstFileName */ - WritePool_setDstFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writePoolCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, * and the user presses Ctrl-C when asked if they wish to overwrite. @@ -2041,7 +2061,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, clearHandler(); DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); - if (WritePool_closeDstFile(ress.writePoolCtx)) { /* error closing dstFile */ + if (WritePool_closeDstFile(ress.writePoolCtx)) { /* error closing file */ DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } @@ -2272,7 +2292,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, if (dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - WritePool_setDstFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writePoolCtx, dstFile); for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); if (!status) fCtx->nbFilesProcessed++; @@ -2341,7 +2361,7 @@ typedef struct { size_t srcBufferSize; size_t srcBufferLoaded; ZSTD_DStream* dctx; - write_pool_ctx_t *writePoolCtx; + io_pool_ctx_t *writePoolCtx; } dRess_t; static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) @@ -2380,7 +2400,7 @@ static void FIO_freeDResources(dRess_t ress) { CHECK( ZSTD_freeDStream(ress.dctx) ); free(ress.srcBuffer); - WritePool_free(ress.writePoolCtx); + IoPool_free(ress.writePoolCtx); } /* FIO_consumeDSrcBuffer: @@ -2466,7 +2486,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, U64 alreadyDecoded) /* for multi-frames streams */ { U64 frameSize = 0; - write_job_t *writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ress->writePoolCtx); /* display last 20 characters only */ { size_t const srcFileLength = strlen(srcFileName); @@ -2499,7 +2519,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, /* Write block */ writeJob->usedBufferSize = outBuff.pos; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); frameSize += outBuff.pos; if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); @@ -2534,7 +2554,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ress->srcBufferLoaded += readSize; } } } - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return frameSize; @@ -2549,7 +2569,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) z_stream strm; int flush = Z_NO_FLUSH; int decodingError = 0; - write_job_t *writeJob = NULL; + io_job_t *writeJob = NULL; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; @@ -2560,7 +2580,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return FIO_ERROR_FRAME_DECODING; - writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writePoolCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.avail_in = (uInt)ress->srcBufferLoaded; @@ -2586,7 +2606,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -2602,7 +2622,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); decodingError = 1; } - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } @@ -2619,7 +2639,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, lzma_action action = LZMA_RUN; lzma_ret initRet; int decodingError = 0; - write_job_t *writeJob = NULL; + io_job_t *writeJob = NULL; strm.next_in = 0; strm.avail_in = 0; @@ -2636,7 +2656,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, return FIO_ERROR_FRAME_DECODING; } - writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writePoolCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = (BYTE const*)ress->srcBuffer; @@ -2664,7 +2684,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -2674,7 +2694,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, FIO_consumeDSrcBuffer(ress, ress->srcBufferLoaded - strm.avail_in); lzma_end(&strm); - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } @@ -2690,7 +2710,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, LZ4F_decompressionContext_t dCtx; LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); int decodingError = 0; - write_job_t *writeJob = WritePool_acquireWriteJob(ress->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ress->writePoolCtx); if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); @@ -2733,7 +2753,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, if (decodedBytes) { UTIL_HumanReadableSize_t hrs; writeJob->usedBufferSize = decodedBytes; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_queueAndReacquireWriteJob(&writeJob); filesize += decodedBytes; hrs = UTIL_makeHumanReadableSize(filesize); DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); @@ -2749,7 +2769,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, } LZ4F_freeDecompressionContext(dCtx); - WritePool_releaseWriteJob(writeJob); + IoPool_releaseWriteJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; @@ -2827,7 +2847,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, #endif } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ return FIO_passThrough(prefs, - ress.writePoolCtx->dstFile, srcFile, + ress.writePoolCtx->file, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); } else { @@ -2850,7 +2870,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, /** FIO_decompressDstFile() : open `dstFileName`, - or pass-through if ress.writePoolCtx->dstFile is already != 0, + or pass-through if ress.writePoolCtx->file is already != 0, then start decompression process (FIO_decompressFrames()). @return : 0 : OK 1 : operation aborted @@ -2865,7 +2885,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, int releaseDstFile = 0; int transferMTime = 0; - if ((ress.writePoolCtx->dstFile == NULL) && (prefs->testMode==0)) { + if ((ress.writePoolCtx->file == NULL) && (prefs->testMode == 0)) { FILE *dstFile; int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ @@ -2880,7 +2900,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; - WritePool_setDstFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writePoolCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -3109,7 +3129,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, if (!prefs->testMode) { FILE* dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); - WritePool_setDstFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writePoolCtx, dstFile); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); From 4c492331997c3e2985b7377ad5514bfcca6b10c9 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Wed, 12 Jan 2022 18:55:04 -0800 Subject: [PATCH 14/25] Async IO compression: Added ReaderPool and more refactors. --- programs/fileio.c | 174 ++++++++++++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 58 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d81cc436e2b..f0baa18a333 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1047,13 +1047,20 @@ typedef struct { /* Controls the file we currently write to, make changes only by using provided utility functions */ FILE* file; unsigned storedSkips; // only used for write io pool + int reachedEof; U64 offset; + U64 nextNeededOffset; - /* The jobs and availableIoJobs fields are accessed by both the main and writer threads and should + /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should * only be mutated after locking the mutex */ ZSTD_pthread_mutex_t ioJobsMutex; - void* jobs[MAX_IO_JOBS]; - int availableIoJobs; + void* availableJobs[MAX_IO_JOBS]; + int availableJobsCount; + + void* completedJobs[MAX_IO_JOBS]; + int completedJobsCount; + ZSTD_pthread_cond_t jobCompletedCond; + } io_pool_ctx_t; typedef struct { @@ -1086,16 +1093,6 @@ static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { return job; } -static io_job_t *FIO_createWriteJob(io_pool_ctx_t *ctx) { - size_t bufferSize = 0; -#ifndef ZSTD_NOCOMPRESS - bufferSize = ZSTD_CStreamOutSize(); -#endif -#ifndef ZSTD_NODECOMPRESS - bufferSize = MAX(bufferSize, ZSTD_DStreamOutSize()); -#endif - return FIO_createIoJob(ctx, bufferSize); -} /* IO_createThreadPool: * Creates a thread pool and a mutex for threaded IO pool. @@ -1105,13 +1102,15 @@ static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { if(prefs->asyncIO) { #ifdef ZSTD_MULTITHREAD if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) - EXM_THROW(102, "Failed creating write jobs mutex"); + EXM_THROW(102, "Failed creating write availableJobs mutex"); + if(ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL)) + EXM_THROW(103, "Failed creating write jobCompletedCond mutex"); /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ assert(MAX_IO_JOBS >= 2); ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2); if (!ctx->threadPool) - EXM_THROW(103, "Failed creating writer thread pool"); + EXM_THROW(104, "Failed creating writer thread pool"); #else /* TODO: move this error message */ DISPLAYLEVEL(2, "Note : asyncio decompression is disabled (lack of multithreading support) \n"); @@ -1120,8 +1119,9 @@ static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { } /* IoPool_create: - * Allocates and sets and a new write pool including its included jobs. */ -static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function poolFunction) { + * Allocates and sets and a new write pool including its included availableJobs. */ +static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function poolFunction, + size_t bufferSize) { io_pool_ctx_t *ctx; int i; ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); @@ -1131,12 +1131,15 @@ static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function pool ctx->prefs = prefs; ctx->poolFunction = poolFunction; ctx->totalIoJobs = ctx->threadPool ? MAX_IO_JOBS : 1; - ctx->availableIoJobs = ctx->totalIoJobs; - for(i=0; i < ctx->availableIoJobs; i++) { - ctx->jobs[i] = FIO_createWriteJob(ctx); + ctx->availableJobsCount = ctx->totalIoJobs; + for(i=0; i < ctx->availableJobsCount; i++) { + ctx->availableJobs[i] = FIO_createIoJob(ctx, bufferSize); } + ctx->completedJobsCount = 0; ctx->storedSkips = 0; + ctx->reachedEof = 0; ctx->offset = 0; + ctx->nextNeededOffset = 0; ctx->file = NULL; return ctx; } @@ -1148,32 +1151,32 @@ static void IoPool_free(io_pool_ctx_t* ctx) { if(ctx->threadPool) { /* Make sure we finish all tasks and then free the resources */ POOL_joinJobs(ctx->threadPool); - /* Make sure we are not leaking jobs */ - assert(ctx->availableIoJobs == ctx->totalIoJobs); + /* Make sure we are not leaking availableJobs */ + assert(ctx->availableJobsCount == ctx->totalIoJobs); POOL_free(ctx->threadPool); ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); } assert(ctx->file == NULL); assert(ctx->storedSkips==0); - for(i=0; iavailableIoJobs; i++) { - io_job_t* job = (io_job_t*) ctx->jobs[i]; + for(i=0; iavailableJobsCount; i++) { + io_job_t* job = (io_job_t*) ctx->availableJobs[i]; free(job->buffer); free(job); } free(ctx); } -/* IoPool_releaseWriteJob: +/* IoPool_releaseIoJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ -static void IoPool_releaseWriteJob(io_job_t *job) { +static void IoPool_releaseIoJob(io_job_t *job) { io_pool_ctx_t *ctx = job->ctx; if(ctx->threadPool) { ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - assert(ctx->availableIoJobs < MAX_IO_JOBS); - ctx->jobs[ctx->availableIoJobs++] = job; + assert(ctx->availableJobsCount < MAX_IO_JOBS); + ctx->availableJobs[ctx->availableJobsCount++] = job; ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); } else { - ctx->availableIoJobs++; + ctx->availableJobsCount++; } } @@ -1184,13 +1187,13 @@ static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { assert(ctx->file != NULL || ctx->prefs->testMode); if(ctx->threadPool) { ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - assert(ctx->availableIoJobs > 0); - job = (io_job_t*) ctx->jobs[--ctx->availableIoJobs]; + assert(ctx->availableJobsCount > 0); + job = (io_job_t*) ctx->availableJobs[--ctx->availableJobsCount]; ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); } else { - assert(ctx->availableIoJobs == 1); - ctx->availableIoJobs--; - job = (io_job_t*)ctx->jobs[0]; + assert(ctx->availableJobsCount == 1); + ctx->availableJobsCount--; + job = (io_job_t*)ctx->availableJobs[0]; } job->usedBufferSize = 0; job->file = ctx->file; @@ -1198,19 +1201,38 @@ static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { return job; } -/* WritePool_executeWriteJob: - * Executes a write job synchronously. Can be used as a function for a thread pool. */ -static void WritePool_executeWriteJob(void* opaque){ - io_job_t* job = (io_job_t*) opaque; - io_pool_ctx_t* ctx = job->ctx; - ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); - IoPool_releaseWriteJob(job); +/* IoPool_addJobToCompleted */ +static void IoPool_addJobToCompleted(io_job_t *job) { + io_pool_ctx_t *ctx = job->ctx; + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->completedJobsCount < MAX_IO_JOBS); + ctx->completedJobs[ctx->completedJobsCount++] = job; + ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); } -/* IoPool_create: - * Allocates and sets and a new write pool including its included jobs. */ -static io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs) { - return IoPool_create(prefs, WritePool_executeWriteJob); +/* IoPool_addJobToCompleted */ +static io_job_t* IoPool_getNextCompletedJob() { + io_job_t *job = NULL; + io_pool_ctx_t *ctx = job->ctx; + int i; + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + while(ctx->completedJobsCount == 0 && ctx->availableJobsCount < ctx->totalIoJobs) { + ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->ioJobsMutex); + if(ctx->completedJobsCount == 0 && ctx->availableJobsCount == ctx->totalIoJobs) { + job = NULL; + } else { + for(i=0; i < ctx->totalIoJobs-ctx->availableJobsCount; i++) { + job = (io_job_t *)ctx->completedJobs[i]; + if(job->offset == ctx->nextNeededOffset) { + ctx->nextNeededOffset += job->usedBufferSize; + break; + } + } + } + } + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + return job; } /* IoPool_queueJob: @@ -1222,9 +1244,9 @@ static void IoPool_queueJob(io_job_t *job) { assert(job->offset == ctx->offset); ctx->offset += job->usedBufferSize; if(ctx->threadPool) - POOL_add(ctx->threadPool, WritePool_executeWriteJob, job); + POOL_add(ctx->threadPool, ctx->poolFunction, job); else - WritePool_executeWriteJob(job); + ctx->poolFunction(job); } /* IoPool_queueAndReacquireWriteJob: @@ -1258,9 +1280,10 @@ static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* dstFile) { if(ctx->threadPool) POOL_joinJobs(ctx->threadPool); assert(ctx->storedSkips == 0); - assert(ctx->availableIoJobs == ctx->totalIoJobs); + assert(ctx->availableJobsCount == ctx->totalIoJobs); ctx->file = dstFile; ctx->offset = 0; + ctx->nextNeededOffset = 0; } /* WritePool_closeDstFile: @@ -1274,6 +1297,40 @@ static int WritePool_closeDstFile(io_pool_ctx_t *ctx) { return fclose(dstFile); } +/* WritePool_executeWriteJob: + * Executes a write job synchronously. Can be used as a function for a thread pool. */ +static void WritePool_executeWriteJob(void* opaque){ + io_job_t* job = (io_job_t*) opaque; + io_pool_ctx_t* ctx = job->ctx; + ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); + IoPool_releaseIoJob(job); +} + +/* WritePool_create: + * Allocates and sets and a new write pool including its included jobs. */ +static io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) { + return IoPool_create(prefs, WritePool_executeWriteJob, bufferSize); +} + +/* ReadPool_executeReadJob: + * Executes a read job synchronously. Can be used as a function for a thread pool. */ +static void ReadPool_executeReadJob(void* opaque){ + io_job_t* job = (io_job_t*) opaque; + io_pool_ctx_t* ctx = job->ctx; + if(ctx->reachedEof) { + job->usedBufferSize = 0; + return; + } + job->usedBufferSize = fread(job->buffer, 1, job->bufferSize, job->file); + if(job->usedBufferSize < job->bufferSize) + ctx->reachedEof = 1; +} + +/* ReadPool_create: + * Allocates and sets and a new write pool including its included jobs. */ +static io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { + return IoPool_create(prefs, ReadPool_executeReadJob, bufferSize); +} #ifndef ZSTD_NOCOMPRESS @@ -1289,6 +1346,7 @@ typedef struct { const char* dictFileName; ZSTD_CStream* cctx; io_pool_ctx_t *writePoolCtx; + io_pool_ctx_t *readPoolCtx; } cRess_t; /** ZSTD_cycleLog() : @@ -1350,7 +1408,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, if (!ress.srcBuffer) EXM_THROW(31, "allocation error : not enough memory"); - ress.writePoolCtx = WritePool_create(prefs); + ress.writePoolCtx = WritePool_create(prefs, ZSTD_CStreamOutSize()); /* Advanced parameters, including dictionary */ if (dictFileName && (ress.dictBuffer==NULL)) @@ -1500,7 +1558,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); } } *readsize = inFileSize; - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; } @@ -1576,7 +1634,7 @@ FIO_compressLzmaFrame(cRess_t* ress, lzma_end(&strm); *readsize = inFileSize; - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; @@ -1679,7 +1737,7 @@ FIO_compressLz4Frame(cRess_t* ress, *readsize = inFileSize; LZ4F_freeCompressionContext(ctx); - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return outFileSize; @@ -1909,7 +1967,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned long long)*readsize, (unsigned long long)fileSize); } - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ressPtr->writePoolCtx); return compressedfilesize; @@ -2391,7 +2449,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi free(dictBuffer); } - ress.writePoolCtx = WritePool_create(prefs); + ress.writePoolCtx = WritePool_create(prefs, ZSTD_DStreamOutSize()); return ress; } @@ -2554,7 +2612,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ress->srcBufferLoaded += readSize; } } } - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return frameSize; @@ -2622,7 +2680,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); decodingError = 1; } - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } @@ -2694,7 +2752,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, FIO_consumeDSrcBuffer(ress, ress->srcBufferLoaded - strm.avail_in); lzma_end(&strm); - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } @@ -2769,7 +2827,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, } LZ4F_freeDecompressionContext(dCtx); - IoPool_releaseWriteJob(writeJob); + IoPool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writePoolCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; From 688f4bce23aa04b46266c7583601bef96f80fcde Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 11:08:18 -0800 Subject: [PATCH 15/25] Async IO compression: Async read working but still needs work. WIP. --- programs/fileio.c | 587 +++++++++++++++++++++++++-------------------- tests/playTests.sh | 2 +- 2 files changed, 333 insertions(+), 256 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f0baa18a333..4ee6dc85b44 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -563,7 +563,13 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) } void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) { +#ifdef ZSTD_MULTITHREAD prefs->asyncIO = value; +#else + (void) prefs; + (void) value; + DISPLAYLEVEL(2, "Note : asyncio is disabled (lack of multithreading support) \n"); +#endif } /* FIO_ctx_t functions */ @@ -726,6 +732,7 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } + setvbuf(f, NULL, _IOFBF, 1024*1024); return f; } } @@ -1048,8 +1055,8 @@ typedef struct { FILE* file; unsigned storedSkips; // only used for write io pool int reachedEof; - U64 offset; - U64 nextNeededOffset; + U64 nextReadOffset; + U64 waitingOnOffset; /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should * only be mutated after locking the mutex */ @@ -1061,6 +1068,10 @@ typedef struct { int completedJobsCount; ZSTD_pthread_cond_t jobCompletedCond; + U8 *srcBufferBase; + size_t srcBufferBaseSize; + U8 *srcBuffer; + size_t srcBufferLoaded; } io_pool_ctx_t; typedef struct { @@ -1100,7 +1111,6 @@ static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { ctx->threadPool = NULL; if(prefs->asyncIO) { -#ifdef ZSTD_MULTITHREAD if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) EXM_THROW(102, "Failed creating write availableJobs mutex"); if(ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL)) @@ -1111,10 +1121,6 @@ static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2); if (!ctx->threadPool) EXM_THROW(104, "Failed creating writer thread pool"); -#else - /* TODO: move this error message */ - DISPLAYLEVEL(2, "Note : asyncio decompression is disabled (lack of multithreading support) \n"); -#endif } } @@ -1138,16 +1144,49 @@ static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function pool ctx->completedJobsCount = 0; ctx->storedSkips = 0; ctx->reachedEof = 0; - ctx->offset = 0; - ctx->nextNeededOffset = 0; + ctx->nextReadOffset = 0; + ctx->waitingOnOffset = 0; ctx->file = NULL; + + ctx->srcBufferBaseSize = 2 * bufferSize; + ctx->srcBufferBase = (U8*) malloc(ctx->srcBufferBaseSize); + ctx->srcBuffer = ctx->srcBufferBase; + ctx->srcBufferLoaded = 0; + return ctx; } + +/* IoPool_releaseIoJob: + * Releases an acquired job back to the pool. Doesn't execute the job. */ +static void IoPool_releaseIoJob(io_job_t *job) { + io_pool_ctx_t *ctx = job->ctx; + if(ctx->threadPool) { + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->availableJobsCount < MAX_IO_JOBS); + ctx->availableJobs[ctx->availableJobsCount++] = job; + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + } else { + assert(ctx->availableJobsCount == 0); + ctx->availableJobsCount++; + } +} + +static void IoPool_releaseAllCompletedJobs(io_pool_ctx_t* ctx) { + int i=0; + for(i=0; icompletedJobsCount; i++) { + io_job_t* job = (io_job_t*) ctx->completedJobs[i]; + IoPool_releaseIoJob(job); + } + ctx->completedJobsCount = 0; +} + + /* IoPool_free: * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ static void IoPool_free(io_pool_ctx_t* ctx) { int i=0; + IoPool_releaseAllCompletedJobs(ctx); if(ctx->threadPool) { /* Make sure we finish all tasks and then free the resources */ POOL_joinJobs(ctx->threadPool); @@ -1155,6 +1194,7 @@ static void IoPool_free(io_pool_ctx_t* ctx) { assert(ctx->availableJobsCount == ctx->totalIoJobs); POOL_free(ctx->threadPool); ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); + ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); } assert(ctx->file == NULL); assert(ctx->storedSkips==0); @@ -1163,23 +1203,10 @@ static void IoPool_free(io_pool_ctx_t* ctx) { free(job->buffer); free(job); } + free(ctx->srcBuffer); free(ctx); } -/* IoPool_releaseIoJob: - * Releases an acquired job back to the pool. Doesn't execute the job. */ -static void IoPool_releaseIoJob(io_job_t *job) { - io_pool_ctx_t *ctx = job->ctx; - if(ctx->threadPool) { - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - assert(ctx->availableJobsCount < MAX_IO_JOBS); - ctx->availableJobs[ctx->availableJobsCount++] = job; - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); - } else { - ctx->availableJobsCount++; - } -} - /* IoPool_acquireJob: * Returns an available write job to be used for a future write. */ static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { @@ -1197,65 +1224,80 @@ static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { } job->usedBufferSize = 0; job->file = ctx->file; - job->offset = ctx->offset; + job->offset = 0; return job; } /* IoPool_addJobToCompleted */ static void IoPool_addJobToCompleted(io_job_t *job) { io_pool_ctx_t *ctx = job->ctx; - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + if(ctx->threadPool) + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); assert(ctx->completedJobsCount < MAX_IO_JOBS); ctx->completedJobs[ctx->completedJobsCount++] = job; - ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + if(ctx->threadPool) { + ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + } } -/* IoPool_addJobToCompleted */ -static io_job_t* IoPool_getNextCompletedJob() { +/* assuming ioJobsMutex is locked */ +static io_job_t* IoPool_findWaitingJob(io_pool_ctx_t *ctx) { io_job_t *job = NULL; - io_pool_ctx_t *ctx = job->ctx; - int i; - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - while(ctx->completedJobsCount == 0 && ctx->availableJobsCount < ctx->totalIoJobs) { - ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->ioJobsMutex); - if(ctx->completedJobsCount == 0 && ctx->availableJobsCount == ctx->totalIoJobs) { - job = NULL; - } else { - for(i=0; i < ctx->totalIoJobs-ctx->availableJobsCount; i++) { - job = (io_job_t *)ctx->completedJobs[i]; - if(job->offset == ctx->nextNeededOffset) { - ctx->nextNeededOffset += job->usedBufferSize; - break; - } - } + int i = 0; + for (i=0; icompletedJobsCount; i++) { + job = (io_job_t *) ctx->completedJobs[i]; + if (job->offset == ctx->waitingOnOffset) { + ctx->completedJobs[i] = ctx->completedJobs[--ctx->completedJobsCount]; + return job; } } - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + return NULL; +} + +/* IoPool_getNextCompletedJob */ +static io_job_t* IoPool_getNextCompletedJob(io_pool_ctx_t *ctx) { + io_job_t *job = NULL; + if(ctx->threadPool) + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + + job = IoPool_findWaitingJob(ctx); + + while (!job && (ctx->availableJobsCount + ctx->completedJobsCount < ctx->totalIoJobs)) { + assert(ctx->threadPool != NULL); + ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->ioJobsMutex); + job = IoPool_findWaitingJob(ctx); + } + + if(job) { + assert(job->offset == ctx->waitingOnOffset); + ctx->waitingOnOffset += job->usedBufferSize; + } + + if(ctx->threadPool) + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); return job; } -/* IoPool_queueJob: +/* IoPool_enqueueJob: * Queues a write job for execution. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ -static void IoPool_queueJob(io_job_t *job) { +static void IoPool_enqueueJob(io_job_t *job) { io_pool_ctx_t* ctx = job->ctx; - assert(job->offset == ctx->offset); - ctx->offset += job->usedBufferSize; if(ctx->threadPool) POOL_add(ctx->threadPool, ctx->poolFunction, job); else ctx->poolFunction(job); } -/* IoPool_queueAndReacquireWriteJob: +/* IoPool_enqueueAndReacquireWriteJob: * Queues a write job for execution and acquires a new one. * After execution `job`'s pointed value would change to the newly acquired job. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ -static void IoPool_queueAndReacquireWriteJob(io_job_t **job) { - IoPool_queueJob(*job); +static void IoPool_enqueueAndReacquireWriteJob(io_job_t **job) { + IoPool_enqueueJob(*job); *job = IoPool_acquireJob((*job)->ctx); } @@ -1274,16 +1316,19 @@ static void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx) { * Sets the destination file for future files in the pool. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. * Also requires ending of sparse write if a previous file was used in sparse mode. */ -static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* dstFile) { +static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { assert(ctx!=NULL); /* We can change the dst file only if we have finished writing */ + IoPool_releaseAllCompletedJobs(ctx); if(ctx->threadPool) POOL_joinJobs(ctx->threadPool); assert(ctx->storedSkips == 0); assert(ctx->availableJobsCount == ctx->totalIoJobs); - ctx->file = dstFile; - ctx->offset = 0; - ctx->nextNeededOffset = 0; + ctx->file = file; + ctx->nextReadOffset = 0; + ctx->waitingOnOffset = 0; + ctx->srcBuffer = ctx->srcBufferBase; + ctx->reachedEof = 0; } /* WritePool_closeDstFile: @@ -1319,11 +1364,19 @@ static void ReadPool_executeReadJob(void* opaque){ io_pool_ctx_t* ctx = job->ctx; if(ctx->reachedEof) { job->usedBufferSize = 0; + IoPool_addJobToCompleted(job); return; } job->usedBufferSize = fread(job->buffer, 1, job->bufferSize, job->file); - if(job->usedBufferSize < job->bufferSize) - ctx->reachedEof = 1; + if(job->usedBufferSize < job->bufferSize) { + if(ferror(job->file)) { + EXM_THROW(37, "Read error"); + } else if(feof(job->file)) { + ctx->reachedEof = 1; + } else + EXM_THROW(37, "Unexpected short read"); + } + IoPool_addJobToCompleted(job); } /* ReadPool_create: @@ -1332,21 +1385,73 @@ static io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSiz return IoPool_create(prefs, ReadPool_executeReadJob, bufferSize); } +static void ReadPool_consumeBytes(io_pool_ctx_t *ctx, size_t n) { + assert(n <= ctx->srcBufferLoaded); + assert(ctx->srcBuffer + n <= ctx->srcBufferBase + ctx->srcBufferBaseSize); + ctx->srcBufferLoaded -= n; + ctx->srcBuffer += n; +} + +static void ReadPool_enqueueRead(io_pool_ctx_t *ctx) { + io_job_t *job = IoPool_acquireJob(ctx); + job->offset = ctx->nextReadOffset; + ctx->nextReadOffset += job->bufferSize; + IoPool_enqueueJob(job); +} + + +static size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n) { + io_job_t *job; + size_t srcBufferOffsetFromBase; + size_t srcBufferRemainingSpace; + size_t bytesRead = 0; + while (ctx->srcBufferLoaded < n) { + job = IoPool_getNextCompletedJob(ctx); + if(job == NULL) + break; + srcBufferOffsetFromBase = ctx->srcBuffer - ctx->srcBufferBase; + srcBufferRemainingSpace = ctx->srcBufferBaseSize - (srcBufferOffsetFromBase + ctx->srcBufferLoaded); + if(job->usedBufferSize > srcBufferRemainingSpace) { + memmove(ctx->srcBufferBase, ctx->srcBuffer, ctx->srcBufferLoaded); + ctx->srcBuffer = ctx->srcBufferBase; + } + memcpy(ctx->srcBuffer + ctx->srcBufferLoaded, job->buffer, job->usedBufferSize); + bytesRead += job->usedBufferSize; + ctx->srcBufferLoaded += job->usedBufferSize; + if(job->usedBufferSize < job->bufferSize) { + IoPool_releaseIoJob(job); + break; + } + IoPool_releaseIoJob(job); + ReadPool_enqueueRead(ctx); + } + return bytesRead; +} + +static size_t ReadPool_consumeAndReadAll(io_pool_ctx_t *ctx) { + ReadPool_consumeBytes(ctx, ctx->srcBufferLoaded); + return ReadPool_readBuffer(ctx, ZSTD_DStreamInSize()); +} + +static void ReadPool_startReading(io_pool_ctx_t *ctx) { + int i; + for (i = 0; i < ctx->availableJobsCount; i++) { + ReadPool_enqueueRead(ctx); + } +} + #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** * Compression ************************************************************************/ typedef struct { - FILE* srcFile; - void* srcBuffer; - size_t srcBufferSize; void* dictBuffer; size_t dictBufferSize; const char* dictFileName; ZSTD_CStream* cctx; - io_pool_ctx_t *writePoolCtx; - io_pool_ctx_t *readPoolCtx; + io_pool_ctx_t *writeCtx; + io_pool_ctx_t *readCtx; } cRess_t; /** ZSTD_cycleLog() : @@ -1395,8 +1500,6 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, if (ress.cctx == NULL) EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx", strerror(errno)); - ress.srcBufferSize = ZSTD_CStreamInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); /* need to update memLimit before calling createDictBuffer * because of memLimit check inside it */ @@ -1405,10 +1508,9 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); } ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ - if (!ress.srcBuffer) - EXM_THROW(31, "allocation error : not enough memory"); - ress.writePoolCtx = WritePool_create(prefs, ZSTD_CStreamOutSize()); + ress.writeCtx = WritePool_create(prefs, ZSTD_CStreamOutSize()); + ress.readCtx = ReadPool_create(prefs, ZSTD_DStreamInSize()); /* Advanced parameters, including dictionary */ if (dictFileName && (ress.dictBuffer==NULL)) @@ -1471,9 +1573,9 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, static void FIO_freeCResources(const cRess_t* const ress) { - free(ress->srcBuffer); free(ress->dictBuffer); - IoPool_free(ress->writePoolCtx); + IoPool_free(ress->writeCtx); + IoPool_free(ress->readCtx); ZSTD_freeCStream(ress->cctx); /* never fails */ } @@ -1502,7 +1604,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); } } - writeJob = IoPool_acquireJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writeCtx); strm.next_in = 0; strm.avail_in = 0; strm.next_out = (Bytef*)writeJob->buffer; @@ -1511,19 +1613,25 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no while (1) { int ret; if (strm.avail_in == 0) { - size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); - if (inSize == 0) break; - inFileSize += inSize; - strm.next_in = (z_const unsigned char*)ress->srcBuffer; - strm.avail_in = (uInt)inSize; + ReadPool_readBuffer(ress->readCtx, ZSTD_CStreamInSize()); + if (ress->readCtx->srcBufferLoaded == 0) break; + inFileSize += ress->readCtx->srcBufferLoaded; + strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; + strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; + } + + { + size_t const availBefore = strm.avail_in; + ret = deflate(&strm, Z_NO_FLUSH); + ReadPool_consumeBytes(ress->readCtx, availBefore - strm.avail_in); } - ret = deflate(&strm, Z_NO_FLUSH); + if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { writeJob->usedBufferSize = cSize; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += cSize; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -1543,7 +1651,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { writeJob->usedBufferSize = cSize; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += cSize; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -1559,7 +1667,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no } } *readsize = inFileSize; IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return outFileSize; } #endif @@ -1593,7 +1701,7 @@ FIO_compressLzmaFrame(cRess_t* ress, EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); } - writeJob = IoPool_acquireJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writeCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = 0; @@ -1601,21 +1709,26 @@ FIO_compressLzmaFrame(cRess_t* ress, while (1) { if (strm.avail_in == 0) { - size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); - if (inSize == 0) action = LZMA_FINISH; + size_t const inSize = ReadPool_readBuffer(ress->readCtx, ZSTD_CStreamInSize()); + if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH; inFileSize += inSize; - strm.next_in = (BYTE const*)ress->srcBuffer; - strm.avail_in = inSize; + strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; + strm.avail_in = ress->readCtx->srcBufferLoaded; + } + + { + size_t const availBefore = strm.avail_in; + ret = lzma_code(&strm, action); + ReadPool_consumeBytes(ress->readCtx, availBefore - strm.avail_in); } - ret = lzma_code(&strm, action); if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); { size_t const compBytes = writeJob->bufferSize - strm.avail_out; if (compBytes) { writeJob->usedBufferSize = compBytes; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += compBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -1635,7 +1748,7 @@ FIO_compressLzmaFrame(cRess_t* ress, *readsize = inFileSize; IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return outFileSize; } @@ -1662,7 +1775,7 @@ FIO_compressLz4Frame(cRess_t* ress, LZ4F_preferences_t prefs; LZ4F_compressionContext_t ctx; - io_job_t *writeJob = IoPool_acquireJob(ress->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ress->writeCtx); LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) @@ -1670,7 +1783,7 @@ FIO_compressLz4Frame(cRess_t* ress, memset(&prefs, 0, sizeof(prefs)); - assert(blockSize <= ress->srcBufferSize); + assert(blockSize <= ress->readCtx->srcBufferBaseSize); prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; @@ -1689,17 +1802,17 @@ FIO_compressLz4Frame(cRess_t* ress, EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); writeJob->usedBufferSize = headerSize; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; /* Read first block */ - readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); + readSize = ReadPool_readBuffer(ress->readCtx, blockSize); inFileSize += readSize; /* Main Loop */ - while (readSize>0) { + while (ress->readCtx->srcBufferLoaded) { size_t const outSize = LZ4F_compressUpdate(ctx, writeJob->buffer, writeJob->bufferSize, - ress->srcBuffer, readSize, NULL); + ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, NULL); if (LZ4F_isError(outSize)) EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", srcFileName, LZ4F_getErrorName(outSize)); @@ -1716,13 +1829,13 @@ FIO_compressLz4Frame(cRess_t* ress, /* Write Block */ writeJob->usedBufferSize = outSize; - WritePool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); /* Read next block */ - readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); + ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded); + readSize = ReadPool_readBuffer(ress->readCtx, blockSize); inFileSize += readSize; } - if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); /* End of Stream mark */ headerSize = LZ4F_compressEnd(ctx, writeJob->buffer, writeJob->bufferSize, NULL); @@ -1731,14 +1844,14 @@ FIO_compressLz4Frame(cRess_t* ress, srcFileName, LZ4F_getErrorName(headerSize)); writeJob->usedBufferSize = headerSize; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; } *readsize = inFileSize; LZ4F_freeCompressionContext(ctx); IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return outFileSize; } @@ -1753,8 +1866,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, int compressionLevel, U64* readsize) { cRess_t const ress = *ressPtr; - FILE* const srcFile = ress.srcFile; - io_job_t *writeJob = IoPool_acquireJob(ressPtr->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ressPtr->writeCtx); U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; @@ -1800,12 +1912,12 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, do { size_t stillToFlush; /* Fill input Buffer */ - size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); - ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; + size_t const inSize = ReadPool_readBuffer(ress.readCtx, ZSTD_CStreamInSize()); + ZSTD_inBuffer inBuff = { ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 }; DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); *readsize += inSize; - if ((inSize == 0) || (*readsize == fileSize)) + if ((ress.readCtx->srcBufferLoaded == 0) || (*readsize == fileSize)) directive = ZSTD_e_end; stillToFlush = 1; @@ -1816,6 +1928,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 }; size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); + ReadPool_consumeBytes(ress.readCtx, inBuff.pos - oldIPos); /* count stats */ inputPresented++; @@ -1827,7 +1940,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); if (outBuff.pos) { writeJob->usedBufferSize = outBuff.pos; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); compressedfilesize += outBuff.pos; } @@ -1959,16 +2072,13 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, } /* while ((inBuff.pos != inBuff.size) */ } while (directive != ZSTD_e_end); - if (ferror(srcFile)) { - EXM_THROW(26, "Read error : I/O error"); - } if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", (unsigned long long)*readsize, (unsigned long long)fileSize); } IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ressPtr->writePoolCtx); + WritePool_sparseWriteEnd(ressPtr->writeCtx); return compressedfilesize; } @@ -2090,8 +2200,8 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, stat_t statbuf; int transferMTime = 0; FILE *dstFile; - assert(ress.srcFile != NULL); - if (ress.writePoolCtx->file == NULL) { + assert(ress.readCtx->file != NULL); + if (ress.writeCtx->file == NULL) { int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) @@ -2105,7 +2215,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; /* could not open dstFileName */ - IoPool_setFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writeCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, * and the user presses Ctrl-C when asked if they wish to overwrite. @@ -2119,7 +2229,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, clearHandler(); DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); - if (WritePool_closeDstFile(ress.writePoolCtx)) { /* error closing file */ + if (WritePool_closeDstFile(ress.writeCtx)) { /* error closing file */ DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } @@ -2165,6 +2275,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, int compressionLevel) { int result; + FILE* srcFile; DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName); /* ensure src is not a directory */ @@ -2188,13 +2299,16 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, return 0; } - ress.srcFile = FIO_openSrcFile(prefs, srcFileName); - if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ + srcFile = FIO_openSrcFile(prefs, srcFileName); + if (srcFile == NULL) return 1; /* srcFile could not be opened */ + IoPool_setFile(ress.readCtx, srcFile); + ReadPool_startReading(ress.readCtx); result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); - fclose(ress.srcFile); - ress.srcFile = NULL; + fclose(srcFile); + // TODO: implement proper & safe close + IoPool_setFile(ress.readCtx, NULL); if ( prefs->removeSrcFile /* --rm */ && result == 0 /* success */ && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ @@ -2350,13 +2464,13 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, if (dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - IoPool_setFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writeCtx, dstFile); for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); if (!status) fCtx->nbFilesProcessed++; error |= status; } - if (WritePool_closeDstFile(ress.writePoolCtx)) + if (WritePool_closeDstFile(ress.writeCtx)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", strerror(errno), outFileName); } @@ -2415,11 +2529,9 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, * Decompression ***************************************************************************/ typedef struct { - void* srcBuffer; - size_t srcBufferSize; - size_t srcBufferLoaded; ZSTD_DStream* dctx; - io_pool_ctx_t *writePoolCtx; + io_pool_ctx_t *writeCtx; + io_pool_ctx_t *readCtx; } dRess_t; static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) @@ -2437,11 +2549,6 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); - ress.srcBufferSize = ZSTD_DStreamInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - if (!ress.srcBuffer) - EXM_THROW(61, "Allocation error : not enough memory"); - /* dictionary */ { void* dictBuffer; size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs); @@ -2449,7 +2556,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi free(dictBuffer); } - ress.writePoolCtx = WritePool_create(prefs, ZSTD_DStreamOutSize()); + ress.writeCtx = WritePool_create(prefs, ZSTD_DStreamOutSize()); + ress.readCtx = ReadPool_create(prefs, ZSTD_DStreamInSize()); return ress; } @@ -2457,47 +2565,31 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi static void FIO_freeDResources(dRess_t ress) { CHECK( ZSTD_freeDStream(ress.dctx) ); - free(ress.srcBuffer); - IoPool_free(ress.writePoolCtx); -} - -/* FIO_consumeDSrcBuffer: - * Consumes len bytes from srcBuffer's start and moves the remaining data and srcBufferLoaded accordingly. */ -static void FIO_consumeDSrcBuffer(dRess_t *ress, size_t len) { - assert(ress->srcBufferLoaded >= len); - ress->srcBufferLoaded -= len; - memmove(ress->srcBuffer, (char *)ress->srcBuffer + len, ress->srcBufferLoaded); + IoPool_free(ress.writeCtx); + IoPool_free(ress.readCtx); } /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ -static int FIO_passThrough(const FIO_prefs_t* const prefs, - FILE* foutput, FILE* finput, - void* buffer, size_t bufferSize, - size_t alreadyLoaded) +static int FIO_passThrough(dRess_t *ress) { - size_t const blockSize = MIN(64 KB, bufferSize); - size_t readFromInput; - unsigned storedSkips = 0; - - /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ - { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); - if (sizeCheck != alreadyLoaded) { - DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno)); - return 1; - } } - - do { - readFromInput = fread(buffer, 1, blockSize, finput); - storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, prefs, storedSkips); - } while (readFromInput == blockSize); - if (ferror(finput)) { - DISPLAYLEVEL(1, "Pass-through read error : %s\n", strerror(errno)); - return 1; - } - assert(feof(finput)); - - FIO_fwriteSparseEnd(prefs, foutput, storedSkips); + size_t const blockSize = MIN(MIN(64 KB, ZSTD_DStreamInSize()), ZSTD_DStreamOutSize()); + io_job_t *writeJob = IoPool_acquireJob(ress->writeCtx); + ReadPool_readBuffer(ress->readCtx, blockSize); + + while(ress->readCtx->srcBufferLoaded) { + size_t writeSize; + ReadPool_readBuffer(ress->readCtx, blockSize); + writeSize = MIN(blockSize, ress->readCtx->srcBufferLoaded); + assert(writeSize <= writeJob->bufferSize); + memcpy(writeJob->buffer, ress->readCtx->srcBuffer, writeSize); + writeJob->usedBufferSize = writeSize; + IoPool_enqueueAndReacquireWriteJob(&writeJob); + ReadPool_consumeBytes(ress->readCtx, writeSize); + } + assert(ress->readCtx->reachedEof); + IoPool_releaseIoJob(writeJob); + WritePool_sparseWriteEnd(ress->writeCtx); return 0; } @@ -2515,7 +2607,7 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, return; /* Try to decode the frame header */ - err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); + err = ZSTD_getFrameHeader(&header, ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded); if (err == 0) { unsigned long long const windowSize = header.windowSize; unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); @@ -2538,13 +2630,13 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, */ #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) static unsigned long long -FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, +FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, const FIO_prefs_t* const prefs, const char* srcFileName, U64 alreadyDecoded) /* for multi-frames streams */ { U64 frameSize = 0; - io_job_t *writeJob = IoPool_acquireJob(ress->writePoolCtx); + io_job_t *writeJob = IoPool_acquireJob(ress->writeCtx); /* display last 20 characters only */ { size_t const srcFileLength = strlen(srcFileName); @@ -2554,16 +2646,12 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only); /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ - { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; - if (ress->srcBufferLoaded < toDecode) { - size_t const toRead = toDecode - ress->srcBufferLoaded; - void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; - ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); - } } + if (ress->readCtx->srcBufferLoaded < ZSTD_FRAMEHEADERSIZE_MAX) + ReadPool_readBuffer(ress->readCtx, ZSTD_FRAMEHEADERSIZE_MAX); /* Main decompression Loop */ while (1) { - ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; + ZSTD_inBuffer inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 }; ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 }; size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2; @@ -2572,12 +2660,13 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", srcFileName, ZSTD_getErrorName(readSizeHint)); FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); + IoPool_releaseIoJob(writeJob); return FIO_ERROR_FRAME_DECODING; } /* Write block */ writeJob->usedBufferSize = outBuff.pos; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); frameSize += outBuff.pos; if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); @@ -2594,26 +2683,24 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, srcFileName, hrs.precision, hrs.value, hrs.suffix); } - FIO_consumeDSrcBuffer(ress, inBuff.pos); + ReadPool_consumeBytes(ress->readCtx, inBuff.pos); if (readSizeHint == 0) break; /* end of frame */ /* Fill input buffer */ - { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ - if (ress->srcBufferLoaded < toDecode) { - size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ - void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; - size_t const readSize = fread(startPosition, 1, toRead, finput); + { size_t const toDecode = MIN(readSizeHint, ZSTD_DStreamInSize()); /* support large skippable frames */ + if (ress->readCtx->srcBufferLoaded < toDecode) { + size_t const readSize = ReadPool_readBuffer(ress->readCtx, toDecode); if (readSize==0) { DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", srcFileName); + IoPool_releaseIoJob(writeJob); return FIO_ERROR_FRAME_DECODING; } - ress->srcBufferLoaded += readSize; } } } IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return frameSize; } @@ -2621,7 +2708,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, #ifdef ZSTD_GZDECOMPRESS static unsigned long long -FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) +FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName) { unsigned long long outFileSize = 0; z_stream strm; @@ -2638,19 +2725,19 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return FIO_ERROR_FRAME_DECODING; - writeJob = IoPool_acquireJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writeCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; - strm.avail_in = (uInt)ress->srcBufferLoaded; - strm.next_in = (z_const unsigned char*)ress->srcBuffer; + strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; + strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; for ( ; ; ) { int ret; if (strm.avail_in == 0) { - ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); - if (ress->srcBufferLoaded == 0) flush = Z_FINISH; - strm.next_in = (z_const unsigned char*)ress->srcBuffer; - strm.avail_in = (uInt)ress->srcBufferLoaded; + ReadPool_consumeAndReadAll(ress->readCtx); + if (ress->readCtx->srcBufferLoaded == 0) flush = Z_FINISH; + strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; + strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; } ret = inflate(&strm, flush); if (ret == Z_BUF_ERROR) { @@ -2664,7 +2751,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -2673,7 +2760,7 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) if (ret == Z_STREAM_END) break; } - FIO_consumeDSrcBuffer(ress, ress->srcBufferLoaded - strm.avail_in); + ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in); if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ && (decodingError==0) ) { @@ -2681,15 +2768,14 @@ FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName) decodingError = 1; } IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif - #ifdef ZSTD_LZMADECOMPRESS static unsigned long long -FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, +FIO_decompressLzmaFrame(dRess_t* ress, const char* srcFileName, int plain_lzma) { unsigned long long outFileSize = 0; @@ -2714,19 +2800,19 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, return FIO_ERROR_FRAME_DECODING; } - writeJob = IoPool_acquireJob(ress->writePoolCtx); + writeJob = IoPool_acquireJob(ress->writeCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; - strm.next_in = (BYTE const*)ress->srcBuffer; - strm.avail_in = ress->srcBufferLoaded; + strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; + strm.avail_in = ress->readCtx->srcBufferLoaded; for ( ; ; ) { lzma_ret ret; if (strm.avail_in == 0) { - ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); - if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; - strm.next_in = (BYTE const*)ress->srcBuffer; - strm.avail_in = ress->srcBufferLoaded; + ReadPool_consumeAndReadAll(ress->readCtx); + if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH; + strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; + strm.avail_in = ress->readCtx->srcBufferLoaded; } ret = lzma_code(&strm, action); @@ -2742,7 +2828,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -2750,68 +2836,62 @@ FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, if (ret == LZMA_STREAM_END) break; } - FIO_consumeDSrcBuffer(ress, ress->srcBufferLoaded - strm.avail_in); + ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in); lzma_end(&strm); IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif #ifdef ZSTD_LZ4DECOMPRESS static unsigned long long -FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, - const char* srcFileName) +FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName) { unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad = 4; LZ4F_decompressionContext_t dCtx; LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); int decodingError = 0; - io_job_t *writeJob = IoPool_acquireJob(ress->writePoolCtx); + io_job_t *writeJob = NULL; if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); return FIO_ERROR_FRAME_DECODING; } + writeJob = IoPool_acquireJob(ress->writeCtx); + /* Main Loop */ for (;nextToLoad;) { - size_t readSize; size_t pos = 0; size_t decodedBytes = writeJob->bufferSize; int fullBufferDecoded = 0; /* Read input */ - nextToLoad = MIN(nextToLoad, ress->srcBufferSize-ress->srcBufferLoaded); - readSize = fread((char *)ress->srcBuffer + ress->srcBufferLoaded, 1, nextToLoad, srcFile); - if(!readSize && ferror(srcFile)) { - DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); - decodingError=1; - break; - } - if(!readSize && !ress->srcBufferLoaded) break; /* reached end of file */ - ress->srcBufferLoaded += readSize; + ReadPool_readBuffer(ress->readCtx, nextToLoad); + if(!ress->readCtx->srcBufferLoaded) break; /* reached end of file */ - while ((pos < ress->srcBufferLoaded) || fullBufferDecoded) { /* still to read, or still to flush */ + while ((pos < ress->readCtx->srcBufferLoaded) || fullBufferDecoded) { /* still to read, or still to flush */ /* Decode Input (at least partially) */ - size_t remaining = ress->srcBufferLoaded - pos; + size_t remaining = ress->readCtx->srcBufferLoaded - pos; decodedBytes = writeJob->bufferSize; - nextToLoad = LZ4F_decompress(dCtx, writeJob->buffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); + nextToLoad = LZ4F_decompress(dCtx, writeJob->buffer, &decodedBytes, (char*)(ress->readCtx->srcBuffer)+pos, + &remaining, NULL); if (LZ4F_isError(nextToLoad)) { DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", srcFileName, LZ4F_getErrorName(nextToLoad)); decodingError = 1; nextToLoad = 0; break; } pos += remaining; - assert(pos <= ress->srcBufferLoaded); + assert(pos <= ress->readCtx->srcBufferLoaded); fullBufferDecoded = decodedBytes == writeJob->bufferSize; /* Write Block */ if (decodedBytes) { UTIL_HumanReadableSize_t hrs; writeJob->usedBufferSize = decodedBytes; - IoPool_queueAndReacquireWriteJob(&writeJob); + IoPool_enqueueAndReacquireWriteJob(&writeJob); filesize += decodedBytes; hrs = UTIL_makeHumanReadableSize(filesize); DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); @@ -2819,7 +2899,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, if (!nextToLoad) break; } - FIO_consumeDSrcBuffer(ress, pos); + ReadPool_consumeBytes(ress->readCtx, pos); } if (nextToLoad!=0) { DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); @@ -2828,7 +2908,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, LZ4F_freeDecompressionContext(dCtx); IoPool_releaseIoJob(writeJob); - WritePool_sparseWriteEnd(ress->writePoolCtx); + WritePool_sparseWriteEnd(ress->writeCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; } @@ -2843,23 +2923,20 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, * 1 : error */ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, - dRess_t ress, FILE* srcFile, - const FIO_prefs_t* const prefs, + dRess_t ress, const FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName) { unsigned readSomething = 0; unsigned long long filesize = 0; - assert(srcFile != NULL); /* for each frame */ for ( ; ; ) { /* check magic number -> version */ size_t const toRead = 4; - const BYTE* const buf = (const BYTE*)ress.srcBuffer; - if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ - ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, - (size_t)1, toRead - ress.srcBufferLoaded, srcFile); - if (ress.srcBufferLoaded==0) { + const BYTE* buf; + ReadPool_readBuffer(ress.readCtx, toRead); + buf = (const BYTE*)ress.readCtx->srcBuffer; + if (ress.readCtx->srcBufferLoaded==0) { if (readSomething==0) { /* srcFile is empty (which is invalid) */ DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); return 1; @@ -2867,17 +2944,17 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, break; /* no more input */ } readSomething = 1; /* there is at least 1 byte in srcFile */ - if (ress.srcBufferLoaded < toRead) { + if (ress.readCtx->srcBufferLoaded < toRead) { DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); return 1; } - if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { - unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize); + if (ZSTD_isFrame(buf, ress.readCtx->srcBufferLoaded)) { + unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, prefs, srcFileName, filesize); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ #ifdef ZSTD_GZDECOMPRESS - unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); + unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFileName); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2887,7 +2964,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ #ifdef ZSTD_LZMADECOMPRESS - unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); + unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFileName, buf[0] != 0xFD); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2896,7 +2973,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, #endif } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { #ifdef ZSTD_LZ4DECOMPRESS - unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); + unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFileName); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2904,10 +2981,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, return 1; #endif } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ - return FIO_passThrough(prefs, - ress.writePoolCtx->file, srcFile, - ress.srcBuffer, ress.srcBufferSize, - ress.srcBufferLoaded); + return FIO_passThrough(&ress); } else { DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); return 1; @@ -2928,14 +3002,14 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, /** FIO_decompressDstFile() : open `dstFileName`, - or pass-through if ress.writePoolCtx->file is already != 0, + or pass-through if ress.writeCtx->file is already != 0, then start decompression process (FIO_decompressFrames()). @return : 0 : OK 1 : operation aborted */ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, - dRess_t ress, FILE* srcFile, + dRess_t ress, const char* dstFileName, const char* srcFileName) { int result; @@ -2943,7 +3017,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, int releaseDstFile = 0; int transferMTime = 0; - if ((ress.writePoolCtx->file == NULL) && (prefs->testMode == 0)) { + if ((ress.writeCtx->file == NULL) && (prefs->testMode == 0)) { FILE *dstFile; int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ @@ -2958,7 +3032,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; - IoPool_setFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writeCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -2967,11 +3041,11 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, addHandler(dstFileName); } - result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); + result = FIO_decompressFrames(fCtx, ress, prefs, dstFileName, srcFileName); if (releaseDstFile) { clearHandler(); - if (WritePool_closeDstFile(ress.writePoolCtx)) { + if (WritePool_closeDstFile(ress.writeCtx)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result = 1; } @@ -3008,9 +3082,12 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile==NULL) return 1; - ress.srcBufferLoaded = 0; + IoPool_setFile(ress.readCtx, srcFile); + ReadPool_startReading(ress.readCtx); + + result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName); - result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName); + IoPool_setFile(ress.readCtx, NULL); /* Close file */ if (fclose(srcFile)) { @@ -3187,14 +3264,14 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, if (!prefs->testMode) { FILE* dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); - IoPool_setFile(ress.writePoolCtx, dstFile); + IoPool_setFile(ress.writeCtx, dstFile); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); if (!status) fCtx->nbFilesProcessed++; error |= status; } - if ((!prefs->testMode) && (WritePool_closeDstFile(ress.writePoolCtx))) + if ((!prefs->testMode) && (WritePool_closeDstFile(ress.writeCtx))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { diff --git a/tests/playTests.sh b/tests/playTests.sh index 78d8e742aa3..7048c7c4d58 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -185,7 +185,7 @@ fi println "\n===> simple tests " -datagen > tmp +datagen -g500K > tmp zstd -h zstd -H zstd -V From 35dc66c555e2d7876054b927f99b813cc5259d3c Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 14:44:16 -0800 Subject: [PATCH 16/25] Async IO compression: Moved some `fileio.c` utilities into `fileio_utils.c`. --- build/VS2008/zstd/zstd.vcproj | 4 + build/VS2010/zstd/zstd.vcxproj | 1 + build/cmake/programs/CMakeLists.txt | 4 +- build/meson/programs/meson.build | 2 + contrib/VS2005/zstd/zstd.vcproj | 4 + programs/Makefile | 8 +- programs/fileio.c | 664 +--------------------------- programs/fileio.h | 8 +- programs/fileio_types.h | 63 +++ programs/fileio_utils.c | 467 +++++++++++++++++++ programs/fileio_utils.h | 219 +++++++++ 11 files changed, 768 insertions(+), 676 deletions(-) create mode 100644 programs/fileio_types.h create mode 100644 programs/fileio_utils.c create mode 100644 programs/fileio_utils.h diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index c7eec577db3..829f7e6e305 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -384,6 +384,10 @@ RelativePath="..\..\..\programs\fileio.c" > + + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 46e22f42e9b..089131669f9 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -62,6 +62,7 @@ + diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 490030783d3..9e8eb35f387 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -32,7 +32,7 @@ if (MSVC) set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc) endif () -add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources}) +add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_utils.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources}) target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET}) if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") target_link_libraries(zstd rt) @@ -75,7 +75,7 @@ if (UNIX) ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}") - add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c) + add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_utils.c) target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET}) set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE") endif () diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index 0ae93fc107c..f599b977d71 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -14,6 +14,7 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/util.c'), join_paths(zstd_rootdir, 'programs/timefn.c'), join_paths(zstd_rootdir, 'programs/fileio.c'), + join_paths(zstd_rootdir, 'programs/fileio_utils.c'), join_paths(zstd_rootdir, 'programs/benchfn.c'), join_paths(zstd_rootdir, 'programs/benchzstd.c'), join_paths(zstd_rootdir, 'programs/datagen.c'), @@ -80,6 +81,7 @@ zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/timefn.c'), join_paths(zstd_rootdir, 'programs/util.c'), join_paths(zstd_rootdir, 'programs/fileio.c'), + join_paths(zstd_rootdir, 'programs/fileio_utils.c'), join_paths(zstd_rootdir, 'lib/common/pool.c'), join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), join_paths(zstd_rootdir, 'lib/common/error_private.c')] diff --git a/contrib/VS2005/zstd/zstd.vcproj b/contrib/VS2005/zstd/zstd.vcproj index 78645d18a36..f5fd6dd88ed 100644 --- a/contrib/VS2005/zstd/zstd.vcproj +++ b/contrib/VS2005/zstd/zstd.vcproj @@ -363,6 +363,10 @@ RelativePath="..\..\..\programs\fileio.c" > + + diff --git a/programs/Makefile b/programs/Makefile index f77e1b7f10f..b6b4a418fde 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -243,17 +243,17 @@ zstd-pgo : ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format. zstd-small: CFLAGS = -Os -s -zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c +zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_utils.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c +zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_utils.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c +zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_utils.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) ## zstd-dictBuilder: executable supporting dictionary creation and compression (only) -zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c +zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_utils.c dibio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT) zstdmt: zstd diff --git a/programs/fileio.c b/programs/fileio.c index 4ee6dc85b44..0970771e4dc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -34,16 +34,14 @@ #include /* INT_MAX */ #include #include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ -#include "../lib/common/pool.h" -#include "../lib/common/threading.h" #if defined (_MSC_VER) # include # include #endif -#include "../lib/common/mem.h" /* U32, U64 */ #include "fileio.h" +#include "fileio_utils.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "../lib/zstd.h" @@ -68,77 +66,6 @@ #endif -/*-************************************* -* Constants -***************************************/ -#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ -#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ - -#define FNSPACE 30 - -/* Default file permissions 0666 (modulated by umask) */ -#if !defined(_WIN32) -/* These macros aren't defined on windows. */ -#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) -#else -#define DEFAULT_FILE_PERMISSIONS (0666) -#endif - -/*-************************************* -* Macros -***************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) -#undef MAX -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -struct FIO_display_prefs_s { - int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ - FIO_progressSetting_e progressSetting; -}; - -static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; - -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) -#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } -#define DISPLAYUPDATE(l, ...) { \ - if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ - if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ - DELAY_NEXT_UPDATE(); \ - DISPLAY(__VA_ARGS__); \ - if (g_display_prefs.displayLevel>=4) fflush(stderr); \ - } } } - -#undef MIN /* in case it would be already defined */ -#define MIN(a,b) ((a) < (b) ? (a) : (b)) - - -#define EXM_THROW(error, ...) \ -{ \ - DISPLAYLEVEL(1, "zstd: "); \ - DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - exit(error); \ -} - -#define CHECK_V(v, f) \ - v = f; \ - if (ZSTD_isError(v)) { \ - DISPLAYLEVEL(5, "%s \n", #f); \ - EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ - } -#define CHECK(f) { size_t err; CHECK_V(err, f); } - /*-************************************ * Signal (Ctrl-C trapping) @@ -250,95 +177,6 @@ void FIO_addAbortHandler() #endif } - -/*-************************************************************ -* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW -***************************************************************/ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -# define LONG_SEEK _fseeki64 -# define LONG_TELL _ftelli64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define LONG_SEEK fseeko -# define LONG_TELL ftello -#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) -# define LONG_SEEK fseeko64 -# define LONG_TELL ftello64 -#elif defined(_WIN32) && !defined(__DJGPP__) -# include - static int LONG_SEEK(FILE* file, __int64 offset, int origin) { - LARGE_INTEGER off; - DWORD method; - off.QuadPart = offset; - if (origin == SEEK_END) - method = FILE_END; - else if (origin == SEEK_CUR) - method = FILE_CURRENT; - else - method = FILE_BEGIN; - - if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) - return 0; - else - return -1; - } - static __int64 LONG_TELL(FILE* file) { - LARGE_INTEGER off, newOff; - off.QuadPart = 0; - newOff.QuadPart = 0; - SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); - return newOff.QuadPart; - } -#else -# define LONG_SEEK fseek -# define LONG_TELL ftell -#endif - - -/*-************************************* -* Parameters: FIO_prefs_t -***************************************/ - -/* typedef'd to FIO_prefs_t within fileio.h */ -struct FIO_prefs_s { - - /* Algorithm preferences */ - FIO_compressionType_t compressionType; - U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ - int dictIDFlag; - int checksumFlag; - int blockSize; - int overlapLog; - U32 adaptiveMode; - U32 useRowMatchFinder; - int rsyncable; - int minAdaptLevel; - int maxAdaptLevel; - int ldmFlag; - int ldmHashLog; - int ldmMinMatch; - int ldmBucketSizeLog; - int ldmHashRateLog; - size_t streamSrcSize; - size_t targetCBlockSize; - int srcSizeHint; - int testMode; - ZSTD_paramSwitch_e literalCompressionMode; - - /* IO preferences */ - U32 removeSrcFile; - U32 overwrite; - U32 asyncIO; - - /* Computation resources preferences */ - unsigned memLimit; - int nbWorkers; - - int excludeCompressedFiles; - int patchFromMode; - int contentSize; - int allowBlockDevices; -}; - /*-************************************* * Parameters: FIO_ctx_t ***************************************/ @@ -732,7 +570,6 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } - setvbuf(f, NULL, _IOFBF, 1024*1024); return f; } } @@ -941,505 +778,6 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* return error; } -/** FIO_fwriteSparse() : -* @return : storedSkips, -* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ -static unsigned -FIO_fwriteSparse(FILE* file, - const void* buffer, size_t bufferSize, - const FIO_prefs_t* const prefs, - unsigned storedSkips) -{ - const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ - size_t bufferSizeT = bufferSize / sizeof(size_t); - const size_t* const bufferTEnd = bufferT + bufferSizeT; - const size_t* ptrT = bufferT; - static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ - - if (prefs->testMode) return 0; /* do not output anything in test mode */ - - if (!prefs->sparseFileSupport) { /* normal write */ - size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); - if (sizeCheck != bufferSize) - EXM_THROW(70, "Write error : cannot write decoded block : %s", - strerror(errno)); - return 0; - } - - /* avoid int overflow */ - if (storedSkips > 1 GB) { - if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) - EXM_THROW(91, "1 GB skip error (sparse file support)"); - storedSkips -= 1 GB; - } - - while (ptrT < bufferTEnd) { - size_t nb0T; - - /* adjust last segment if < 32 KB */ - size_t seg0SizeT = segmentSizeT; - if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; - bufferSizeT -= seg0SizeT; - - /* count leading zeroes */ - for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeof(size_t)); - - if (nb0T != seg0SizeT) { /* not all 0s */ - size_t const nbNon0ST = seg0SizeT - nb0T; - /* skip leading zeros */ - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - /* write the rest */ - if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) - EXM_THROW(93, "Write error : cannot write decoded block : %s", - strerror(errno)); - } - ptrT += seg0SizeT; - } - - { static size_t const maskT = sizeof(size_t)-1; - if (bufferSize & maskT) { - /* size not multiple of sizeof(size_t) : implies end of block */ - const char* const restStart = (const char*)bufferTEnd; - const char* restPtr = restStart; - const char* const restEnd = (const char*)buffer + bufferSize; - assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); - for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) { - /* not all remaining bytes are 0 */ - size_t const restSize = (size_t)(restEnd - restPtr); - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - if (fwrite(restPtr, 1, restSize, file) != restSize) - EXM_THROW(95, "Write error : cannot write end of decoded block : %s", - strerror(errno)); - storedSkips = 0; - } } } - - return storedSkips; -} - -static void -FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) -{ - if (prefs->testMode) assert(storedSkips == 0); - if (storedSkips>0) { - assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ - (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ - if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) - EXM_THROW(69, "Final skip error (sparse file support)"); - /* last zero must be explicitly written, - * so that skipped ones get implicitly translated as zero by FS */ - { const char lastZeroByte[1] = { 0 }; - if (fwrite(lastZeroByte, 1, 1, file) != 1) - EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); - } } -} - -/* ********************************************************************** - * AsyncIO functionality - ************************************************************************/ -#define MAX_IO_JOBS (10) - -typedef struct { - /* These struct fields should be set only on creation and not changed afterwards */ - POOL_ctx* threadPool; - int totalIoJobs; - FIO_prefs_t* prefs; - POOL_function poolFunction; - - /* Controls the file we currently write to, make changes only by using provided utility functions */ - FILE* file; - unsigned storedSkips; // only used for write io pool - int reachedEof; - U64 nextReadOffset; - U64 waitingOnOffset; - - /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should - * only be mutated after locking the mutex */ - ZSTD_pthread_mutex_t ioJobsMutex; - void* availableJobs[MAX_IO_JOBS]; - int availableJobsCount; - - void* completedJobs[MAX_IO_JOBS]; - int completedJobsCount; - ZSTD_pthread_cond_t jobCompletedCond; - - U8 *srcBufferBase; - size_t srcBufferBaseSize; - U8 *srcBuffer; - size_t srcBufferLoaded; -} io_pool_ctx_t; - -typedef struct { - /* These fields are automatically set and shouldn't be changed by non WritePool code. */ - io_pool_ctx_t *ctx; - FILE* file; - void *buffer; - size_t bufferSize; - - /* This field should be changed before a job is queued for execution and should contain the number - * of bytes to write from the buffer. */ - size_t usedBufferSize; - U64 offset; -} io_job_t; - - -static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { - void *buffer; - io_job_t *job; - job = (io_job_t*) malloc(sizeof(io_job_t)); - buffer = malloc(bufferSize); - if(!job || !buffer) - EXM_THROW(101, "Allocation error : not enough memory"); - job->buffer = buffer; - job->bufferSize = bufferSize; - job->usedBufferSize = 0; - job->file = NULL; - job->ctx = ctx; - job->offset = 0; - return job; -} - - -/* IO_createThreadPool: - * Creates a thread pool and a mutex for threaded IO pool. - * Displays warning if asyncio is requested but MT isn't available. */ -static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { - ctx->threadPool = NULL; - if(prefs->asyncIO) { - if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) - EXM_THROW(102, "Failed creating write availableJobs mutex"); - if(ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL)) - EXM_THROW(103, "Failed creating write jobCompletedCond mutex"); - /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to - * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ - assert(MAX_IO_JOBS >= 2); - ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2); - if (!ctx->threadPool) - EXM_THROW(104, "Failed creating writer thread pool"); - } -} - -/* IoPool_create: - * Allocates and sets and a new write pool including its included availableJobs. */ -static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function poolFunction, - size_t bufferSize) { - io_pool_ctx_t *ctx; - int i; - ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); - if(!ctx) - EXM_THROW(100, "Allocation error : not enough memory"); - IO_createThreadPool(ctx, prefs); - ctx->prefs = prefs; - ctx->poolFunction = poolFunction; - ctx->totalIoJobs = ctx->threadPool ? MAX_IO_JOBS : 1; - ctx->availableJobsCount = ctx->totalIoJobs; - for(i=0; i < ctx->availableJobsCount; i++) { - ctx->availableJobs[i] = FIO_createIoJob(ctx, bufferSize); - } - ctx->completedJobsCount = 0; - ctx->storedSkips = 0; - ctx->reachedEof = 0; - ctx->nextReadOffset = 0; - ctx->waitingOnOffset = 0; - ctx->file = NULL; - - ctx->srcBufferBaseSize = 2 * bufferSize; - ctx->srcBufferBase = (U8*) malloc(ctx->srcBufferBaseSize); - ctx->srcBuffer = ctx->srcBufferBase; - ctx->srcBufferLoaded = 0; - - return ctx; -} - - -/* IoPool_releaseIoJob: - * Releases an acquired job back to the pool. Doesn't execute the job. */ -static void IoPool_releaseIoJob(io_job_t *job) { - io_pool_ctx_t *ctx = job->ctx; - if(ctx->threadPool) { - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - assert(ctx->availableJobsCount < MAX_IO_JOBS); - ctx->availableJobs[ctx->availableJobsCount++] = job; - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); - } else { - assert(ctx->availableJobsCount == 0); - ctx->availableJobsCount++; - } -} - -static void IoPool_releaseAllCompletedJobs(io_pool_ctx_t* ctx) { - int i=0; - for(i=0; icompletedJobsCount; i++) { - io_job_t* job = (io_job_t*) ctx->completedJobs[i]; - IoPool_releaseIoJob(job); - } - ctx->completedJobsCount = 0; -} - - -/* IoPool_free: - * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ -static void IoPool_free(io_pool_ctx_t* ctx) { - int i=0; - IoPool_releaseAllCompletedJobs(ctx); - if(ctx->threadPool) { - /* Make sure we finish all tasks and then free the resources */ - POOL_joinJobs(ctx->threadPool); - /* Make sure we are not leaking availableJobs */ - assert(ctx->availableJobsCount == ctx->totalIoJobs); - POOL_free(ctx->threadPool); - ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); - ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); - } - assert(ctx->file == NULL); - assert(ctx->storedSkips==0); - for(i=0; iavailableJobsCount; i++) { - io_job_t* job = (io_job_t*) ctx->availableJobs[i]; - free(job->buffer); - free(job); - } - free(ctx->srcBuffer); - free(ctx); -} - -/* IoPool_acquireJob: - * Returns an available write job to be used for a future write. */ -static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { - io_job_t *job; - assert(ctx->file != NULL || ctx->prefs->testMode); - if(ctx->threadPool) { - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - assert(ctx->availableJobsCount > 0); - job = (io_job_t*) ctx->availableJobs[--ctx->availableJobsCount]; - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); - } else { - assert(ctx->availableJobsCount == 1); - ctx->availableJobsCount--; - job = (io_job_t*)ctx->availableJobs[0]; - } - job->usedBufferSize = 0; - job->file = ctx->file; - job->offset = 0; - return job; -} - -/* IoPool_addJobToCompleted */ -static void IoPool_addJobToCompleted(io_job_t *job) { - io_pool_ctx_t *ctx = job->ctx; - if(ctx->threadPool) - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - assert(ctx->completedJobsCount < MAX_IO_JOBS); - ctx->completedJobs[ctx->completedJobsCount++] = job; - if(ctx->threadPool) { - ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); - } -} - -/* assuming ioJobsMutex is locked */ -static io_job_t* IoPool_findWaitingJob(io_pool_ctx_t *ctx) { - io_job_t *job = NULL; - int i = 0; - for (i=0; icompletedJobsCount; i++) { - job = (io_job_t *) ctx->completedJobs[i]; - if (job->offset == ctx->waitingOnOffset) { - ctx->completedJobs[i] = ctx->completedJobs[--ctx->completedJobsCount]; - return job; - } - } - return NULL; -} - -/* IoPool_getNextCompletedJob */ -static io_job_t* IoPool_getNextCompletedJob(io_pool_ctx_t *ctx) { - io_job_t *job = NULL; - if(ctx->threadPool) - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); - - job = IoPool_findWaitingJob(ctx); - - while (!job && (ctx->availableJobsCount + ctx->completedJobsCount < ctx->totalIoJobs)) { - assert(ctx->threadPool != NULL); - ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->ioJobsMutex); - job = IoPool_findWaitingJob(ctx); - } - - if(job) { - assert(job->offset == ctx->waitingOnOffset); - ctx->waitingOnOffset += job->usedBufferSize; - } - - if(ctx->threadPool) - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); - return job; -} - -/* IoPool_enqueueJob: - * Queues a write job for execution. - * Make sure to set `usedBufferSize` to the wanted length before call. - * The queued job shouldn't be used directly after queueing it. */ -static void IoPool_enqueueJob(io_job_t *job) { - io_pool_ctx_t* ctx = job->ctx; - if(ctx->threadPool) - POOL_add(ctx->threadPool, ctx->poolFunction, job); - else - ctx->poolFunction(job); -} - -/* IoPool_enqueueAndReacquireWriteJob: - * Queues a write job for execution and acquires a new one. - * After execution `job`'s pointed value would change to the newly acquired job. - * Make sure to set `usedBufferSize` to the wanted length before call. - * The queued job shouldn't be used directly after queueing it. */ -static void IoPool_enqueueAndReacquireWriteJob(io_job_t **job) { - IoPool_enqueueJob(*job); - *job = IoPool_acquireJob((*job)->ctx); -} - -/* WritePool_sparseWriteEnd: - * Ends sparse writes to the current file. - * Blocks on completion of all current write jobs before executing. */ -static void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx) { - assert(ctx != NULL); - if(ctx->threadPool) - POOL_joinJobs(ctx->threadPool); - FIO_fwriteSparseEnd(ctx->prefs, ctx->file, ctx->storedSkips); - ctx->storedSkips = 0; -} - -/* IoPool_setFile: - * Sets the destination file for future files in the pool. - * Requires completion of all queues write jobs and release of all otherwise acquired jobs. - * Also requires ending of sparse write if a previous file was used in sparse mode. */ -static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { - assert(ctx!=NULL); - /* We can change the dst file only if we have finished writing */ - IoPool_releaseAllCompletedJobs(ctx); - if(ctx->threadPool) - POOL_joinJobs(ctx->threadPool); - assert(ctx->storedSkips == 0); - assert(ctx->availableJobsCount == ctx->totalIoJobs); - ctx->file = file; - ctx->nextReadOffset = 0; - ctx->waitingOnOffset = 0; - ctx->srcBuffer = ctx->srcBufferBase; - ctx->reachedEof = 0; -} - -/* WritePool_closeDstFile: - * Ends sparse write and closes the writePool's current file and sets the file to NULL. - * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -static int WritePool_closeDstFile(io_pool_ctx_t *ctx) { - FILE *dstFile = ctx->file; - assert(dstFile!=NULL || ctx->prefs->testMode!=0); - WritePool_sparseWriteEnd(ctx); - IoPool_setFile(ctx, NULL); - return fclose(dstFile); -} - -/* WritePool_executeWriteJob: - * Executes a write job synchronously. Can be used as a function for a thread pool. */ -static void WritePool_executeWriteJob(void* opaque){ - io_job_t* job = (io_job_t*) opaque; - io_pool_ctx_t* ctx = job->ctx; - ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); - IoPool_releaseIoJob(job); -} - -/* WritePool_create: - * Allocates and sets and a new write pool including its included jobs. */ -static io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) { - return IoPool_create(prefs, WritePool_executeWriteJob, bufferSize); -} - -/* ReadPool_executeReadJob: - * Executes a read job synchronously. Can be used as a function for a thread pool. */ -static void ReadPool_executeReadJob(void* opaque){ - io_job_t* job = (io_job_t*) opaque; - io_pool_ctx_t* ctx = job->ctx; - if(ctx->reachedEof) { - job->usedBufferSize = 0; - IoPool_addJobToCompleted(job); - return; - } - job->usedBufferSize = fread(job->buffer, 1, job->bufferSize, job->file); - if(job->usedBufferSize < job->bufferSize) { - if(ferror(job->file)) { - EXM_THROW(37, "Read error"); - } else if(feof(job->file)) { - ctx->reachedEof = 1; - } else - EXM_THROW(37, "Unexpected short read"); - } - IoPool_addJobToCompleted(job); -} - -/* ReadPool_create: - * Allocates and sets and a new write pool including its included jobs. */ -static io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { - return IoPool_create(prefs, ReadPool_executeReadJob, bufferSize); -} - -static void ReadPool_consumeBytes(io_pool_ctx_t *ctx, size_t n) { - assert(n <= ctx->srcBufferLoaded); - assert(ctx->srcBuffer + n <= ctx->srcBufferBase + ctx->srcBufferBaseSize); - ctx->srcBufferLoaded -= n; - ctx->srcBuffer += n; -} - -static void ReadPool_enqueueRead(io_pool_ctx_t *ctx) { - io_job_t *job = IoPool_acquireJob(ctx); - job->offset = ctx->nextReadOffset; - ctx->nextReadOffset += job->bufferSize; - IoPool_enqueueJob(job); -} - - -static size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n) { - io_job_t *job; - size_t srcBufferOffsetFromBase; - size_t srcBufferRemainingSpace; - size_t bytesRead = 0; - while (ctx->srcBufferLoaded < n) { - job = IoPool_getNextCompletedJob(ctx); - if(job == NULL) - break; - srcBufferOffsetFromBase = ctx->srcBuffer - ctx->srcBufferBase; - srcBufferRemainingSpace = ctx->srcBufferBaseSize - (srcBufferOffsetFromBase + ctx->srcBufferLoaded); - if(job->usedBufferSize > srcBufferRemainingSpace) { - memmove(ctx->srcBufferBase, ctx->srcBuffer, ctx->srcBufferLoaded); - ctx->srcBuffer = ctx->srcBufferBase; - } - memcpy(ctx->srcBuffer + ctx->srcBufferLoaded, job->buffer, job->usedBufferSize); - bytesRead += job->usedBufferSize; - ctx->srcBufferLoaded += job->usedBufferSize; - if(job->usedBufferSize < job->bufferSize) { - IoPool_releaseIoJob(job); - break; - } - IoPool_releaseIoJob(job); - ReadPool_enqueueRead(ctx); - } - return bytesRead; -} - -static size_t ReadPool_consumeAndReadAll(io_pool_ctx_t *ctx) { - ReadPool_consumeBytes(ctx, ctx->srcBufferLoaded); - return ReadPool_readBuffer(ctx, ZSTD_DStreamInSize()); -} - -static void ReadPool_startReading(io_pool_ctx_t *ctx) { - int i; - for (i = 0; i < ctx->availableJobsCount; i++) { - ReadPool_enqueueRead(ctx); - } -} - #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** diff --git a/programs/fileio.h b/programs/fileio.h index 398937a64e8..9d6ebb1accb 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -13,6 +13,7 @@ #define FILEIO_H_23981798732 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "fileio_types.h" #include "../lib/zstd.h" /* ZSTD_* */ #if defined (__cplusplus) @@ -53,10 +54,6 @@ extern "C" { /*-************************************* * Types ***************************************/ -typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t; - -typedef struct FIO_prefs_s FIO_prefs_t; - FIO_prefs_t* FIO_createPreferences(void); void FIO_freePreferences(FIO_prefs_t* const prefs); @@ -66,9 +63,6 @@ typedef struct FIO_ctx_s FIO_ctx_t; FIO_ctx_t* FIO_createContext(void); void FIO_freeContext(FIO_ctx_t* const fCtx); -typedef struct FIO_display_prefs_s FIO_display_prefs_t; - -typedef enum { FIO_ps_auto, FIO_ps_never, FIO_ps_always } FIO_progressSetting_e; /*-************************************* * Parameters diff --git a/programs/fileio_types.h b/programs/fileio_types.h new file mode 100644 index 00000000000..80f55394fad --- /dev/null +++ b/programs/fileio_types.h @@ -0,0 +1,63 @@ +#ifndef FILEIO_TYPES_HEADER +#define FILEIO_TYPES_HEADER + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "../lib/zstd.h" /* ZSTD_* */ + +/*-************************************* +* Parameters: FIO_prefs_t +***************************************/ + +typedef struct FIO_display_prefs_s FIO_display_prefs_t; + +typedef enum { FIO_ps_auto, FIO_ps_never, FIO_ps_always } FIO_progressSetting_e; + +struct FIO_display_prefs_s { + int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ + FIO_progressSetting_e progressSetting; +}; + + +typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t; + +typedef struct FIO_prefs_s { + + /* Algorithm preferences */ + FIO_compressionType_t compressionType; + U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ + int dictIDFlag; + int checksumFlag; + int blockSize; + int overlapLog; + U32 adaptiveMode; + U32 useRowMatchFinder; + int rsyncable; + int minAdaptLevel; + int maxAdaptLevel; + int ldmFlag; + int ldmHashLog; + int ldmMinMatch; + int ldmBucketSizeLog; + int ldmHashRateLog; + size_t streamSrcSize; + size_t targetCBlockSize; + int srcSizeHint; + int testMode; + ZSTD_paramSwitch_e literalCompressionMode; + + /* IO preferences */ + U32 removeSrcFile; + U32 overwrite; + U32 asyncIO; + + /* Computation resources preferences */ + unsigned memLimit; + int nbWorkers; + + int excludeCompressedFiles; + int patchFromMode; + int contentSize; + int allowBlockDevices; +} FIO_prefs_t; + +#endif /* FILEIO_TYPES_HEADER */ \ No newline at end of file diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c new file mode 100644 index 00000000000..13d44ee5df3 --- /dev/null +++ b/programs/fileio_utils.c @@ -0,0 +1,467 @@ +#include /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ +#include /* malloc, free */ +#include +#include /* errno */ + +#include "fileio_utils.h" + +FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; +UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +/* ********************************************************************** + * Sparse write + ************************************************************************/ + +/** FIO_fwriteSparse() : +* @return : storedSkips, +* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ +static unsigned +FIO_fwriteSparse(FILE* file, + const void* buffer, size_t bufferSize, + const FIO_prefs_t* const prefs, + unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + size_t bufferSizeT = bufferSize / sizeof(size_t); + const size_t* const bufferTEnd = bufferT + bufferSizeT; + const size_t* ptrT = bufferT; + static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ + + if (prefs->testMode) return 0; /* do not output anything in test mode */ + + if (!prefs->sparseFileSupport) { /* normal write */ + size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) + EXM_THROW(70, "Write error : cannot write decoded block : %s", + strerror(errno)); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) { + if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) + EXM_THROW(91, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t nb0T; + + /* adjust last segment if < 32 KB */ + size_t seg0SizeT = segmentSizeT; + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + + /* count leading zeroes */ + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeof(size_t)); + + if (nb0T != seg0SizeT) { /* not all 0s */ + size_t const nbNon0ST = seg0SizeT - nb0T; + /* skip leading zeros */ + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + /* write the rest */ + if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) + EXM_THROW(93, "Write error : cannot write decoded block : %s", + strerror(errno)); + } + ptrT += seg0SizeT; + } + + { static size_t const maskT = sizeof(size_t)-1; + if (bufferSize & maskT) { + /* size not multiple of sizeof(size_t) : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + const char* const restEnd = (const char*)buffer + bufferSize; + assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); + for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + /* not all remaining bytes are 0 */ + size_t const restSize = (size_t)(restEnd - restPtr); + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + if (fwrite(restPtr, 1, restSize, file) != restSize) + EXM_THROW(95, "Write error : cannot write end of decoded block : %s", + strerror(errno)); + storedSkips = 0; + } } } + + return storedSkips; +} + +static void +FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) +{ + if (prefs->testMode) assert(storedSkips == 0); + if (storedSkips>0) { + assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ + (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ + if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) + EXM_THROW(69, "Final skip error (sparse file support)"); + /* last zero must be explicitly written, + * so that skipped ones get implicitly translated as zero by FS */ + { const char lastZeroByte[1] = { 0 }; + if (fwrite(lastZeroByte, 1, 1, file) != 1) + EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); + } } +} + + +/* ********************************************************************** + * AsyncIO functionality + ************************************************************************/ +static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { + void *buffer; + io_job_t *job; + job = (io_job_t*) malloc(sizeof(io_job_t)); + buffer = malloc(bufferSize); + if(!job || !buffer) + EXM_THROW(101, "Allocation error : not enough memory"); + job->buffer = buffer; + job->bufferSize = bufferSize; + job->usedBufferSize = 0; + job->file = NULL; + job->ctx = ctx; + job->offset = 0; + return job; +} + + +/* IO_createThreadPool: + * Creates a thread pool and a mutex for threaded IO pool. + * Displays warning if asyncio is requested but MT isn't available. */ +static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { + ctx->threadPool = NULL; + if(prefs->asyncIO) { + if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) + EXM_THROW(102, "Failed creating write availableJobs mutex"); + if(ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL)) + EXM_THROW(103, "Failed creating write jobCompletedCond mutex"); + /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to + * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ + assert(MAX_IO_JOBS >= 2); + ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2); + if (!ctx->threadPool) + EXM_THROW(104, "Failed creating writer thread pool"); + } +} + +/* IoPool_create: + * Allocates and sets and a new write pool including its included availableJobs. */ +static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function poolFunction, + size_t bufferSize) { + io_pool_ctx_t *ctx; + int i; + ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); + if(!ctx) + EXM_THROW(100, "Allocation error : not enough memory"); + IO_createThreadPool(ctx, prefs); + ctx->prefs = prefs; + ctx->poolFunction = poolFunction; + ctx->totalIoJobs = ctx->threadPool ? MAX_IO_JOBS : 1; + ctx->availableJobsCount = ctx->totalIoJobs; + for(i=0; i < ctx->availableJobsCount; i++) { + ctx->availableJobs[i] = FIO_createIoJob(ctx, bufferSize); + } + ctx->completedJobsCount = 0; + ctx->storedSkips = 0; + ctx->reachedEof = 0; + ctx->nextReadOffset = 0; + ctx->waitingOnOffset = 0; + ctx->file = NULL; + + ctx->srcBufferBaseSize = 2 * bufferSize; + ctx->srcBufferBase = (U8*) malloc(ctx->srcBufferBaseSize); + ctx->srcBuffer = ctx->srcBufferBase; + ctx->srcBufferLoaded = 0; + + return ctx; +} + + +/* IoPool_releaseIoJob: + * Releases an acquired job back to the pool. Doesn't execute the job. */ +void IoPool_releaseIoJob(io_job_t *job) { + io_pool_ctx_t *ctx = job->ctx; + if(ctx->threadPool) { + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->availableJobsCount < MAX_IO_JOBS); + ctx->availableJobs[ctx->availableJobsCount++] = job; + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + } else { + assert(ctx->availableJobsCount == 0); + ctx->availableJobsCount++; + } +} + +static void IoPool_releaseAllCompletedJobs(io_pool_ctx_t* ctx) { + int i=0; + for(i=0; icompletedJobsCount; i++) { + io_job_t* job = (io_job_t*) ctx->completedJobs[i]; + IoPool_releaseIoJob(job); + } + ctx->completedJobsCount = 0; +} + + +/* IoPool_free: + * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ +void IoPool_free(io_pool_ctx_t* ctx) { + int i=0; + IoPool_releaseAllCompletedJobs(ctx); + if(ctx->threadPool) { + /* Make sure we finish all tasks and then free the resources */ + POOL_joinJobs(ctx->threadPool); + /* Make sure we are not leaking availableJobs */ + assert(ctx->availableJobsCount == ctx->totalIoJobs); + POOL_free(ctx->threadPool); + ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); + ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); + } + assert(ctx->file == NULL); + assert(ctx->storedSkips==0); + for(i=0; iavailableJobsCount; i++) { + io_job_t* job = (io_job_t*) ctx->availableJobs[i]; + free(job->buffer); + free(job); + } + free(ctx->srcBuffer); + free(ctx); +} + +/* IoPool_acquireJob: + * Returns an available write job to be used for a future write. */ +io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { + io_job_t *job; + assert(ctx->file != NULL || ctx->prefs->testMode); + if(ctx->threadPool) { + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->availableJobsCount > 0); + job = (io_job_t*) ctx->availableJobs[--ctx->availableJobsCount]; + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + } else { + assert(ctx->availableJobsCount == 1); + ctx->availableJobsCount--; + job = (io_job_t*)ctx->availableJobs[0]; + } + job->usedBufferSize = 0; + job->file = ctx->file; + job->offset = 0; + return job; +} + +/* IoPool_addJobToCompleted */ +static void IoPool_addJobToCompleted(io_job_t *job) { + io_pool_ctx_t *ctx = job->ctx; + if(ctx->threadPool) + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + assert(ctx->completedJobsCount < MAX_IO_JOBS); + ctx->completedJobs[ctx->completedJobsCount++] = job; + if(ctx->threadPool) { + ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + } +} + +/* assuming ioJobsMutex is locked */ +static io_job_t* IoPool_findWaitingJob(io_pool_ctx_t *ctx) { + io_job_t *job = NULL; + int i = 0; + for (i=0; icompletedJobsCount; i++) { + job = (io_job_t *) ctx->completedJobs[i]; + if (job->offset == ctx->waitingOnOffset) { + ctx->completedJobs[i] = ctx->completedJobs[--ctx->completedJobsCount]; + return job; + } + } + return NULL; +} + +/* IoPool_getNextCompletedJob */ +static io_job_t* IoPool_getNextCompletedJob(io_pool_ctx_t *ctx) { + io_job_t *job = NULL; + if(ctx->threadPool) + ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + + job = IoPool_findWaitingJob(ctx); + + while (!job && (ctx->availableJobsCount + ctx->completedJobsCount < ctx->totalIoJobs)) { + assert(ctx->threadPool != NULL); + ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->ioJobsMutex); + job = IoPool_findWaitingJob(ctx); + } + + if(job) { + assert(job->offset == ctx->waitingOnOffset); + ctx->waitingOnOffset += job->usedBufferSize; + } + + if(ctx->threadPool) + ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + return job; +} + +/* IoPool_enqueueJob: + * Queues a write job for execution. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +static void IoPool_enqueueJob(io_job_t *job) { + io_pool_ctx_t* ctx = job->ctx; + if(ctx->threadPool) + POOL_add(ctx->threadPool, ctx->poolFunction, job); + else + ctx->poolFunction(job); +} + +/* IoPool_enqueueAndReacquireWriteJob: + * Queues a write job for execution and acquires a new one. + * After execution `job`'s pointed value would change to the newly acquired job. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +void IoPool_enqueueAndReacquireWriteJob(io_job_t **job) { + IoPool_enqueueJob(*job); + *job = IoPool_acquireJob((*job)->ctx); +} + +/* WritePool_sparseWriteEnd: + * Ends sparse writes to the current file. + * Blocks on completion of all current write jobs before executing. */ +void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx) { + assert(ctx != NULL); + if(ctx->threadPool) + POOL_joinJobs(ctx->threadPool); + FIO_fwriteSparseEnd(ctx->prefs, ctx->file, ctx->storedSkips); + ctx->storedSkips = 0; +} + +/* IoPool_setFile: + * Sets the destination file for future files in the pool. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. + * Also requires ending of sparse write if a previous file was used in sparse mode. */ +void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { + assert(ctx!=NULL); + /* We can change the dst file only if we have finished writing */ + IoPool_releaseAllCompletedJobs(ctx); + if(ctx->threadPool) + POOL_joinJobs(ctx->threadPool); + assert(ctx->storedSkips == 0); + assert(ctx->availableJobsCount == ctx->totalIoJobs); + ctx->file = file; + ctx->nextReadOffset = 0; + ctx->waitingOnOffset = 0; + ctx->srcBuffer = ctx->srcBufferBase; + ctx->reachedEof = 0; +} + +/* WritePool_closeDstFile: + * Ends sparse write and closes the writePool's current file and sets the file to NULL. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ +int WritePool_closeDstFile(io_pool_ctx_t *ctx) { + FILE *dstFile = ctx->file; + assert(dstFile!=NULL || ctx->prefs->testMode!=0); + WritePool_sparseWriteEnd(ctx); + IoPool_setFile(ctx, NULL); + return fclose(dstFile); +} + +/* WritePool_executeWriteJob: + * Executes a write job synchronously. Can be used as a function for a thread pool. */ +static void WritePool_executeWriteJob(void* opaque){ + io_job_t* job = (io_job_t*) opaque; + io_pool_ctx_t* ctx = job->ctx; + ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); + IoPool_releaseIoJob(job); +} + +/* WritePool_create: + * Allocates and sets and a new write pool including its included jobs. */ +io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) { + return IoPool_create(prefs, WritePool_executeWriteJob, bufferSize); +} + +/* ReadPool_executeReadJob: + * Executes a read job synchronously. Can be used as a function for a thread pool. */ +static void ReadPool_executeReadJob(void* opaque){ + io_job_t* job = (io_job_t*) opaque; + io_pool_ctx_t* ctx = job->ctx; + if(ctx->reachedEof) { + job->usedBufferSize = 0; + IoPool_addJobToCompleted(job); + return; + } + job->usedBufferSize = fread(job->buffer, 1, job->bufferSize, job->file); + if(job->usedBufferSize < job->bufferSize) { + if(ferror(job->file)) { + EXM_THROW(37, "Read error"); + } else if(feof(job->file)) { + ctx->reachedEof = 1; + } else + EXM_THROW(37, "Unexpected short read"); + } + IoPool_addJobToCompleted(job); +} + +/* ReadPool_create: + * Allocates and sets and a new write pool including its included jobs. */ +io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { + return IoPool_create(prefs, ReadPool_executeReadJob, bufferSize); +} + +void ReadPool_consumeBytes(io_pool_ctx_t *ctx, size_t n) { + assert(n <= ctx->srcBufferLoaded); + assert(ctx->srcBuffer + n <= ctx->srcBufferBase + ctx->srcBufferBaseSize); + ctx->srcBufferLoaded -= n; + ctx->srcBuffer += n; +} + +static void ReadPool_enqueueRead(io_pool_ctx_t *ctx) { + io_job_t *job = IoPool_acquireJob(ctx); + job->offset = ctx->nextReadOffset; + ctx->nextReadOffset += job->bufferSize; + IoPool_enqueueJob(job); +} + + +size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n) { + io_job_t *job; + size_t srcBufferOffsetFromBase; + size_t srcBufferRemainingSpace; + size_t bytesRead = 0; + while (ctx->srcBufferLoaded < n) { + job = IoPool_getNextCompletedJob(ctx); + if(job == NULL) + break; + srcBufferOffsetFromBase = ctx->srcBuffer - ctx->srcBufferBase; + srcBufferRemainingSpace = ctx->srcBufferBaseSize - (srcBufferOffsetFromBase + ctx->srcBufferLoaded); + if(job->usedBufferSize > srcBufferRemainingSpace) { + memmove(ctx->srcBufferBase, ctx->srcBuffer, ctx->srcBufferLoaded); + ctx->srcBuffer = ctx->srcBufferBase; + } + memcpy(ctx->srcBuffer + ctx->srcBufferLoaded, job->buffer, job->usedBufferSize); + bytesRead += job->usedBufferSize; + ctx->srcBufferLoaded += job->usedBufferSize; + if(job->usedBufferSize < job->bufferSize) { + IoPool_releaseIoJob(job); + break; + } + IoPool_releaseIoJob(job); + ReadPool_enqueueRead(ctx); + } + return bytesRead; +} + +size_t ReadPool_consumeAndReadAll(io_pool_ctx_t *ctx) { + ReadPool_consumeBytes(ctx, ctx->srcBufferLoaded); + return ReadPool_readBuffer(ctx, ZSTD_DStreamInSize()); +} + +void ReadPool_startReading(io_pool_ctx_t *ctx) { + int i; + for (i = 0; i < ctx->availableJobsCount; i++) { + ReadPool_enqueueRead(ctx); + } +} \ No newline at end of file diff --git a/programs/fileio_utils.h b/programs/fileio_utils.h new file mode 100644 index 00000000000..bce82493451 --- /dev/null +++ b/programs/fileio_utils.h @@ -0,0 +1,219 @@ +#ifndef FILEIO_UTILS_HEADER +#define FILEIO_UTILS_HEADER + +#include "../lib/common/mem.h" /* U32, U64 */ +#include "timefn.h" +#include "fileio_types.h" +#include "platform.h" +#include "util.h" + +/*-************************************* +* Constants +***************************************/ +#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ +#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ + +/* Default file permissions 0666 (modulated by umask) */ +#if !defined(_WIN32) +/* These macros aren't defined on windows. */ +#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) +#else +#define DEFAULT_FILE_PERMISSIONS (0666) +#endif + +/*-************************************* +* Macros +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) +#undef MAX +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +extern FIO_display_prefs_t g_display_prefs; + +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +extern UTIL_time_t g_displayClock; + +#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } +#define DISPLAYUPDATE(l, ...) { \ + if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ + if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ + DELAY_NEXT_UPDATE(); \ + DISPLAY(__VA_ARGS__); \ + if (g_display_prefs.displayLevel>=4) fflush(stderr); \ + } } } + +#undef MIN /* in case it would be already defined */ +#define MIN(a,b) ((a) < (b) ? (a) : (b)) + + +#define EXM_THROW(error, ...) \ +{ \ + DISPLAYLEVEL(1, "zstd: "); \ + DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + exit(error); \ +} + +#define CHECK_V(v, f) \ + v = f; \ + if (ZSTD_isError(v)) { \ + DISPLAYLEVEL(5, "%s \n", #f); \ + EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ + } +#define CHECK(f) { size_t err; CHECK_V(err, f); } + + +/* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define LONG_SEEK _fseeki64 +# define LONG_TELL _ftelli64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define LONG_SEEK fseeko +# define LONG_TELL ftello +#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) +# define LONG_SEEK fseeko64 +# define LONG_TELL ftello64 +#elif defined(_WIN32) && !defined(__DJGPP__) +# include + static int LONG_SEEK(FILE* file, __int64 offset, int origin) { + LARGE_INTEGER off; + DWORD method; + off.QuadPart = offset; + if (origin == SEEK_END) + method = FILE_END; + else if (origin == SEEK_CUR) + method = FILE_CURRENT; + else + method = FILE_BEGIN; + + if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) + return 0; + else + return -1; + } + static __int64 LONG_TELL(FILE* file) { + LARGE_INTEGER off, newOff; + off.QuadPart = 0; + newOff.QuadPart = 0; + SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); + return newOff.QuadPart; + } +#else +# define LONG_SEEK fseek +# define LONG_TELL ftell +#endif + + +/* ********************************************************************** + * AsyncIO functionality + ************************************************************************/ +#include "../lib/common/pool.h" +#include "../lib/common/threading.h" + +#define MAX_IO_JOBS (10) + +typedef struct { + /* These struct fields should be set only on creation and not changed afterwards */ + POOL_ctx* threadPool; + int totalIoJobs; + FIO_prefs_t* prefs; + POOL_function poolFunction; + + /* Controls the file we currently write to, make changes only by using provided utility functions */ + FILE* file; + unsigned storedSkips; // only used for write io pool + int reachedEof; + U64 nextReadOffset; + U64 waitingOnOffset; + + /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should + * only be mutated after locking the mutex */ + ZSTD_pthread_mutex_t ioJobsMutex; + void* availableJobs[MAX_IO_JOBS]; + int availableJobsCount; + + void* completedJobs[MAX_IO_JOBS]; + int completedJobsCount; + ZSTD_pthread_cond_t jobCompletedCond; + + U8 *srcBufferBase; + size_t srcBufferBaseSize; + U8 *srcBuffer; + size_t srcBufferLoaded; +} io_pool_ctx_t; + +typedef struct { + /* These fields are automatically set and shouldn't be changed by non WritePool code. */ + io_pool_ctx_t *ctx; + FILE* file; + void *buffer; + size_t bufferSize; + + /* This field should be changed before a job is queued for execution and should contain the number + * of bytes to write from the buffer. */ + size_t usedBufferSize; + U64 offset; +} io_job_t; + + +/* IoPool_releaseIoJob: + * Releases an acquired job back to the pool. Doesn't execute the job. */ +void IoPool_releaseIoJob(io_job_t *job); + + +/* IoPool_free: + * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ +void IoPool_free(io_pool_ctx_t* ctx); + +/* IoPool_acquireJob: + * Returns an available write job to be used for a future write. */ +io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx); + +/* IoPool_enqueueAndReacquireWriteJob: + * Queues a write job for execution and acquires a new one. + * After execution `job`'s pointed value would change to the newly acquired job. + * Make sure to set `usedBufferSize` to the wanted length before call. + * The queued job shouldn't be used directly after queueing it. */ +void IoPool_enqueueAndReacquireWriteJob(io_job_t **job); + +/* WritePool_sparseWriteEnd: + * Ends sparse writes to the current file. + * Blocks on completion of all current write jobs before executing. */ +void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx); + +/* IoPool_setFile: + * Sets the destination file for future files in the pool. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. + * Also requires ending of sparse write if a previous file was used in sparse mode. */ +void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file); + +/* WritePool_closeDstFile: + * Ends sparse write and closes the writePool's current file and sets the file to NULL. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ +int WritePool_closeDstFile(io_pool_ctx_t *ctx); + +/* WritePool_create: + * Allocates and sets and a new write pool including its included jobs. */ +io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize); + +/* ReadPool_create: + * Allocates and sets and a new write pool including its included jobs. */ +io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize); + +void ReadPool_consumeBytes(io_pool_ctx_t *ctx, size_t n); + +size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n); + +size_t ReadPool_consumeAndReadAll(io_pool_ctx_t *ctx); + +void ReadPool_startReading(io_pool_ctx_t *ctx); +#endif /* FILEIO_UTILS_HEADER */ \ No newline at end of file From 4ea77b7313111362f79a9166b694de199c10770a Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 16:17:06 -0800 Subject: [PATCH 17/25] Async IO compression: Created `read_pool_ctx_t` to remove read-only parts from `io_pool_ctx_t`. --- programs/fileio.c | 18 +++--- programs/fileio_utils.c | 140 +++++++++++++++++++++------------------- programs/fileio_utils.h | 39 +++++++---- 3 files changed, 109 insertions(+), 88 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 0970771e4dc..c6671396597 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -789,7 +789,7 @@ typedef struct { const char* dictFileName; ZSTD_CStream* cctx; io_pool_ctx_t *writeCtx; - io_pool_ctx_t *readCtx; + read_pool_ctx_t *readCtx; } cRess_t; /** ZSTD_cycleLog() : @@ -913,7 +913,7 @@ static void FIO_freeCResources(const cRess_t* const ress) { free(ress->dictBuffer); IoPool_free(ress->writeCtx); - IoPool_free(ress->readCtx); + ReadPool_free(ress->readCtx); ZSTD_freeCStream(ress->cctx); /* never fails */ } @@ -1538,7 +1538,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, stat_t statbuf; int transferMTime = 0; FILE *dstFile; - assert(ress.readCtx->file != NULL); + assert(ress.readCtx->base.file != NULL); if (ress.writeCtx->file == NULL) { int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp (srcFileName, stdinmark) @@ -1639,14 +1639,14 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile == NULL) return 1; /* srcFile could not be opened */ - IoPool_setFile(ress.readCtx, srcFile); + ReadPool_setFile(ress.readCtx, srcFile); ReadPool_startReading(ress.readCtx); result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); fclose(srcFile); // TODO: implement proper & safe close - IoPool_setFile(ress.readCtx, NULL); + ReadPool_setFile(ress.readCtx, NULL); if ( prefs->removeSrcFile /* --rm */ && result == 0 /* success */ && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ @@ -1869,7 +1869,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, typedef struct { ZSTD_DStream* dctx; io_pool_ctx_t *writeCtx; - io_pool_ctx_t *readCtx; + read_pool_ctx_t *readCtx; } dRess_t; static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) @@ -1904,7 +1904,7 @@ static void FIO_freeDResources(dRess_t ress) { CHECK( ZSTD_freeDStream(ress.dctx) ); IoPool_free(ress.writeCtx); - IoPool_free(ress.readCtx); + ReadPool_free(ress.readCtx); } /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @@ -2420,12 +2420,12 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile==NULL) return 1; - IoPool_setFile(ress.readCtx, srcFile); + ReadPool_setFile(ress.readCtx, srcFile); ReadPool_startReading(ress.readCtx); result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName); - IoPool_setFile(ress.readCtx, NULL); + ReadPool_setFile(ress.readCtx, NULL); /* Close file */ if (fclose(srcFile)) { diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index 13d44ee5df3..1409f8ba3da 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -138,9 +138,7 @@ static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { ctx->threadPool = NULL; if(prefs->asyncIO) { if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) - EXM_THROW(102, "Failed creating write availableJobs mutex"); - if(ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL)) - EXM_THROW(103, "Failed creating write jobCompletedCond mutex"); + EXM_THROW(102,"Failed creating write availableJobs mutex"); /* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to * decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */ assert(MAX_IO_JOBS >= 2); @@ -150,15 +148,10 @@ static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { } } -/* IoPool_create: +/* IoPool_init: * Allocates and sets and a new write pool including its included availableJobs. */ -static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function poolFunction, - size_t bufferSize) { - io_pool_ctx_t *ctx; +static void IoPool_init(io_pool_ctx_t *ctx, FIO_prefs_t* const prefs, POOL_function poolFunction, size_t bufferSize) { int i; - ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); - if(!ctx) - EXM_THROW(100, "Allocation error : not enough memory"); IO_createThreadPool(ctx, prefs); ctx->prefs = prefs; ctx->poolFunction = poolFunction; @@ -167,26 +160,15 @@ static io_pool_ctx_t* IoPool_create(FIO_prefs_t* const prefs, POOL_function pool for(i=0; i < ctx->availableJobsCount; i++) { ctx->availableJobs[i] = FIO_createIoJob(ctx, bufferSize); } - ctx->completedJobsCount = 0; ctx->storedSkips = 0; - ctx->reachedEof = 0; - ctx->nextReadOffset = 0; - ctx->waitingOnOffset = 0; ctx->file = NULL; - - ctx->srcBufferBaseSize = 2 * bufferSize; - ctx->srcBufferBase = (U8*) malloc(ctx->srcBufferBaseSize); - ctx->srcBuffer = ctx->srcBufferBase; - ctx->srcBufferLoaded = 0; - - return ctx; } /* IoPool_releaseIoJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ void IoPool_releaseIoJob(io_job_t *job) { - io_pool_ctx_t *ctx = job->ctx; + io_pool_ctx_t *ctx = (io_pool_ctx_t *) job->ctx; if(ctx->threadPool) { ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); assert(ctx->availableJobsCount < MAX_IO_JOBS); @@ -198,8 +180,8 @@ void IoPool_releaseIoJob(io_job_t *job) { } } -static void IoPool_releaseAllCompletedJobs(io_pool_ctx_t* ctx) { - int i=0; +static void ReadPool_releaseAllCompletedJobs(read_pool_ctx_t* ctx) { + int i; for(i=0; icompletedJobsCount; i++) { io_job_t* job = (io_job_t*) ctx->completedJobs[i]; IoPool_releaseIoJob(job); @@ -212,7 +194,6 @@ static void IoPool_releaseAllCompletedJobs(io_pool_ctx_t* ctx) { * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ void IoPool_free(io_pool_ctx_t* ctx) { int i=0; - IoPool_releaseAllCompletedJobs(ctx); if(ctx->threadPool) { /* Make sure we finish all tasks and then free the resources */ POOL_joinJobs(ctx->threadPool); @@ -220,7 +201,6 @@ void IoPool_free(io_pool_ctx_t* ctx) { assert(ctx->availableJobsCount == ctx->totalIoJobs); POOL_free(ctx->threadPool); ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); - ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); } assert(ctx->file == NULL); assert(ctx->storedSkips==0); @@ -229,7 +209,6 @@ void IoPool_free(io_pool_ctx_t* ctx) { free(job->buffer); free(job); } - free(ctx->srcBuffer); free(ctx); } @@ -254,23 +233,23 @@ io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { return job; } -/* IoPool_addJobToCompleted */ -static void IoPool_addJobToCompleted(io_job_t *job) { - io_pool_ctx_t *ctx = job->ctx; - if(ctx->threadPool) - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); +/* ReadPool_addJobToCompleted */ +static void ReadPool_addJobToCompleted(io_job_t *job) { + read_pool_ctx_t *ctx = (read_pool_ctx_t *)job->ctx; + if(ctx->base.threadPool) + ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); assert(ctx->completedJobsCount < MAX_IO_JOBS); ctx->completedJobs[ctx->completedJobsCount++] = job; - if(ctx->threadPool) { + if(ctx->base.threadPool) { ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex); } } /* assuming ioJobsMutex is locked */ -static io_job_t* IoPool_findWaitingJob(io_pool_ctx_t *ctx) { +static io_job_t* ReadPool_findWaitingJob(read_pool_ctx_t *ctx) { io_job_t *job = NULL; - int i = 0; + int i; for (i=0; icompletedJobsCount; i++) { job = (io_job_t *) ctx->completedJobs[i]; if (job->offset == ctx->waitingOnOffset) { @@ -281,18 +260,18 @@ static io_job_t* IoPool_findWaitingJob(io_pool_ctx_t *ctx) { return NULL; } -/* IoPool_getNextCompletedJob */ -static io_job_t* IoPool_getNextCompletedJob(io_pool_ctx_t *ctx) { +/* ReadPool_getNextCompletedJob */ +static io_job_t* ReadPool_getNextCompletedJob(read_pool_ctx_t *ctx) { io_job_t *job = NULL; - if(ctx->threadPool) - ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); + if(ctx->base.threadPool) + ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); - job = IoPool_findWaitingJob(ctx); + job = ReadPool_findWaitingJob(ctx); - while (!job && (ctx->availableJobsCount + ctx->completedJobsCount < ctx->totalIoJobs)) { - assert(ctx->threadPool != NULL); - ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->ioJobsMutex); - job = IoPool_findWaitingJob(ctx); + while (!job && (ctx->base.availableJobsCount + ctx->completedJobsCount < ctx->base.totalIoJobs)) { + assert(ctx->base.threadPool != NULL); + ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->base.ioJobsMutex); + job = ReadPool_findWaitingJob(ctx); } if(job) { @@ -300,8 +279,8 @@ static io_job_t* IoPool_getNextCompletedJob(io_pool_ctx_t *ctx) { ctx->waitingOnOffset += job->usedBufferSize; } - if(ctx->threadPool) - ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex); + if(ctx->base.threadPool) + ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex); return job; } @@ -310,7 +289,7 @@ static io_job_t* IoPool_getNextCompletedJob(io_pool_ctx_t *ctx) { * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ static void IoPool_enqueueJob(io_job_t *job) { - io_pool_ctx_t* ctx = job->ctx; + io_pool_ctx_t* ctx = (io_pool_ctx_t *)job->ctx; if(ctx->threadPool) POOL_add(ctx->threadPool, ctx->poolFunction, job); else @@ -324,7 +303,7 @@ static void IoPool_enqueueJob(io_job_t *job) { * The queued job shouldn't be used directly after queueing it. */ void IoPool_enqueueAndReacquireWriteJob(io_job_t **job) { IoPool_enqueueJob(*job); - *job = IoPool_acquireJob((*job)->ctx); + *job = IoPool_acquireJob((io_pool_ctx_t *)(*job)->ctx); } /* WritePool_sparseWriteEnd: @@ -345,12 +324,18 @@ void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx) { void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { assert(ctx!=NULL); /* We can change the dst file only if we have finished writing */ - IoPool_releaseAllCompletedJobs(ctx); if(ctx->threadPool) POOL_joinJobs(ctx->threadPool); assert(ctx->storedSkips == 0); assert(ctx->availableJobsCount == ctx->totalIoJobs); ctx->file = file; +} + +/* ReadPool_setFile */ +void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file) { + assert(ctx!=NULL); + ReadPool_releaseAllCompletedJobs(ctx); + IoPool_setFile(&ctx->base, file); ctx->nextReadOffset = 0; ctx->waitingOnOffset = 0; ctx->srcBuffer = ctx->srcBufferBase; @@ -372,7 +357,7 @@ int WritePool_closeDstFile(io_pool_ctx_t *ctx) { * Executes a write job synchronously. Can be used as a function for a thread pool. */ static void WritePool_executeWriteJob(void* opaque){ io_job_t* job = (io_job_t*) opaque; - io_pool_ctx_t* ctx = job->ctx; + io_pool_ctx_t* ctx = (io_pool_ctx_t*) job->ctx; ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); IoPool_releaseIoJob(job); } @@ -380,17 +365,20 @@ static void WritePool_executeWriteJob(void* opaque){ /* WritePool_create: * Allocates and sets and a new write pool including its included jobs. */ io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) { - return IoPool_create(prefs, WritePool_executeWriteJob, bufferSize); + io_pool_ctx_t* ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); + if(!ctx) EXM_THROW(100, "Allocation error : not enough memory"); + IoPool_init(ctx, prefs, WritePool_executeWriteJob, bufferSize); + return ctx; } /* ReadPool_executeReadJob: * Executes a read job synchronously. Can be used as a function for a thread pool. */ static void ReadPool_executeReadJob(void* opaque){ io_job_t* job = (io_job_t*) opaque; - io_pool_ctx_t* ctx = job->ctx; + read_pool_ctx_t* ctx = (read_pool_ctx_t *)job->ctx; if(ctx->reachedEof) { job->usedBufferSize = 0; - IoPool_addJobToCompleted(job); + ReadPool_addJobToCompleted(job); return; } job->usedBufferSize = fread(job->buffer, 1, job->bufferSize, job->file); @@ -402,37 +390,59 @@ static void ReadPool_executeReadJob(void* opaque){ } else EXM_THROW(37, "Unexpected short read"); } - IoPool_addJobToCompleted(job); + ReadPool_addJobToCompleted(job); } /* ReadPool_create: * Allocates and sets and a new write pool including its included jobs. */ -io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { - return IoPool_create(prefs, ReadPool_executeReadJob, bufferSize); +read_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { + read_pool_ctx_t* ctx = (read_pool_ctx_t*) malloc(sizeof(read_pool_ctx_t)); + if(!ctx) EXM_THROW(100, "Allocation error : not enough memory"); + IoPool_init(&ctx->base, prefs, ReadPool_executeReadJob, bufferSize); + + ctx->srcBufferBaseSize = 2 * bufferSize; + ctx->srcBufferBase = (U8*) malloc(ctx->srcBufferBaseSize); + ctx->srcBuffer = ctx->srcBufferBase; + ctx->srcBufferLoaded = 0; + ctx->completedJobsCount = 0; + + if(ctx->base.threadPool) + if (ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL)) + EXM_THROW(103,"Failed creating write jobCompletedCond mutex"); + + return ctx; +} + +void ReadPool_free(read_pool_ctx_t* ctx) { + ReadPool_releaseAllCompletedJobs(ctx); + if(ctx->base.threadPool) + ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); + IoPool_free(&ctx->base); + free(ctx->srcBufferBase); } -void ReadPool_consumeBytes(io_pool_ctx_t *ctx, size_t n) { +void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n) { assert(n <= ctx->srcBufferLoaded); assert(ctx->srcBuffer + n <= ctx->srcBufferBase + ctx->srcBufferBaseSize); ctx->srcBufferLoaded -= n; ctx->srcBuffer += n; } -static void ReadPool_enqueueRead(io_pool_ctx_t *ctx) { - io_job_t *job = IoPool_acquireJob(ctx); +static void ReadPool_enqueueRead(read_pool_ctx_t *ctx) { + io_job_t *job = IoPool_acquireJob(&ctx->base); job->offset = ctx->nextReadOffset; ctx->nextReadOffset += job->bufferSize; IoPool_enqueueJob(job); } -size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n) { +size_t ReadPool_readBuffer(read_pool_ctx_t *ctx, size_t n) { io_job_t *job; size_t srcBufferOffsetFromBase; size_t srcBufferRemainingSpace; size_t bytesRead = 0; while (ctx->srcBufferLoaded < n) { - job = IoPool_getNextCompletedJob(ctx); + job = ReadPool_getNextCompletedJob(ctx); if(job == NULL) break; srcBufferOffsetFromBase = ctx->srcBuffer - ctx->srcBufferBase; @@ -454,14 +464,14 @@ size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n) { return bytesRead; } -size_t ReadPool_consumeAndReadAll(io_pool_ctx_t *ctx) { +size_t ReadPool_consumeAndReadAll(read_pool_ctx_t *ctx) { ReadPool_consumeBytes(ctx, ctx->srcBufferLoaded); return ReadPool_readBuffer(ctx, ZSTD_DStreamInSize()); } -void ReadPool_startReading(io_pool_ctx_t *ctx) { +void ReadPool_startReading(read_pool_ctx_t *ctx) { int i; - for (i = 0; i < ctx->availableJobsCount; i++) { + for (i = 0; i < ctx->base.availableJobsCount; i++) { ReadPool_enqueueRead(ctx); } } \ No newline at end of file diff --git a/programs/fileio_utils.h b/programs/fileio_utils.h index bce82493451..deeff83ae6d 100644 --- a/programs/fileio_utils.h +++ b/programs/fileio_utils.h @@ -131,29 +131,34 @@ typedef struct { /* Controls the file we currently write to, make changes only by using provided utility functions */ FILE* file; unsigned storedSkips; // only used for write io pool - int reachedEof; - U64 nextReadOffset; - U64 waitingOnOffset; /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should * only be mutated after locking the mutex */ ZSTD_pthread_mutex_t ioJobsMutex; void* availableJobs[MAX_IO_JOBS]; int availableJobsCount; +} io_pool_ctx_t; - void* completedJobs[MAX_IO_JOBS]; - int completedJobsCount; - ZSTD_pthread_cond_t jobCompletedCond; +typedef struct { + io_pool_ctx_t base; + + int reachedEof; + U64 nextReadOffset; + U64 waitingOnOffset; U8 *srcBufferBase; size_t srcBufferBaseSize; U8 *srcBuffer; size_t srcBufferLoaded; -} io_pool_ctx_t; + + void* completedJobs[MAX_IO_JOBS]; + int completedJobsCount; + ZSTD_pthread_cond_t jobCompletedCond; +} read_pool_ctx_t; typedef struct { /* These fields are automatically set and shouldn't be changed by non WritePool code. */ - io_pool_ctx_t *ctx; + void *ctx; FILE* file; void *buffer; size_t bufferSize; @@ -169,7 +174,6 @@ typedef struct { * Releases an acquired job back to the pool. Doesn't execute the job. */ void IoPool_releaseIoJob(io_job_t *job); - /* IoPool_free: * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ void IoPool_free(io_pool_ctx_t* ctx); @@ -207,13 +211,20 @@ io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize); /* ReadPool_create: * Allocates and sets and a new write pool including its included jobs. */ -io_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize); +read_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize); + +/* ReadPool_free: + * Allocates and sets and a new write pool including its included jobs. */ +void ReadPool_free(read_pool_ctx_t* ctx); + +void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n); + +size_t ReadPool_readBuffer(read_pool_ctx_t *ctx, size_t n); -void ReadPool_consumeBytes(io_pool_ctx_t *ctx, size_t n); +size_t ReadPool_consumeAndReadAll(read_pool_ctx_t *ctx); -size_t ReadPool_readBuffer(io_pool_ctx_t *ctx, size_t n); +void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file); -size_t ReadPool_consumeAndReadAll(io_pool_ctx_t *ctx); +void ReadPool_startReading(read_pool_ctx_t *ctx); -void ReadPool_startReading(io_pool_ctx_t *ctx); #endif /* FILEIO_UTILS_HEADER */ \ No newline at end of file From 6106b7c4577d47a63c750a4edda1da0e73e9ceb1 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 17:11:02 -0800 Subject: [PATCH 18/25] Async IO compression: Created `write_pool_ctx_t` to remove write-only parts from `io_pool_ctx_t`. Also adjusted function ordering. --- programs/fileio.c | 84 +++++++------- programs/fileio_utils.c | 237 +++++++++++++++++++++++----------------- programs/fileio_utils.h | 38 ++++--- 3 files changed, 197 insertions(+), 162 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c6671396597..ed12dd6ed68 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -788,7 +788,7 @@ typedef struct { size_t dictBufferSize; const char* dictFileName; ZSTD_CStream* cctx; - io_pool_ctx_t *writeCtx; + write_pool_ctx_t *writeCtx; read_pool_ctx_t *readCtx; } cRess_t; @@ -912,7 +912,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, static void FIO_freeCResources(const cRess_t* const ress) { free(ress->dictBuffer); - IoPool_free(ress->writeCtx); + WritePool_free(ress->writeCtx); ReadPool_free(ress->readCtx); ZSTD_freeCStream(ress->cctx); /* never fails */ } @@ -942,7 +942,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); } } - writeJob = IoPool_acquireJob(ress->writeCtx); + writeJob = WritePool_acquireJob(ress->writeCtx); strm.next_in = 0; strm.avail_in = 0; strm.next_out = (Bytef*)writeJob->buffer; @@ -969,7 +969,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { writeJob->usedBufferSize = cSize; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += cSize; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -989,7 +989,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no { size_t const cSize = writeJob->bufferSize - strm.avail_out; if (cSize) { writeJob->usedBufferSize = cSize; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += cSize; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -1004,7 +1004,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); } } *readsize = inFileSize; - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return outFileSize; } @@ -1039,7 +1039,7 @@ FIO_compressLzmaFrame(cRess_t* ress, EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); } - writeJob = IoPool_acquireJob(ress->writeCtx); + writeJob = WritePool_acquireJob(ress->writeCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = 0; @@ -1066,7 +1066,7 @@ FIO_compressLzmaFrame(cRess_t* ress, { size_t const compBytes = writeJob->bufferSize - strm.avail_out; if (compBytes) { writeJob->usedBufferSize = compBytes; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += compBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -1085,7 +1085,7 @@ FIO_compressLzmaFrame(cRess_t* ress, lzma_end(&strm); *readsize = inFileSize; - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return outFileSize; @@ -1113,7 +1113,7 @@ FIO_compressLz4Frame(cRess_t* ress, LZ4F_preferences_t prefs; LZ4F_compressionContext_t ctx; - io_job_t *writeJob = IoPool_acquireJob(ress->writeCtx); + io_job_t *writeJob = WritePool_acquireJob(ress->writeCtx); LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) @@ -1140,7 +1140,7 @@ FIO_compressLz4Frame(cRess_t* ress, EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); writeJob->usedBufferSize = headerSize; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; /* Read first block */ @@ -1167,7 +1167,7 @@ FIO_compressLz4Frame(cRess_t* ress, /* Write Block */ writeJob->usedBufferSize = outSize; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); /* Read next block */ ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded); @@ -1182,13 +1182,13 @@ FIO_compressLz4Frame(cRess_t* ress, srcFileName, LZ4F_getErrorName(headerSize)); writeJob->usedBufferSize = headerSize; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += headerSize; } *readsize = inFileSize; LZ4F_freeCompressionContext(ctx); - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return outFileSize; @@ -1204,7 +1204,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, int compressionLevel, U64* readsize) { cRess_t const ress = *ressPtr; - io_job_t *writeJob = IoPool_acquireJob(ressPtr->writeCtx); + io_job_t *writeJob = WritePool_acquireJob(ressPtr->writeCtx); U64 compressedfilesize = 0; ZSTD_EndDirective directive = ZSTD_e_continue; @@ -1278,7 +1278,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); if (outBuff.pos) { writeJob->usedBufferSize = outBuff.pos; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); compressedfilesize += outBuff.pos; } @@ -1415,7 +1415,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned long long)*readsize, (unsigned long long)fileSize); } - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ressPtr->writeCtx); return compressedfilesize; @@ -1539,7 +1539,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, int transferMTime = 0; FILE *dstFile; assert(ress.readCtx->base.file != NULL); - if (ress.writeCtx->file == NULL) { + if (ress.writeCtx->base.file == NULL) { int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) @@ -1553,7 +1553,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; /* could not open dstFileName */ - IoPool_setFile(ress.writeCtx, dstFile); + WritePool_setFile(ress.writeCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, * and the user presses Ctrl-C when asked if they wish to overwrite. @@ -1802,7 +1802,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, if (dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - IoPool_setFile(ress.writeCtx, dstFile); + WritePool_setFile(ress.writeCtx, dstFile); for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); if (!status) fCtx->nbFilesProcessed++; @@ -1868,7 +1868,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, ***************************************************************************/ typedef struct { ZSTD_DStream* dctx; - io_pool_ctx_t *writeCtx; + write_pool_ctx_t *writeCtx; read_pool_ctx_t *readCtx; } dRess_t; @@ -1903,7 +1903,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi static void FIO_freeDResources(dRess_t ress) { CHECK( ZSTD_freeDStream(ress.dctx) ); - IoPool_free(ress.writeCtx); + WritePool_free(ress.writeCtx); ReadPool_free(ress.readCtx); } @@ -1912,7 +1912,7 @@ static void FIO_freeDResources(dRess_t ress) static int FIO_passThrough(dRess_t *ress) { size_t const blockSize = MIN(MIN(64 KB, ZSTD_DStreamInSize()), ZSTD_DStreamOutSize()); - io_job_t *writeJob = IoPool_acquireJob(ress->writeCtx); + io_job_t *writeJob = WritePool_acquireJob(ress->writeCtx); ReadPool_readBuffer(ress->readCtx, blockSize); while(ress->readCtx->srcBufferLoaded) { @@ -1922,11 +1922,11 @@ static int FIO_passThrough(dRess_t *ress) assert(writeSize <= writeJob->bufferSize); memcpy(writeJob->buffer, ress->readCtx->srcBuffer, writeSize); writeJob->usedBufferSize = writeSize; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); ReadPool_consumeBytes(ress->readCtx, writeSize); } assert(ress->readCtx->reachedEof); - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return 0; } @@ -1974,7 +1974,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, U64 alreadyDecoded) /* for multi-frames streams */ { U64 frameSize = 0; - io_job_t *writeJob = IoPool_acquireJob(ress->writeCtx); + io_job_t *writeJob = WritePool_acquireJob(ress->writeCtx); /* display last 20 characters only */ { size_t const srcFileLength = strlen(srcFileName); @@ -1998,13 +1998,13 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", srcFileName, ZSTD_getErrorName(readSizeHint)); FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); return FIO_ERROR_FRAME_DECODING; } /* Write block */ writeJob->usedBufferSize = outBuff.pos; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); frameSize += outBuff.pos; if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); @@ -2032,12 +2032,12 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, if (readSize==0) { DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", srcFileName); - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); return FIO_ERROR_FRAME_DECODING; } } } } - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return frameSize; @@ -2063,7 +2063,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName) if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return FIO_ERROR_FRAME_DECODING; - writeJob = IoPool_acquireJob(ress->writeCtx); + writeJob = WritePool_acquireJob(ress->writeCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; @@ -2089,7 +2089,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName) { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; @@ -2105,7 +2105,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName) DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); decodingError = 1; } - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } @@ -2138,7 +2138,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, return FIO_ERROR_FRAME_DECODING; } - writeJob = IoPool_acquireJob(ress->writeCtx); + writeJob = WritePool_acquireJob(ress->writeCtx); strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = (uInt)writeJob->bufferSize; strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; @@ -2166,7 +2166,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; if (decompBytes) { writeJob->usedBufferSize = decompBytes; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); outFileSize += decompBytes; strm.next_out = (Bytef*)writeJob->buffer; strm.avail_out = writeJob->bufferSize; @@ -2176,7 +2176,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in); lzma_end(&strm); - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } @@ -2198,7 +2198,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName) return FIO_ERROR_FRAME_DECODING; } - writeJob = IoPool_acquireJob(ress->writeCtx); + writeJob = WritePool_acquireJob(ress->writeCtx); /* Main Loop */ for (;nextToLoad;) { @@ -2229,7 +2229,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName) if (decodedBytes) { UTIL_HumanReadableSize_t hrs; writeJob->usedBufferSize = decodedBytes; - IoPool_enqueueAndReacquireWriteJob(&writeJob); + WritePool_enqueueAndReacquireWriteJob(&writeJob); filesize += decodedBytes; hrs = UTIL_makeHumanReadableSize(filesize); DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); @@ -2245,7 +2245,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName) } LZ4F_freeDecompressionContext(dCtx); - IoPool_releaseIoJob(writeJob); + WritePool_releaseIoJob(writeJob); WritePool_sparseWriteEnd(ress->writeCtx); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; @@ -2355,7 +2355,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, int releaseDstFile = 0; int transferMTime = 0; - if ((ress.writeCtx->file == NULL) && (prefs->testMode == 0)) { + if ((ress.writeCtx->base.file == NULL) && (prefs->testMode == 0)) { FILE *dstFile; int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ @@ -2370,7 +2370,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (dstFile==NULL) return 1; - IoPool_setFile(ress.writeCtx, dstFile); + WritePool_setFile(ress.writeCtx, dstFile); /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -2602,7 +2602,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, if (!prefs->testMode) { FILE* dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); - IoPool_setFile(ress.writeCtx, dstFile); + WritePool_setFile(ress.writeCtx, dstFile); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index 1409f8ba3da..612ab58f745 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -114,6 +114,12 @@ FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedS /* ********************************************************************** * AsyncIO functionality ************************************************************************/ + + +/* *********************************** + * General IoPool implementation + *************************************/ + static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { void *buffer; io_job_t *job; @@ -160,14 +166,13 @@ static void IoPool_init(io_pool_ctx_t *ctx, FIO_prefs_t* const prefs, POOL_funct for(i=0; i < ctx->availableJobsCount; i++) { ctx->availableJobs[i] = FIO_createIoJob(ctx, bufferSize); } - ctx->storedSkips = 0; ctx->file = NULL; } /* IoPool_releaseIoJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ -void IoPool_releaseIoJob(io_job_t *job) { +static void IoPool_releaseIoJob(io_job_t *job) { io_pool_ctx_t *ctx = (io_pool_ctx_t *) job->ctx; if(ctx->threadPool) { ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex); @@ -180,19 +185,9 @@ void IoPool_releaseIoJob(io_job_t *job) { } } -static void ReadPool_releaseAllCompletedJobs(read_pool_ctx_t* ctx) { - int i; - for(i=0; icompletedJobsCount; i++) { - io_job_t* job = (io_job_t*) ctx->completedJobs[i]; - IoPool_releaseIoJob(job); - } - ctx->completedJobsCount = 0; -} - - /* IoPool_free: * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ -void IoPool_free(io_pool_ctx_t* ctx) { +static void IoPool_destroy(io_pool_ctx_t* ctx) { int i=0; if(ctx->threadPool) { /* Make sure we finish all tasks and then free the resources */ @@ -203,18 +198,16 @@ void IoPool_free(io_pool_ctx_t* ctx) { ZSTD_pthread_mutex_destroy(&ctx->ioJobsMutex); } assert(ctx->file == NULL); - assert(ctx->storedSkips==0); for(i=0; iavailableJobsCount; i++) { io_job_t* job = (io_job_t*) ctx->availableJobs[i]; free(job->buffer); free(job); } - free(ctx); } /* IoPool_acquireJob: * Returns an available write job to be used for a future write. */ -io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { +static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { io_job_t *job; assert(ctx->file != NULL || ctx->prefs->testMode); if(ctx->threadPool) { @@ -233,55 +226,18 @@ io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { return job; } -/* ReadPool_addJobToCompleted */ -static void ReadPool_addJobToCompleted(io_job_t *job) { - read_pool_ctx_t *ctx = (read_pool_ctx_t *)job->ctx; - if(ctx->base.threadPool) - ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); - assert(ctx->completedJobsCount < MAX_IO_JOBS); - ctx->completedJobs[ctx->completedJobsCount++] = job; - if(ctx->base.threadPool) { - ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); - ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex); - } -} - -/* assuming ioJobsMutex is locked */ -static io_job_t* ReadPool_findWaitingJob(read_pool_ctx_t *ctx) { - io_job_t *job = NULL; - int i; - for (i=0; icompletedJobsCount; i++) { - job = (io_job_t *) ctx->completedJobs[i]; - if (job->offset == ctx->waitingOnOffset) { - ctx->completedJobs[i] = ctx->completedJobs[--ctx->completedJobsCount]; - return job; - } - } - return NULL; -} -/* ReadPool_getNextCompletedJob */ -static io_job_t* ReadPool_getNextCompletedJob(read_pool_ctx_t *ctx) { - io_job_t *job = NULL; - if(ctx->base.threadPool) - ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); - - job = ReadPool_findWaitingJob(ctx); - - while (!job && (ctx->base.availableJobsCount + ctx->completedJobsCount < ctx->base.totalIoJobs)) { - assert(ctx->base.threadPool != NULL); - ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->base.ioJobsMutex); - job = ReadPool_findWaitingJob(ctx); - } - - if(job) { - assert(job->offset == ctx->waitingOnOffset); - ctx->waitingOnOffset += job->usedBufferSize; - } - - if(ctx->base.threadPool) - ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex); - return job; +/* IoPool_setFile: + * Sets the destination file for future files in the pool. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. + * Also requires ending of sparse write if a previous file was used in sparse mode. */ +static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { + assert(ctx!=NULL); + /* We can change the dst file only if we have finished writing */ + if(ctx->threadPool) + POOL_joinJobs(ctx->threadPool); + assert(ctx->availableJobsCount == ctx->totalIoJobs); + ctx->file = file; } /* IoPool_enqueueJob: @@ -296,12 +252,20 @@ static void IoPool_enqueueJob(io_job_t *job) { ctx->poolFunction(job); } -/* IoPool_enqueueAndReacquireWriteJob: +/* *********************************** + * WritePool implementation + *************************************/ + +io_job_t* WritePool_acquireJob(write_pool_ctx_t *ctx) { + return IoPool_acquireJob(&ctx->base); +} + +/* WritePool_enqueueAndReacquireWriteJob: * Queues a write job for execution and acquires a new one. * After execution `job`'s pointed value would change to the newly acquired job. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ -void IoPool_enqueueAndReacquireWriteJob(io_job_t **job) { +void WritePool_enqueueAndReacquireWriteJob(io_job_t **job) { IoPool_enqueueJob(*job); *job = IoPool_acquireJob((io_pool_ctx_t *)(*job)->ctx); } @@ -309,47 +273,32 @@ void IoPool_enqueueAndReacquireWriteJob(io_job_t **job) { /* WritePool_sparseWriteEnd: * Ends sparse writes to the current file. * Blocks on completion of all current write jobs before executing. */ -void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx) { +void WritePool_sparseWriteEnd(write_pool_ctx_t *ctx) { assert(ctx != NULL); - if(ctx->threadPool) - POOL_joinJobs(ctx->threadPool); - FIO_fwriteSparseEnd(ctx->prefs, ctx->file, ctx->storedSkips); + if(ctx->base.threadPool) + POOL_joinJobs(ctx->base.threadPool); + FIO_fwriteSparseEnd(ctx->base.prefs, ctx->base.file, ctx->storedSkips); ctx->storedSkips = 0; } -/* IoPool_setFile: - * Sets the destination file for future files in the pool. - * Requires completion of all queues write jobs and release of all otherwise acquired jobs. - * Also requires ending of sparse write if a previous file was used in sparse mode. */ -void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { - assert(ctx!=NULL); - /* We can change the dst file only if we have finished writing */ - if(ctx->threadPool) - POOL_joinJobs(ctx->threadPool); +/* ReadPool_setFile */ +void WritePool_setFile(write_pool_ctx_t *ctx, FILE* file) { + IoPool_setFile(&ctx->base, file); assert(ctx->storedSkips == 0); - assert(ctx->availableJobsCount == ctx->totalIoJobs); - ctx->file = file; } -/* ReadPool_setFile */ -void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file) { - assert(ctx!=NULL); - ReadPool_releaseAllCompletedJobs(ctx); - IoPool_setFile(&ctx->base, file); - ctx->nextReadOffset = 0; - ctx->waitingOnOffset = 0; - ctx->srcBuffer = ctx->srcBufferBase; - ctx->reachedEof = 0; +void WritePool_releaseIoJob(io_job_t *job) { + IoPool_releaseIoJob(job); } /* WritePool_closeDstFile: * Ends sparse write and closes the writePool's current file and sets the file to NULL. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -int WritePool_closeDstFile(io_pool_ctx_t *ctx) { - FILE *dstFile = ctx->file; - assert(dstFile!=NULL || ctx->prefs->testMode!=0); +int WritePool_closeDstFile(write_pool_ctx_t *ctx) { + FILE *dstFile = ctx->base.file; + assert(dstFile!=NULL || ctx->base.prefs->testMode!=0); WritePool_sparseWriteEnd(ctx); - IoPool_setFile(ctx, NULL); + IoPool_setFile(&ctx->base, NULL); return fclose(dstFile); } @@ -357,20 +306,104 @@ int WritePool_closeDstFile(io_pool_ctx_t *ctx) { * Executes a write job synchronously. Can be used as a function for a thread pool. */ static void WritePool_executeWriteJob(void* opaque){ io_job_t* job = (io_job_t*) opaque; - io_pool_ctx_t* ctx = (io_pool_ctx_t*) job->ctx; - ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->prefs, ctx->storedSkips); + write_pool_ctx_t* ctx = (write_pool_ctx_t*) job->ctx; + ctx->storedSkips = FIO_fwriteSparse(job->file, job->buffer, job->usedBufferSize, ctx->base.prefs, ctx->storedSkips); IoPool_releaseIoJob(job); } /* WritePool_create: * Allocates and sets and a new write pool including its included jobs. */ -io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) { - io_pool_ctx_t* ctx = (io_pool_ctx_t*) malloc(sizeof(io_pool_ctx_t)); +write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) { + write_pool_ctx_t* ctx = (write_pool_ctx_t*) malloc(sizeof(write_pool_ctx_t)); if(!ctx) EXM_THROW(100, "Allocation error : not enough memory"); - IoPool_init(ctx, prefs, WritePool_executeWriteJob, bufferSize); + IoPool_init(&ctx->base, prefs, WritePool_executeWriteJob, bufferSize); + ctx->storedSkips = 0; return ctx; } +/* WritePool_free: */ +void WritePool_free(write_pool_ctx_t* ctx) { + /* Make sure we finish all tasks and then free the resources */ + IoPool_destroy(&ctx->base); + assert(ctx->storedSkips==0); +} + + +/* *********************************** + * ReadPool implementation + *************************************/ +static void ReadPool_releaseAllCompletedJobs(read_pool_ctx_t* ctx) { + int i; + for(i=0; icompletedJobsCount; i++) { + io_job_t* job = (io_job_t*) ctx->completedJobs[i]; + IoPool_releaseIoJob(job); + } + ctx->completedJobsCount = 0; +} + +/* ReadPool_setFile */ +void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file) { + assert(ctx!=NULL); + ReadPool_releaseAllCompletedJobs(ctx); + IoPool_setFile(&ctx->base, file); + ctx->nextReadOffset = 0; + ctx->waitingOnOffset = 0; + ctx->srcBuffer = ctx->srcBufferBase; + ctx->reachedEof = 0; +} + +/* ReadPool_addJobToCompleted */ +static void ReadPool_addJobToCompleted(io_job_t *job) { + read_pool_ctx_t *ctx = (read_pool_ctx_t *)job->ctx; + if(ctx->base.threadPool) + ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); + assert(ctx->completedJobsCount < MAX_IO_JOBS); + ctx->completedJobs[ctx->completedJobsCount++] = job; + if(ctx->base.threadPool) { + ZSTD_pthread_cond_signal(&ctx->jobCompletedCond); + ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex); + } +} + +/* assuming ioJobsMutex is locked */ +static io_job_t* ReadPool_findWaitingJob(read_pool_ctx_t *ctx) { + io_job_t *job = NULL; + int i; + for (i=0; icompletedJobsCount; i++) { + job = (io_job_t *) ctx->completedJobs[i]; + if (job->offset == ctx->waitingOnOffset) { + ctx->completedJobs[i] = ctx->completedJobs[--ctx->completedJobsCount]; + return job; + } + } + return NULL; +} + +/* ReadPool_getNextCompletedJob */ +static io_job_t* ReadPool_getNextCompletedJob(read_pool_ctx_t *ctx) { + io_job_t *job = NULL; + if(ctx->base.threadPool) + ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); + + job = ReadPool_findWaitingJob(ctx); + + while (!job && (ctx->base.availableJobsCount + ctx->completedJobsCount < ctx->base.totalIoJobs)) { + assert(ctx->base.threadPool != NULL); + ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->base.ioJobsMutex); + job = ReadPool_findWaitingJob(ctx); + } + + if(job) { + assert(job->offset == ctx->waitingOnOffset); + ctx->waitingOnOffset += job->usedBufferSize; + } + + if(ctx->base.threadPool) + ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex); + return job; +} + + /* ReadPool_executeReadJob: * Executes a read job synchronously. Can be used as a function for a thread pool. */ static void ReadPool_executeReadJob(void* opaque){ @@ -417,8 +450,9 @@ void ReadPool_free(read_pool_ctx_t* ctx) { ReadPool_releaseAllCompletedJobs(ctx); if(ctx->base.threadPool) ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); - IoPool_free(&ctx->base); + IoPool_destroy(&ctx->base); free(ctx->srcBufferBase); + free(ctx); } void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n) { @@ -435,7 +469,6 @@ static void ReadPool_enqueueRead(read_pool_ctx_t *ctx) { IoPool_enqueueJob(job); } - size_t ReadPool_readBuffer(read_pool_ctx_t *ctx, size_t n) { io_job_t *job; size_t srcBufferOffsetFromBase; @@ -474,4 +507,4 @@ void ReadPool_startReading(read_pool_ctx_t *ctx) { for (i = 0; i < ctx->base.availableJobsCount; i++) { ReadPool_enqueueRead(ctx); } -} \ No newline at end of file +} diff --git a/programs/fileio_utils.h b/programs/fileio_utils.h index deeff83ae6d..c56f2d7c2a4 100644 --- a/programs/fileio_utils.h +++ b/programs/fileio_utils.h @@ -130,7 +130,6 @@ typedef struct { /* Controls the file we currently write to, make changes only by using provided utility functions */ FILE* file; - unsigned storedSkips; // only used for write io pool /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should * only be mutated after locking the mutex */ @@ -156,6 +155,11 @@ typedef struct { ZSTD_pthread_cond_t jobCompletedCond; } read_pool_ctx_t; +typedef struct { + io_pool_ctx_t base; + unsigned storedSkips; +} write_pool_ctx_t; + typedef struct { /* These fields are automatically set and shouldn't be changed by non WritePool code. */ void *ctx; @@ -170,51 +174,49 @@ typedef struct { } io_job_t; -/* IoPool_releaseIoJob: +/* WritePool_releaseIoJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ -void IoPool_releaseIoJob(io_job_t *job); +void WritePool_releaseIoJob(io_job_t *job); -/* IoPool_free: - * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ -void IoPool_free(io_pool_ctx_t* ctx); - -/* IoPool_acquireJob: +/* WritePool_acquireJob: * Returns an available write job to be used for a future write. */ -io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx); +io_job_t* WritePool_acquireJob(write_pool_ctx_t *ctx); -/* IoPool_enqueueAndReacquireWriteJob: +/* WritePool_enqueueAndReacquireWriteJob: * Queues a write job for execution and acquires a new one. * After execution `job`'s pointed value would change to the newly acquired job. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ -void IoPool_enqueueAndReacquireWriteJob(io_job_t **job); +void WritePool_enqueueAndReacquireWriteJob(io_job_t **job); /* WritePool_sparseWriteEnd: * Ends sparse writes to the current file. * Blocks on completion of all current write jobs before executing. */ -void WritePool_sparseWriteEnd(io_pool_ctx_t* ctx); +void WritePool_sparseWriteEnd(write_pool_ctx_t *ctx); -/* IoPool_setFile: +/* WritePool_setFile: * Sets the destination file for future files in the pool. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. * Also requires ending of sparse write if a previous file was used in sparse mode. */ -void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file); +void WritePool_setFile(write_pool_ctx_t *ctx, FILE* file); /* WritePool_closeDstFile: * Ends sparse write and closes the writePool's current file and sets the file to NULL. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -int WritePool_closeDstFile(io_pool_ctx_t *ctx); +int WritePool_closeDstFile(write_pool_ctx_t *ctx); /* WritePool_create: * Allocates and sets and a new write pool including its included jobs. */ -io_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize); +write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize); + +/* WritePool_free: */ +void WritePool_free(write_pool_ctx_t* ctx); /* ReadPool_create: * Allocates and sets and a new write pool including its included jobs. */ read_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize); -/* ReadPool_free: - * Allocates and sets and a new write pool including its included jobs. */ +/* ReadPool_free: */ void ReadPool_free(read_pool_ctx_t* ctx); void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n); From 448409527b3032794b5c2b9c4c006015faf6b9a6 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 18:13:00 -0800 Subject: [PATCH 19/25] Async IO compression: Some more refactoring to write/read pools code. Also added a bunch of documentation. --- programs/fileio.c | 51 +++++++------- programs/fileio_utils.c | 147 +++++++++++++++++++++++++++------------- programs/fileio_utils.h | 58 ++++++++++++---- 3 files changed, 170 insertions(+), 86 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ed12dd6ed68..42865fd9b68 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -951,7 +951,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no while (1) { int ret; if (strm.avail_in == 0) { - ReadPool_readBuffer(ress->readCtx, ZSTD_CStreamInSize()); + ReadPool_fillBuffer(ress->readCtx, ZSTD_CStreamInSize()); if (ress->readCtx->srcBufferLoaded == 0) break; inFileSize += ress->readCtx->srcBufferLoaded; strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; @@ -1047,7 +1047,7 @@ FIO_compressLzmaFrame(cRess_t* ress, while (1) { if (strm.avail_in == 0) { - size_t const inSize = ReadPool_readBuffer(ress->readCtx, ZSTD_CStreamInSize()); + size_t const inSize = ReadPool_fillBuffer(ress->readCtx, ZSTD_CStreamInSize()); if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH; inFileSize += inSize; strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; @@ -1144,7 +1144,7 @@ FIO_compressLz4Frame(cRess_t* ress, outFileSize += headerSize; /* Read first block */ - readSize = ReadPool_readBuffer(ress->readCtx, blockSize); + readSize = ReadPool_fillBuffer(ress->readCtx, blockSize); inFileSize += readSize; /* Main Loop */ @@ -1171,7 +1171,7 @@ FIO_compressLz4Frame(cRess_t* ress, /* Read next block */ ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded); - readSize = ReadPool_readBuffer(ress->readCtx, blockSize); + readSize = ReadPool_fillBuffer(ress->readCtx, blockSize); inFileSize += readSize; } @@ -1250,7 +1250,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, do { size_t stillToFlush; /* Fill input Buffer */ - size_t const inSize = ReadPool_readBuffer(ress.readCtx, ZSTD_CStreamInSize()); + size_t const inSize = ReadPool_fillBuffer(ress.readCtx, ZSTD_CStreamInSize()); ZSTD_inBuffer inBuff = { ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 }; DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); *readsize += inSize; @@ -1538,8 +1538,8 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, stat_t statbuf; int transferMTime = 0; FILE *dstFile; - assert(ress.readCtx->base.file != NULL); - if (ress.writeCtx->base.file == NULL) { + assert(ReadPool_getFile(ress.readCtx) != NULL); + if (WritePool_getFile(ress.writeCtx) == NULL) { int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) @@ -1567,7 +1567,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, clearHandler(); DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); - if (WritePool_closeDstFile(ress.writeCtx)) { /* error closing file */ + if (WritePool_closeFile(ress.writeCtx)) { /* error closing file */ DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } @@ -1639,14 +1639,11 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile == NULL) return 1; /* srcFile could not be opened */ - ReadPool_setFile(ress.readCtx, srcFile); - ReadPool_startReading(ress.readCtx); + ReadPool_setFile(ress.readCtx, srcFile); result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); + ReadPool_closeFile(ress.readCtx); - fclose(srcFile); - // TODO: implement proper & safe close - ReadPool_setFile(ress.readCtx, NULL); if ( prefs->removeSrcFile /* --rm */ && result == 0 /* success */ && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ @@ -1808,7 +1805,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, if (!status) fCtx->nbFilesProcessed++; error |= status; } - if (WritePool_closeDstFile(ress.writeCtx)) + if (WritePool_closeFile(ress.writeCtx)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", strerror(errno), outFileName); } @@ -1913,11 +1910,11 @@ static int FIO_passThrough(dRess_t *ress) { size_t const blockSize = MIN(MIN(64 KB, ZSTD_DStreamInSize()), ZSTD_DStreamOutSize()); io_job_t *writeJob = WritePool_acquireJob(ress->writeCtx); - ReadPool_readBuffer(ress->readCtx, blockSize); + ReadPool_fillBuffer(ress->readCtx, blockSize); while(ress->readCtx->srcBufferLoaded) { size_t writeSize; - ReadPool_readBuffer(ress->readCtx, blockSize); + ReadPool_fillBuffer(ress->readCtx, blockSize); writeSize = MIN(blockSize, ress->readCtx->srcBufferLoaded); assert(writeSize <= writeJob->bufferSize); memcpy(writeJob->buffer, ress->readCtx->srcBuffer, writeSize); @@ -1985,7 +1982,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ if (ress->readCtx->srcBufferLoaded < ZSTD_FRAMEHEADERSIZE_MAX) - ReadPool_readBuffer(ress->readCtx, ZSTD_FRAMEHEADERSIZE_MAX); + ReadPool_fillBuffer(ress->readCtx, ZSTD_FRAMEHEADERSIZE_MAX); /* Main decompression Loop */ while (1) { @@ -2028,7 +2025,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, /* Fill input buffer */ { size_t const toDecode = MIN(readSizeHint, ZSTD_DStreamInSize()); /* support large skippable frames */ if (ress->readCtx->srcBufferLoaded < toDecode) { - size_t const readSize = ReadPool_readBuffer(ress->readCtx, toDecode); + size_t const readSize = ReadPool_fillBuffer(ress->readCtx, toDecode); if (readSize==0) { DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", srcFileName); @@ -2072,7 +2069,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName) for ( ; ; ) { int ret; if (strm.avail_in == 0) { - ReadPool_consumeAndReadAll(ress->readCtx); + ReadPool_consumeAndRefill(ress->readCtx); if (ress->readCtx->srcBufferLoaded == 0) flush = Z_FINISH; strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; @@ -2147,7 +2144,7 @@ FIO_decompressLzmaFrame(dRess_t* ress, for ( ; ; ) { lzma_ret ret; if (strm.avail_in == 0) { - ReadPool_consumeAndReadAll(ress->readCtx); + ReadPool_consumeAndRefill(ress->readCtx); if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH; strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; strm.avail_in = ress->readCtx->srcBufferLoaded; @@ -2207,7 +2204,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName) int fullBufferDecoded = 0; /* Read input */ - ReadPool_readBuffer(ress->readCtx, nextToLoad); + ReadPool_fillBuffer(ress->readCtx, nextToLoad); if(!ress->readCtx->srcBufferLoaded) break; /* reached end of file */ while ((pos < ress->readCtx->srcBufferLoaded) || fullBufferDecoded) { /* still to read, or still to flush */ @@ -2272,7 +2269,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, /* check magic number -> version */ size_t const toRead = 4; const BYTE* buf; - ReadPool_readBuffer(ress.readCtx, toRead); + ReadPool_fillBuffer(ress.readCtx, toRead); buf = (const BYTE*)ress.readCtx->srcBuffer; if (ress.readCtx->srcBufferLoaded==0) { if (readSomething==0) { /* srcFile is empty (which is invalid) */ @@ -2339,8 +2336,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, } /** FIO_decompressDstFile() : - open `dstFileName`, - or pass-through if ress.writeCtx->file is already != 0, + open `dstFileName`, or pass-through if writeCtx's file is already != 0, then start decompression process (FIO_decompressFrames()). @return : 0 : OK 1 : operation aborted @@ -2355,7 +2351,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, int releaseDstFile = 0; int transferMTime = 0; - if ((ress.writeCtx->base.file == NULL) && (prefs->testMode == 0)) { + if ((WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) { FILE *dstFile; int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ @@ -2383,7 +2379,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, if (releaseDstFile) { clearHandler(); - if (WritePool_closeDstFile(ress.writeCtx)) { + if (WritePool_closeFile(ress.writeCtx)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result = 1; } @@ -2421,7 +2417,6 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile==NULL) return 1; ReadPool_setFile(ress.readCtx, srcFile); - ReadPool_startReading(ress.readCtx); result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName); @@ -2609,7 +2604,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, if (!status) fCtx->nbFilesProcessed++; error |= status; } - if ((!prefs->testMode) && (WritePool_closeDstFile(ress.writeCtx))) + if ((!prefs->testMode) && (WritePool_closeFile(ress.writeCtx))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index 612ab58f745..646d8c07a6c 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -115,12 +115,11 @@ FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedS * AsyncIO functionality ************************************************************************/ - /* *********************************** * General IoPool implementation *************************************/ -static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { +static io_job_t *IoPool_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { void *buffer; io_job_t *job; job = (io_job_t*) malloc(sizeof(io_job_t)); @@ -137,10 +136,10 @@ static io_job_t *FIO_createIoJob(io_pool_ctx_t *ctx, size_t bufferSize) { } -/* IO_createThreadPool: +/* IoPool_createThreadPool: * Creates a thread pool and a mutex for threaded IO pool. * Displays warning if asyncio is requested but MT isn't available. */ -static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { +static void IoPool_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { ctx->threadPool = NULL; if(prefs->asyncIO) { if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL)) @@ -158,13 +157,13 @@ static void IO_createThreadPool(io_pool_ctx_t *ctx, const FIO_prefs_t *prefs) { * Allocates and sets and a new write pool including its included availableJobs. */ static void IoPool_init(io_pool_ctx_t *ctx, FIO_prefs_t* const prefs, POOL_function poolFunction, size_t bufferSize) { int i; - IO_createThreadPool(ctx, prefs); + IoPool_createThreadPool(ctx, prefs); ctx->prefs = prefs; ctx->poolFunction = poolFunction; ctx->totalIoJobs = ctx->threadPool ? MAX_IO_JOBS : 1; ctx->availableJobsCount = ctx->totalIoJobs; for(i=0; i < ctx->availableJobsCount; i++) { - ctx->availableJobs[i] = FIO_createIoJob(ctx, bufferSize); + ctx->availableJobs[i] = IoPool_createIoJob(ctx, bufferSize); } ctx->file = NULL; } @@ -185,13 +184,20 @@ static void IoPool_releaseIoJob(io_job_t *job) { } } +/* IoPool_join: + * Waits for all tasks in the pool to finish executing. */ +static void IoPool_join(io_pool_ctx_t* ctx) { + if(ctx->threadPool) + POOL_joinJobs(ctx->threadPool); +} + /* IoPool_free: * Release a previously allocated write thread pool. Makes sure all takss are done and released. */ static void IoPool_destroy(io_pool_ctx_t* ctx) { - int i=0; + int i; if(ctx->threadPool) { /* Make sure we finish all tasks and then free the resources */ - POOL_joinJobs(ctx->threadPool); + IoPool_join(ctx); /* Make sure we are not leaking availableJobs */ assert(ctx->availableJobsCount == ctx->totalIoJobs); POOL_free(ctx->threadPool); @@ -206,7 +212,7 @@ static void IoPool_destroy(io_pool_ctx_t* ctx) { } /* IoPool_acquireJob: - * Returns an available write job to be used for a future write. */ + * Returns an available io job to be used for a future io. */ static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { io_job_t *job; assert(ctx->file != NULL || ctx->prefs->testMode); @@ -233,16 +239,17 @@ static io_job_t* IoPool_acquireJob(io_pool_ctx_t *ctx) { * Also requires ending of sparse write if a previous file was used in sparse mode. */ static void IoPool_setFile(io_pool_ctx_t *ctx, FILE* file) { assert(ctx!=NULL); - /* We can change the dst file only if we have finished writing */ - if(ctx->threadPool) - POOL_joinJobs(ctx->threadPool); + IoPool_join(ctx); assert(ctx->availableJobsCount == ctx->totalIoJobs); ctx->file = file; } +static FILE* IoPool_getFile(io_pool_ctx_t *ctx) { + return ctx->file; +} + /* IoPool_enqueueJob: - * Queues a write job for execution. - * Make sure to set `usedBufferSize` to the wanted length before call. + * Enqueues an io job for execution. * The queued job shouldn't be used directly after queueing it. */ static void IoPool_enqueueJob(io_job_t *job) { io_pool_ctx_t* ctx = (io_pool_ctx_t *)job->ctx; @@ -256,6 +263,8 @@ static void IoPool_enqueueJob(io_job_t *job) { * WritePool implementation *************************************/ +/* WritePool_acquireJob: + * Returns an available write job to be used for a future write. */ io_job_t* WritePool_acquireJob(write_pool_ctx_t *ctx) { return IoPool_acquireJob(&ctx->base); } @@ -281,20 +290,25 @@ void WritePool_sparseWriteEnd(write_pool_ctx_t *ctx) { ctx->storedSkips = 0; } -/* ReadPool_setFile */ +/* WritePool_setFile: + * Sets the destination file for future writes in the pool. + * Requires completion of all queues write jobs and release of all otherwise acquired jobs. + * Also requires ending of sparse write if a previous file was used in sparse mode. */ void WritePool_setFile(write_pool_ctx_t *ctx, FILE* file) { IoPool_setFile(&ctx->base, file); assert(ctx->storedSkips == 0); } +/* WritePool_releaseIoJob: + * Releases an acquired job back to the pool. Doesn't execute the job. */ void WritePool_releaseIoJob(io_job_t *job) { IoPool_releaseIoJob(job); } -/* WritePool_closeDstFile: +/* WritePool_closeFile: * Ends sparse write and closes the writePool's current file and sets the file to NULL. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -int WritePool_closeDstFile(write_pool_ctx_t *ctx) { +int WritePool_closeFile(write_pool_ctx_t *ctx) { FILE *dstFile = ctx->base.file; assert(dstFile!=NULL || ctx->base.prefs->testMode!=0); WritePool_sparseWriteEnd(ctx); @@ -321,9 +335,12 @@ write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize) return ctx; } -/* WritePool_free: */ +/* WritePool_free: + * Frees and releases a writePool and its resources. Closes destination file if needs to. */ void WritePool_free(write_pool_ctx_t* ctx) { /* Make sure we finish all tasks and then free the resources */ + if(WritePool_getFile(ctx)) + WritePool_closeFile(ctx); IoPool_destroy(&ctx->base); assert(ctx->storedSkips==0); } @@ -341,17 +358,6 @@ static void ReadPool_releaseAllCompletedJobs(read_pool_ctx_t* ctx) { ctx->completedJobsCount = 0; } -/* ReadPool_setFile */ -void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file) { - assert(ctx!=NULL); - ReadPool_releaseAllCompletedJobs(ctx); - IoPool_setFile(&ctx->base, file); - ctx->nextReadOffset = 0; - ctx->waitingOnOffset = 0; - ctx->srcBuffer = ctx->srcBufferBase; - ctx->reachedEof = 0; -} - /* ReadPool_addJobToCompleted */ static void ReadPool_addJobToCompleted(io_job_t *job) { read_pool_ctx_t *ctx = (read_pool_ctx_t *)job->ctx; @@ -426,8 +432,44 @@ static void ReadPool_executeReadJob(void* opaque){ ReadPool_addJobToCompleted(job); } +static void ReadPool_enqueueRead(read_pool_ctx_t *ctx) { + io_job_t *job = IoPool_acquireJob(&ctx->base); + job->offset = ctx->nextReadOffset; + ctx->nextReadOffset += job->bufferSize; + IoPool_enqueueJob(job); +} + +static void ReadPool_startReading(read_pool_ctx_t *ctx) { + int i; + for (i = 0; i < ctx->base.availableJobsCount; i++) { + ReadPool_enqueueRead(ctx); + } +} + +/* ReadPool_setFile: + * Sets the source file for future read in the pool. Initiates reading immediately if file is not NULL. + * Waits for all current enqueued tasks to complete if a previous file was set. */ +void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file) { + assert(ctx!=NULL); + IoPool_join(&ctx->base); + ReadPool_releaseAllCompletedJobs(ctx); + IoPool_setFile(&ctx->base, file); + ctx->nextReadOffset = 0; + ctx->waitingOnOffset = 0; + ctx->srcBuffer = ctx->srcBufferBase; + ctx->reachedEof = 0; + if(file != NULL) + ReadPool_startReading(ctx); +} + +FILE* WritePool_getFile(write_pool_ctx_t *ctx) { + return IoPool_getFile(&ctx->base); +} + /* ReadPool_create: - * Allocates and sets and a new write pool including its included jobs. */ + * Allocates and sets and a new readPool including its included jobs. + * bufferSize should be set to the maximal buffer we want to read at a time, will also be used + * as our basic read size. */ read_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { read_pool_ctx_t* ctx = (read_pool_ctx_t*) malloc(sizeof(read_pool_ctx_t)); if(!ctx) EXM_THROW(100, "Allocation error : not enough memory"); @@ -446,8 +488,11 @@ read_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize) { return ctx; } +/* ReadPool_free: + * Frees and releases a readPool and its resources. Closes source file. */ void ReadPool_free(read_pool_ctx_t* ctx) { - ReadPool_releaseAllCompletedJobs(ctx); + if(ReadPool_getFile(ctx)) + ReadPool_closeFile(ctx); if(ctx->base.threadPool) ZSTD_pthread_cond_destroy(&ctx->jobCompletedCond); IoPool_destroy(&ctx->base); @@ -455,6 +500,8 @@ void ReadPool_free(read_pool_ctx_t* ctx) { free(ctx); } +/* ReadPool_consumeBytes: + * Consumes byes from srcBuffer's beginning and updates srcBufferLoaded accordingly. */ void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n) { assert(n <= ctx->srcBufferLoaded); assert(ctx->srcBuffer + n <= ctx->srcBufferBase + ctx->srcBufferBaseSize); @@ -462,18 +509,17 @@ void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n) { ctx->srcBuffer += n; } -static void ReadPool_enqueueRead(read_pool_ctx_t *ctx) { - io_job_t *job = IoPool_acquireJob(&ctx->base); - job->offset = ctx->nextReadOffset; - ctx->nextReadOffset += job->bufferSize; - IoPool_enqueueJob(job); -} - -size_t ReadPool_readBuffer(read_pool_ctx_t *ctx, size_t n) { +/* ReadPool_fillBuffer: + * Makes sure buffer has at least n bytes loaded (as long as n is not bigger than the initalized bufferSize). + * Returns if srcBuffer has at least n bytes loaded or if we've reached the end of the file. + * Return value is the number of bytes added to the buffer. + * Note that srcBuffer might have up to 2 times bufferSize bytes. */ +size_t ReadPool_fillBuffer(read_pool_ctx_t *ctx, size_t n) { io_job_t *job; size_t srcBufferOffsetFromBase; size_t srcBufferRemainingSpace; size_t bytesRead = 0; + assert(n <= ctx->srcBufferBaseSize/2); while (ctx->srcBufferLoaded < n) { job = ReadPool_getNextCompletedJob(ctx); if(job == NULL) @@ -497,14 +543,23 @@ size_t ReadPool_readBuffer(read_pool_ctx_t *ctx, size_t n) { return bytesRead; } -size_t ReadPool_consumeAndReadAll(read_pool_ctx_t *ctx) { +/* ReadPool_consumeAndRefill: + * Consumes the current buffer and refills it with bufferSize bytes. */ +size_t ReadPool_consumeAndRefill(read_pool_ctx_t *ctx) { ReadPool_consumeBytes(ctx, ctx->srcBufferLoaded); - return ReadPool_readBuffer(ctx, ZSTD_DStreamInSize()); + return ReadPool_fillBuffer(ctx, ctx->srcBufferBaseSize/2); } -void ReadPool_startReading(read_pool_ctx_t *ctx) { - int i; - for (i = 0; i < ctx->base.availableJobsCount; i++) { - ReadPool_enqueueRead(ctx); - } +/* ReadPool_getFile: + * Returns the current file set for the read pool. */ +FILE* ReadPool_getFile(read_pool_ctx_t *ctx) { + return IoPool_getFile(&ctx->base); +} + +/* ReadPool_closeFile: + * Closes the current set file. Waits for all current enqueued tasks to complete and resets state. */ +int ReadPool_closeFile(read_pool_ctx_t *ctx) { + FILE* file = ReadPool_getFile(ctx); + ReadPool_setFile(ctx, NULL); + return fclose(file); } diff --git a/programs/fileio_utils.h b/programs/fileio_utils.h index c56f2d7c2a4..1ab24ad3339 100644 --- a/programs/fileio_utils.h +++ b/programs/fileio_utils.h @@ -131,7 +131,7 @@ typedef struct { /* Controls the file we currently write to, make changes only by using provided utility functions */ FILE* file; - /* The jobs and availableJobsCount fields are accessed by both the main and writer threads and should + /* The jobs and availableJobsCount fields are accessed by both the main and worker threads and should * only be mutated after locking the mutex */ ZSTD_pthread_mutex_t ioJobsMutex; void* availableJobs[MAX_IO_JOBS]; @@ -141,15 +141,22 @@ typedef struct { typedef struct { io_pool_ctx_t base; + /* State regarding the currently read file */ int reachedEof; U64 nextReadOffset; U64 waitingOnOffset; + /* Bases buffer, shouldn't be accessed from outside ot utility functions. */ U8 *srcBufferBase; size_t srcBufferBaseSize; + + /* Read buffer can be used by consumer code, take care when copying this pointer aside as it might + * change when consuming / refilling buffer. */ U8 *srcBuffer; size_t srcBufferLoaded; + /* We need to know what tasks completed so we can use their buffers when their time comes. + * Should only be accessed after locking base.ioJobsMutex . */ void* completedJobs[MAX_IO_JOBS]; int completedJobsCount; ZSTD_pthread_cond_t jobCompletedCond; @@ -183,7 +190,7 @@ void WritePool_releaseIoJob(io_job_t *job); io_job_t* WritePool_acquireJob(write_pool_ctx_t *ctx); /* WritePool_enqueueAndReacquireWriteJob: - * Queues a write job for execution and acquires a new one. + * Enqueues a write job for execution and acquires a new one. * After execution `job`'s pointed value would change to the newly acquired job. * Make sure to set `usedBufferSize` to the wanted length before call. * The queued job shouldn't be used directly after queueing it. */ @@ -195,38 +202,65 @@ void WritePool_enqueueAndReacquireWriteJob(io_job_t **job); void WritePool_sparseWriteEnd(write_pool_ctx_t *ctx); /* WritePool_setFile: - * Sets the destination file for future files in the pool. + * Sets the destination file for future writes in the pool. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. * Also requires ending of sparse write if a previous file was used in sparse mode. */ void WritePool_setFile(write_pool_ctx_t *ctx, FILE* file); -/* WritePool_closeDstFile: +/* WritePool_getFile: + * Returns the file the writePool is currently set to write to. */ +FILE* WritePool_getFile(write_pool_ctx_t *ctx); + +/* WritePool_closeFile: * Ends sparse write and closes the writePool's current file and sets the file to NULL. * Requires completion of all queues write jobs and release of all otherwise acquired jobs. */ -int WritePool_closeDstFile(write_pool_ctx_t *ctx); +int WritePool_closeFile(write_pool_ctx_t *ctx); /* WritePool_create: - * Allocates and sets and a new write pool including its included jobs. */ + * Allocates and sets and a new write pool including its included jobs. + * bufferSize should be set to the maximal buffer we want to write to at a time. */ write_pool_ctx_t* WritePool_create(FIO_prefs_t* const prefs, size_t bufferSize); -/* WritePool_free: */ +/* WritePool_free: + * Frees and releases a writePool and its resources. Closes destination file. */ void WritePool_free(write_pool_ctx_t* ctx); /* ReadPool_create: - * Allocates and sets and a new write pool including its included jobs. */ + * Allocates and sets and a new readPool including its included jobs. + * bufferSize should be set to the maximal buffer we want to read at a time, will also be used + * as our basic read size. */ read_pool_ctx_t* ReadPool_create(FIO_prefs_t* const prefs, size_t bufferSize); -/* ReadPool_free: */ +/* ReadPool_free: + * Frees and releases a readPool and its resources. Closes source file. */ void ReadPool_free(read_pool_ctx_t* ctx); +/* ReadPool_consumeBytes: + * Consumes byes from srcBuffer's beginning and updates srcBufferLoaded accordingly. */ void ReadPool_consumeBytes(read_pool_ctx_t *ctx, size_t n); -size_t ReadPool_readBuffer(read_pool_ctx_t *ctx, size_t n); +/* ReadPool_fillBuffer: + * Makes sure buffer has at least n bytes loaded (as long as n is not bigger than the initalized bufferSize). + * Returns if srcBuffer has at least n bytes loaded or if we've reached the end of the file. + * Return value is the number of bytes added to the buffer. + * Note that srcBuffer might have up to 2 times bufferSize bytes. */ +size_t ReadPool_fillBuffer(read_pool_ctx_t *ctx, size_t n); -size_t ReadPool_consumeAndReadAll(read_pool_ctx_t *ctx); +/* ReadPool_consumeAndRefill: + * Consumes the current buffer and refills it with bufferSize bytes. */ +size_t ReadPool_consumeAndRefill(read_pool_ctx_t *ctx); +/* ReadPool_setFile: + * Sets the source file for future read in the pool. Initiates reading immediately if file is not NULL. + * Waits for all current enqueued tasks to complete if a previous file was set. */ void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file); -void ReadPool_startReading(read_pool_ctx_t *ctx); +/* ReadPool_getFile: + * Returns the current file set for the read pool. */ +FILE* ReadPool_getFile(read_pool_ctx_t *ctx); + +/* ReadPool_closeFile: + * Closes the current set file. Waits for all current enqueued tasks to complete and resets state. */ +int ReadPool_closeFile(read_pool_ctx_t *ctx); #endif /* FILEIO_UTILS_HEADER */ \ No newline at end of file From 8502b9d10e4287331790d821fb9f8e81a782ca2b Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 19:44:17 -0800 Subject: [PATCH 20/25] Async IO compression: bug fixes --- programs/fileio.c | 2 +- programs/fileio_utils.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 42865fd9b68..264792da9f3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -848,7 +848,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ ress.writeCtx = WritePool_create(prefs, ZSTD_CStreamOutSize()); - ress.readCtx = ReadPool_create(prefs, ZSTD_DStreamInSize()); + ress.readCtx = ReadPool_create(prefs, ZSTD_CStreamInSize()); /* Advanced parameters, including dictionary */ if (dictFileName && (ress.dictBuffer==NULL)) diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index 646d8c07a6c..94d85b6251b 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -2,6 +2,7 @@ #include /* malloc, free */ #include #include /* errno */ +#include /* strerror */ #include "fileio_utils.h" @@ -343,6 +344,7 @@ void WritePool_free(write_pool_ctx_t* ctx) { WritePool_closeFile(ctx); IoPool_destroy(&ctx->base); assert(ctx->storedSkips==0); + free(ctx); } From c6aea5c1f6860c2100c5a818825f60a51717ab44 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 14 Jan 2022 21:55:18 -0800 Subject: [PATCH 21/25] Async IO compression: fix build failures --- programs/fileio_utils.c | 8 +++++++- programs/fileio_utils.h | 14 +++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index 94d85b6251b..7a6f9e821a5 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -1,8 +1,14 @@ +#include "platform.h" #include /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include #include /* errno */ -#include /* strerror */ +#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ + +#if defined (_MSC_VER) +# include +# include +#endif #include "fileio_utils.h" diff --git a/programs/fileio_utils.h b/programs/fileio_utils.h index 1ab24ad3339..4d45c66b4a0 100644 --- a/programs/fileio_utils.h +++ b/programs/fileio_utils.h @@ -1,5 +1,9 @@ -#ifndef FILEIO_UTILS_HEADER -#define FILEIO_UTILS_HEADER +#ifndef FILEIO_UTILS_H_MODULE +#define FILEIO_UTILS_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif #include "../lib/common/mem.h" /* U32, U64 */ #include "timefn.h" @@ -263,4 +267,8 @@ FILE* ReadPool_getFile(read_pool_ctx_t *ctx); * Closes the current set file. Waits for all current enqueued tasks to complete and resets state. */ int ReadPool_closeFile(read_pool_ctx_t *ctx); -#endif /* FILEIO_UTILS_HEADER */ \ No newline at end of file +#if defined (__cplusplus) +} +#endif + +#endif /* FILEIO_UTILS_H_MODULE */ From b7b9c0682c890681d2a73e70e0be38706837ee26 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Tue, 18 Jan 2022 10:48:48 -0800 Subject: [PATCH 22/25] Async IO compression: test string change --- programs/fileio_utils.c | 2 +- tests/playTests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index 7a6f9e821a5..a45d6aee04e 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -530,7 +530,7 @@ size_t ReadPool_fillBuffer(read_pool_ctx_t *ctx, size_t n) { assert(n <= ctx->srcBufferBaseSize/2); while (ctx->srcBufferLoaded < n) { job = ReadPool_getNextCompletedJob(ctx); - if(job == NULL) + if(job == NULL) break; srcBufferOffsetFromBase = ctx->srcBuffer - ctx->srcBufferBase; srcBufferRemainingSpace = ctx->srcBufferBaseSize - (srcBufferOffsetFromBase + ctx->srcBufferLoaded); diff --git a/tests/playTests.sh b/tests/playTests.sh index 7048c7c4d58..b795823d782 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1575,7 +1575,7 @@ elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then exit 1 fi -println "\n===> zstd asyncio decompression tests " +println "\n===> zstd asyncio tests " addFrame() { datagen -g2M -s$2 >> tmp_uncompressed From 8256c07bac3262b8388effa6a673bb641ec8652f Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Tue, 18 Jan 2022 10:59:55 -0800 Subject: [PATCH 23/25] Async IO compression: Small changes --- programs/fileio_utils.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index a45d6aee04e..c000215c3b1 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -366,7 +366,6 @@ static void ReadPool_releaseAllCompletedJobs(read_pool_ctx_t* ctx) { ctx->completedJobsCount = 0; } -/* ReadPool_addJobToCompleted */ static void ReadPool_addJobToCompleted(io_job_t *job) { read_pool_ctx_t *ctx = (read_pool_ctx_t *)job->ctx; if(ctx->base.threadPool) @@ -379,8 +378,11 @@ static void ReadPool_addJobToCompleted(io_job_t *job) { } } -/* assuming ioJobsMutex is locked */ -static io_job_t* ReadPool_findWaitingJob(read_pool_ctx_t *ctx) { +/* ReadPool_findNextWaitingOffsetCompletedJob: + * Looks through the completed jobs for a job matching the waitingOnOffset and returns it, + * if job wasn't found returns NULL. + * IMPORTANT: assumes ioJobsMutex is locked. */ +static io_job_t* ReadPool_findNextWaitingOffsetCompletedJob(read_pool_ctx_t *ctx) { io_job_t *job = NULL; int i; for (i=0; icompletedJobsCount; i++) { @@ -393,18 +395,21 @@ static io_job_t* ReadPool_findWaitingJob(read_pool_ctx_t *ctx) { return NULL; } -/* ReadPool_getNextCompletedJob */ +/* ReadPool_getNextCompletedJob: + * Returns a completed io_job_t for the next read in line based on waitingOnOffset and advances waitingOnOffset. + * Would block. */ static io_job_t* ReadPool_getNextCompletedJob(read_pool_ctx_t *ctx) { io_job_t *job = NULL; if(ctx->base.threadPool) ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex); - job = ReadPool_findWaitingJob(ctx); + job = ReadPool_findNextWaitingOffsetCompletedJob(ctx); + /* As long as we didn't find the job matching the next read, and we have some reads in flight continue waiting */ while (!job && (ctx->base.availableJobsCount + ctx->completedJobsCount < ctx->base.totalIoJobs)) { - assert(ctx->base.threadPool != NULL); + assert(ctx->base.threadPool != NULL); /* we shouldn't be here if we work in sync mode */ ZSTD_pthread_cond_wait(&ctx->jobCompletedCond, &ctx->base.ioJobsMutex); - job = ReadPool_findWaitingJob(ctx); + job = ReadPool_findNextWaitingOffsetCompletedJob(ctx); } if(job) { From 990fc70c79dad149f8a430d1d91acd3156f38143 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Tue, 18 Jan 2022 14:28:41 -0800 Subject: [PATCH 24/25] Async IO compression: Small changes --- programs/fileio_utils.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/programs/fileio_utils.c b/programs/fileio_utils.c index c000215c3b1..76a6d7d9ac8 100644 --- a/programs/fileio_utils.c +++ b/programs/fileio_utils.c @@ -306,6 +306,12 @@ void WritePool_setFile(write_pool_ctx_t *ctx, FILE* file) { assert(ctx->storedSkips == 0); } +/* WritePool_getFile: + * Returns the file the writePool is currently set to write to. */ +FILE* WritePool_getFile(write_pool_ctx_t *ctx) { + return IoPool_getFile(&ctx->base); +} + /* WritePool_releaseIoJob: * Releases an acquired job back to the pool. Doesn't execute the job. */ void WritePool_releaseIoJob(io_job_t *job) { @@ -475,10 +481,6 @@ void ReadPool_setFile(read_pool_ctx_t *ctx, FILE* file) { ReadPool_startReading(ctx); } -FILE* WritePool_getFile(write_pool_ctx_t *ctx) { - return IoPool_getFile(&ctx->base); -} - /* ReadPool_create: * Allocates and sets and a new readPool including its included jobs. * bufferSize should be set to the maximal buffer we want to read at a time, will also be used From e7f3afaf54a5f2b630ec890c094ce1bcfd93e891 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik Date: Fri, 21 Jan 2022 11:51:10 -0800 Subject: [PATCH 25/25] Async IO compression: Some refactors for better clarity and code locations --- build/VS2008/zstd/zstd.vcproj | 2 +- build/VS2010/zstd/zstd.vcxproj | 2 +- build/cmake/programs/CMakeLists.txt | 4 +- build/meson/programs/meson.build | 4 +- contrib/VS2005/zstd/zstd.vcproj | 2 +- programs/Makefile | 8 +- programs/fileio.c | 21 +++- programs/{fileio_utils.c => fileio_asycio.c} | 7 +- programs/{fileio_utils.h => fileio_asycio.h} | 117 +------------------ programs/fileio_common.h | 108 +++++++++++++++++ 10 files changed, 144 insertions(+), 131 deletions(-) rename programs/{fileio_utils.c => fileio_asycio.c} (99%) rename programs/{fileio_utils.h => fileio_asycio.h} (57%) create mode 100644 programs/fileio_common.h diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 829f7e6e305..91f2bda536c 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -385,7 +385,7 @@ > - + diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 9e8eb35f387..28b1e1d166b 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -32,7 +32,7 @@ if (MSVC) set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc) endif () -add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_utils.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources}) +add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources}) target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET}) if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") target_link_libraries(zstd rt) @@ -75,7 +75,7 @@ if (UNIX) ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}") - add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_utils.c) + add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c) target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET}) set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE") endif () diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index f599b977d71..5ccd679a167 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -14,7 +14,7 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/util.c'), join_paths(zstd_rootdir, 'programs/timefn.c'), join_paths(zstd_rootdir, 'programs/fileio.c'), - join_paths(zstd_rootdir, 'programs/fileio_utils.c'), + join_paths(zstd_rootdir, 'programs/fileio_asyncio.c'), join_paths(zstd_rootdir, 'programs/benchfn.c'), join_paths(zstd_rootdir, 'programs/benchzstd.c'), join_paths(zstd_rootdir, 'programs/datagen.c'), @@ -81,7 +81,7 @@ zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/timefn.c'), join_paths(zstd_rootdir, 'programs/util.c'), join_paths(zstd_rootdir, 'programs/fileio.c'), - join_paths(zstd_rootdir, 'programs/fileio_utils.c'), + join_paths(zstd_rootdir, 'programs/fileio_asyncio.c'), join_paths(zstd_rootdir, 'lib/common/pool.c'), join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), join_paths(zstd_rootdir, 'lib/common/error_private.c')] diff --git a/contrib/VS2005/zstd/zstd.vcproj b/contrib/VS2005/zstd/zstd.vcproj index f5fd6dd88ed..e37ebee3911 100644 --- a/contrib/VS2005/zstd/zstd.vcproj +++ b/contrib/VS2005/zstd/zstd.vcproj @@ -364,7 +364,7 @@ > /* malloc, free */ #include #include /* errno */ -#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ #if defined (_MSC_VER) # include # include #endif -#include "fileio_utils.h" - -FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; -UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; +#include "fileio_asycio.h" +#include "fileio_common.h" /* ********************************************************************** * Sparse write diff --git a/programs/fileio_utils.h b/programs/fileio_asycio.h similarity index 57% rename from programs/fileio_utils.h rename to programs/fileio_asycio.h index 4d45c66b4a0..88d55c0ffb9 100644 --- a/programs/fileio_utils.h +++ b/programs/fileio_asycio.h @@ -1,125 +1,14 @@ -#ifndef FILEIO_UTILS_H_MODULE -#define FILEIO_UTILS_H_MODULE +#ifndef ZSTD_FILEIO_ASYNCIO_H +#define ZSTD_FILEIO_ASYNCIO_H #if defined (__cplusplus) extern "C" { #endif #include "../lib/common/mem.h" /* U32, U64 */ -#include "timefn.h" #include "fileio_types.h" #include "platform.h" #include "util.h" - -/*-************************************* -* Constants -***************************************/ -#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ -#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ - -/* Default file permissions 0666 (modulated by umask) */ -#if !defined(_WIN32) -/* These macros aren't defined on windows. */ -#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) -#else -#define DEFAULT_FILE_PERMISSIONS (0666) -#endif - -/*-************************************* -* Macros -***************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) -#undef MAX -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -extern FIO_display_prefs_t g_display_prefs; - -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -extern UTIL_time_t g_displayClock; - -#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) -#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } -#define DISPLAYUPDATE(l, ...) { \ - if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ - if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ - DELAY_NEXT_UPDATE(); \ - DISPLAY(__VA_ARGS__); \ - if (g_display_prefs.displayLevel>=4) fflush(stderr); \ - } } } - -#undef MIN /* in case it would be already defined */ -#define MIN(a,b) ((a) < (b) ? (a) : (b)) - - -#define EXM_THROW(error, ...) \ -{ \ - DISPLAYLEVEL(1, "zstd: "); \ - DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - exit(error); \ -} - -#define CHECK_V(v, f) \ - v = f; \ - if (ZSTD_isError(v)) { \ - DISPLAYLEVEL(5, "%s \n", #f); \ - EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ - } -#define CHECK(f) { size_t err; CHECK_V(err, f); } - - -/* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW */ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -# define LONG_SEEK _fseeki64 -# define LONG_TELL _ftelli64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define LONG_SEEK fseeko -# define LONG_TELL ftello -#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) -# define LONG_SEEK fseeko64 -# define LONG_TELL ftello64 -#elif defined(_WIN32) && !defined(__DJGPP__) -# include - static int LONG_SEEK(FILE* file, __int64 offset, int origin) { - LARGE_INTEGER off; - DWORD method; - off.QuadPart = offset; - if (origin == SEEK_END) - method = FILE_END; - else if (origin == SEEK_CUR) - method = FILE_CURRENT; - else - method = FILE_BEGIN; - - if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) - return 0; - else - return -1; - } - static __int64 LONG_TELL(FILE* file) { - LARGE_INTEGER off, newOff; - off.QuadPart = 0; - newOff.QuadPart = 0; - SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); - return newOff.QuadPart; - } -#else -# define LONG_SEEK fseek -# define LONG_TELL ftell -#endif - - -/* ********************************************************************** - * AsyncIO functionality - ************************************************************************/ #include "../lib/common/pool.h" #include "../lib/common/threading.h" @@ -271,4 +160,4 @@ int ReadPool_closeFile(read_pool_ctx_t *ctx); } #endif -#endif /* FILEIO_UTILS_H_MODULE */ +#endif /* ZSTD_FILEIO_ASYNCIO_H */ diff --git a/programs/fileio_common.h b/programs/fileio_common.h new file mode 100644 index 00000000000..ba01eee5f8b --- /dev/null +++ b/programs/fileio_common.h @@ -0,0 +1,108 @@ +#ifndef ZSTD_FILEIO_COMMON_H +#define ZSTD_FILEIO_COMMON_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../lib/common/mem.h" /* U32, U64 */ +#include "timefn.h" +#include "fileio_types.h" +#include "platform.h" +#include "util.h" + +/*-************************************* +* Macros +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) +#undef MAX +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +extern FIO_display_prefs_t g_display_prefs; + +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +extern UTIL_time_t g_displayClock; + +#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } +#define DISPLAYUPDATE(l, ...) { \ + if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ + if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ + DELAY_NEXT_UPDATE(); \ + DISPLAY(__VA_ARGS__); \ + if (g_display_prefs.displayLevel>=4) fflush(stderr); \ + } } } + +#undef MIN /* in case it would be already defined */ +#define MIN(a,b) ((a) < (b) ? (a) : (b)) + + +#define EXM_THROW(error, ...) \ +{ \ + DISPLAYLEVEL(1, "zstd: "); \ + DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + exit(error); \ +} + +#define CHECK_V(v, f) \ + v = f; \ + if (ZSTD_isError(v)) { \ + DISPLAYLEVEL(5, "%s \n", #f); \ + EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ + } +#define CHECK(f) { size_t err; CHECK_V(err, f); } + + +/* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define LONG_SEEK _fseeki64 +# define LONG_TELL _ftelli64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define LONG_SEEK fseeko +# define LONG_TELL ftello +#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) +# define LONG_SEEK fseeko64 +# define LONG_TELL ftello64 +#elif defined(_WIN32) && !defined(__DJGPP__) +# include + static int LONG_SEEK(FILE* file, __int64 offset, int origin) { + LARGE_INTEGER off; + DWORD method; + off.QuadPart = offset; + if (origin == SEEK_END) + method = FILE_END; + else if (origin == SEEK_CUR) + method = FILE_CURRENT; + else + method = FILE_BEGIN; + + if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) + return 0; + else + return -1; + } + static __int64 LONG_TELL(FILE* file) { + LARGE_INTEGER off, newOff; + off.QuadPart = 0; + newOff.QuadPart = 0; + SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); + return newOff.QuadPart; + } +#else +# define LONG_SEEK fseek +# define LONG_TELL ftell +#endif + +#if defined (__cplusplus) +} +#endif +#endif //ZSTD_FILEIO_COMMON_H