Skip to content

Compression asyncio - diff to decompression pool branch #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
yoniko wants to merge 25 commits into
base: decompression_asyncio_pool
Choose a base branch
from
Draft

Compression asyncio - diff to decompression pool branch #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
1bd4f18
Async IO decompression:
yoniko Dec 31, 2021
d735b88
Async IO decompression: CR fixes
yoniko Jan 6, 2022
c74b263
LZ4 decompression: Fixed multiframe decompression
yoniko Jan 6, 2022
dd5a540
Async IO decompression: Added multiframe test for --[no-]asyncio
yoniko Jan 6, 2022
702a9a0
Async IO decompression: C standard fix
yoniko Jan 6, 2022
05b174a
Async IO decompression: fix meson compilation
yoniko Jan 11, 2022
4a1b6b1
Async IO compression: Added support for async write in compression
yoniko Jan 7, 2022
309987d
ZSTD CLI decompression: Use buffered writes for improved performance
yoniko Jan 7, 2022
479afae
Async IO compression: bugfix
yoniko Jan 11, 2022
8fbd1ef
Revert "ZSTD CLI decompression: Use buffered writes for improved perf…
yoniko Jan 11, 2022
87cae9b
Async IO compression: moved FIO_fwriteSparse* functions
yoniko Jan 12, 2022
2626600
Async IO compression: fixed typose
yoniko Jan 12, 2022
89a78e7
Async IO compression: Refactored WritePool into IoPool so code can be…
yoniko Jan 13, 2022
4c49233
Async IO compression: Added ReaderPool and more refactors.
yoniko Jan 13, 2022
688f4bc
Async IO compression: Async read working but still needs work. WIP.
yoniko Jan 14, 2022
35dc66c
Async IO compression: Moved some `fileio.c` utilities into `fileio_ut…
yoniko Jan 14, 2022
4ea77b7
Async IO compression: Created `read_pool_ctx_t` to remove read-only p…
yoniko Jan 15, 2022
6106b7c
Async IO compression: Created `write_pool_ctx_t` to remove write-only…
yoniko Jan 15, 2022
4484095
Async IO compression: Some more refactoring to write/read pools code.…
yoniko Jan 15, 2022
8502b9d
Async IO compression: bug fixes
yoniko Jan 15, 2022
c6aea5c
Async IO compression: fix build failures
yoniko Jan 15, 2022
b7b9c06
Async IO compression: test string change
yoniko Jan 18, 2022
8256c07
Async IO compression: Small changes
yoniko Jan 18, 2022
990fc70
Async IO compression: Small changes
yoniko Jan 18, 2022
e7f3afa
Async IO compression: Some refactors for better clarity and code loca…
yoniko Jan 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions build/VS2008/zstd/zstd.vcproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,10 @@
RelativePath="..\..\..\programs\fileio.c"
>
</File>
<File
RelativePath="..\..\..\programs\fileio_asyncio.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
Expand Down
1 change: 1 addition & 0 deletions build/VS2010/zstd/zstd.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\dibio.c" />
<ClCompile Include="..\..\..\programs\fileio.c" />
<ClCompile Include="..\..\..\programs\fileio_asyncio.c" />
<ClCompile Include="..\..\..\programs\zstdcli.c" />
<ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
</ItemGroup>
Expand Down
4 changes: 2 additions & 2 deletions build/cmake/programs/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ if (MSVC)
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
endif ()

add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
target_link_libraries(zstd rt)
Expand Down Expand Up @@ -75,7 +75,7 @@ if (UNIX)
${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
DESTINATION "${MAN_INSTALL_DIR}")

add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c)
target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
endif ()
Expand Down
25 changes: 20 additions & 5 deletions build/meson/programs/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,31 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
join_paths(zstd_rootdir, 'programs/util.c'),
join_paths(zstd_rootdir, 'programs/timefn.c'),
join_paths(zstd_rootdir, 'programs/fileio.c'),
join_paths(zstd_rootdir, 'programs/fileio_asyncio.c'),
join_paths(zstd_rootdir, 'programs/benchfn.c'),
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
join_paths(zstd_rootdir, 'programs/datagen.c'),
join_paths(zstd_rootdir, 'programs/dibio.c'),
join_paths(zstd_rootdir, 'programs/zstdcli_trace.c'),
# needed due to use of private symbol + -fvisibility=hidden
join_paths(zstd_rootdir, 'lib/common/xxhash.c')]
join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
join_paths(zstd_rootdir, 'lib/common/pool.c'),
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
join_paths(zstd_rootdir, 'lib/common/error_private.c')]

zstd_deps = [ libzstd_dep ]
zstd_c_args = libzstd_debug_cflags

zstd_frugal_deps = [ libzstd_dep ]
zstd_frugal_c_args = [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ]

if use_multi_thread
zstd_deps += [ thread_dep ]
zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
zstd_frugal_deps += [ thread_dep ]
zstd_frugal_c_args += [ '-DZSTD_MULTITHREAD' ]
endif

zstd_deps = [ libzstd_dep ]
if use_zlib
zstd_deps += [ zlib_dep ]
zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
Expand Down Expand Up @@ -69,14 +80,18 @@ zstd = executable('zstd',
zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
join_paths(zstd_rootdir, 'programs/timefn.c'),
join_paths(zstd_rootdir, 'programs/util.c'),
join_paths(zstd_rootdir, 'programs/fileio.c')]
join_paths(zstd_rootdir, 'programs/fileio.c'),
join_paths(zstd_rootdir, 'programs/fileio_asyncio.c'),
join_paths(zstd_rootdir, 'lib/common/pool.c'),
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
join_paths(zstd_rootdir, 'lib/common/error_private.c')]

# Minimal target, with only zstd compression and decompression.
# No bench. No legacy.
executable('zstd-frugal',
zstd_frugal_sources,
dependencies: libzstd_dep,
c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ],
dependencies: zstd_frugal_deps,
c_args: zstd_frugal_c_args,
install: true)

install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
Expand Down
4 changes: 4 additions & 0 deletions contrib/VS2005/zstd/zstd.vcproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,10 @@
RelativePath="..\..\..\programs\fileio.c"
>
</File>
<File
RelativePath="..\..\..\programs\fileio_asyncio.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
Expand Down
20 changes: 17 additions & 3 deletions lib/common/pool.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,7 @@ static void* POOL_thread(void* opaque) {
/* If the intended queue size was 0, signal after finishing job */
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
ctx->numThreadsBusy--;
if (ctx->queueSize == 1) {
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
}
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
}
} /* for (;;) */
Expand Down Expand Up @@ -190,6 +188,17 @@ void POOL_free(POOL_ctx *ctx) {
ZSTD_customFree(ctx, ctx->customMem);
}

/*! POOL_joinJobs() :
* Waits for all queued jobs to finish executing.
*/
void POOL_joinJobs(POOL_ctx* ctx) {
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) {
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
}
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
}

void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
POOL_free (pool);
}
Expand Down Expand Up @@ -330,6 +339,11 @@ void POOL_free(POOL_ctx* ctx) {
(void)ctx;
}

void POOL_joinJobs(POOL_ctx* ctx){
assert(!ctx || ctx == &g_poolCtx);
(void)ctx;
}

int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
(void)ctx; (void)numThreads;
return 0;
Expand Down
6 changes: 6 additions & 0 deletions lib/common/pool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
*/
void POOL_free(POOL_ctx* ctx);


/*! POOL_joinJobs() :
* Waits for all queued jobs to finish executing.
*/
void POOL_joinJobs(POOL_ctx* ctx);

/*! POOL_resize() :
* Expands or shrinks pool's number of threads.
* This is more efficient than releasing + creating a new context,
Expand Down
8 changes: 4 additions & 4 deletions programs/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,17 +243,17 @@ zstd-pgo :

## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asycio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)

zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c
zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asycio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)

zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c
zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asycio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)

## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asycio.c dibio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)

zstdmt: zstd
Expand Down
Loading