diff --git a/ADS.md b/ADS.md new file mode 100644 index 00000000..0f841318 --- /dev/null +++ b/ADS.md @@ -0,0 +1,27 @@ +# ADS Notes + +Context +- Table-size ADS (keys:Algorithm_Hash_Mask.TableSize) is only used to seed + RequestedNumberOfTableElements for the current run. If missing, creation + still works; the requested size stays 0 and the solver proceeds normally. +- Table-info ADS (table.pht1:Info) is required for Table->Vtbl->Load() and + for downstream generators/tests that read Table->TableInfoOnDisk. + +Key locations +- Table-size load/save: `src/PerfectHash/PerfectHashKeysLoadTableSize.c` +- Table-info stream save: `src/PerfectHash/Chm01FileWorkTableInfoStream.c` +- Table-info load: `src/PerfectHash/PerfectHashTableLoad.c` + +Implications +- Dropping table-size ADS is low risk if we accept losing the "reuse previous + table size" hint. It is already optional from a correctness standpoint. +- Dropping table-info ADS breaks load-from-disk and any tools that read + TableInfoOnDisk (CSV outputs, generators, tests). A replacement on-disk + metadata format would be required first. + +Notes +- ReFS/ADS issues observed: zero-length TableSize streams caused + PH_E_INVALID_END_OF_FILE. Fixed by extending empty existing streams when + NoTruncate is set. +- If we want to avoid ADS entirely, table-size is a good first candidate to + move to a sidecar file (e.g., .TableSize) without affecting core table load. diff --git a/AGENTS.md b/AGENTS.md index 7f098774..dd88a486 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,3 +15,6 @@ ## File enum capacity - File enums are tracked via a 64-bit bitmap. When we run out of bits, add a new enum group with a `_2`, `_3`, etc. suffix and keep the existing enum ordering rules intact. + +## Auto-generated files +- `include/PerfectHashEvents.h` is auto-generated; discard local changes before rebasing or merging. diff --git a/CMakeLists.txt b/CMakeLists.txt index ba18621c..cf67c01b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,8 @@ if(PERFECTHASH_ENABLE_INSTALL) PerfectHash PerfectHashCreateExe PerfectHashBulkCreateExe + PerfectHashServerExe + PerfectHashClientExe RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md new file mode 100644 index 00000000..d1b4d792 --- /dev/null +++ b/IOCP-LOGS.md @@ -0,0 +1,217 @@ +# IOCP Logs + +## 2026-02-24 +- Consolidated IOCP project ledgers for fresh-session handoff: `IOCP-LOGS.md`, `IOCP-TOOD.md`, `IOCP-PROMPT.md`, and new `IOCP-README.md`. +- Captured current implementation status: + - IOCP server/client architecture is additive (new executables, OG paths retained). + - Per-NUMA-node IOCP runtime exists (one port per node, manual worker threads, affinity wiring). + - Named-pipe protocol supports ping/shutdown/table-create/bulk-directory requests. + - Bulk-directory request model provides single-token completion for client wait semantics. + - CHM01 async path exists and is IOCP-driven, with per-file concurrency ramp controls. + - IOCP file I/O path includes overlapped writes and overlapped keys loads (OG mmap path preserved). + - IOCP buffer pooling moved toward lookaside-style size classes with guard-page toggle support. +- Captured key stability/perf findings to carry forward: + - Fixed multiple hangs/crashes caused by async accounting leaks, request lifetime bugs, and stale file-write byte tracking. + - Fixed `TpIsTimerSet` crash by guarding legacy threadpool callbacks when IOCP mode bypasses threadpool init. + - Full sys32 file-I/O IOCP runs complete; runtime is better than OG in some runs, but tuning is workload-sensitive. + - High-concurrency configurations can over-allocate memory; pool sizing policy requires further hardening. +- No source-code behavior changes were introduced in this pass beyond documentation updates. + +## 2026-01-19 +- Added IOCP context/server/client interfaces and COM scaffolding. +- Added NUMA-aware IOCP runtime skeleton (per-node IOCPs, worker threads with affinity). +- Wired new components into PerfectHash constants/build definitions. +- Added server/client executables with basic CLI parsing and IOCP startup wiring. +- Fixed /Wall padding warnings and confirmed Debug builds for server/client targets. +- Implemented named-pipe IOCP transport in the server with per-node pipe instances and a request/response state machine. +- Implemented client-side named-pipe connect and request submission with wire protocol headers. +- Fixed /Wall issues in new IOCP code (Rtl-backed ZeroMemory usage, struct padding, enum exhaustiveness) and rebuilt server/client executables. +- Wired server request dispatch to parse command-line payloads and call IOCP context bulk/table create entrypoints. +- Implemented IOCP ArgvW entrypoints by delegating to legacy PerfectHashContext workflows. +- Added server endpoint/local-only API + CLI configuration, plus error-response payload handling and client-side retrieval/printing. +- Added a PowerShell smoke test script for server/client shutdown round-trips. +- Ran `scripts/iocp-smoke.ps1` after fixes to confirm clean startup/shutdown. +- Diagnosed a Debug breakpoint during table create to a TLS global-component bitfield mismatch; realigned TLS flag bit positions to interface IDs and removed fragile asserts. +- Added a create-exe unhandled-exception stack trace hook (DbgHelp) to aid future diagnostics. +- Hardened `scripts/iocp-smoke.ps1` to handle missing exit codes after timed waits; smoke test now completes successfully. +- Added IOCP work item header/signature and per-item completion dispatch in the IOCP worker loop. +- Updated server pipe handling to use per-pipe IOCP work items instead of global completion callbacks. +- Added stress scripts for sys32 bulk create (direct and IOCP variants). +- Rebuilt PerfectHashServerExe and reran `scripts/iocp-smoke.ps1` successfully. +- Added bulk-create directory server request with per-request completion token (event + mapping). +- Implemented server-side directory walk and per-file IOCP work item dispatch for bulk requests. +- Added client-side `--BulkCreateDirectory` support with token wait and bulk result handling. +- Fixed bulk enqueue completion accounting and failure handling to avoid double-decrement. +- Updated IOCP sys32 stress script to use bulk-create directory request. +- Regenerated PerfectHash error headers/resources for new status codes. +- Rebuilt PerfectHashServerExe/PerfectHashClientExe with `/p:CL_MPCount=1` and reran `scripts/iocp-smoke.ps1`. +- Built Release targets for PerfectHashBulkCreate/Server/Client with `/p:CL_MPCount=1`. +- BulkCreate Release sys32 run at concurrency 32 crashed (`0xC0000005`); repeated `FlushConsoleInputBuffer` errors observed. +- BulkCreateDirectory argv handling fixed to always inject a program name for parsing. +- IOCP Release sys32 run at concurrency 32 fails with `SetThreadpoolThreadMinimum` access denied during per-file context init. +- Created `D:\\src\\perfecthash-main` worktree, built Release `PerfectHashBulkCreate.exe` and tested baseline bulk-create. +- `hard2` run succeeded but printed repeated `FlushConsoleInputBuffer` errors. +- `sys32` run crashed with access violation (`0xC0000005`) and the same flush errors, indicating the crash exists on `main`. + +## 2026-01-20 +- Rebased `iocp-dev` onto `origin/main` and resolved `PerfectHashBulkCreateExe.c` conflict by keeping upstream minidump handler. +- Ignored `ERROR_ACCESS_DENIED` from `SetMaximumConcurrency` in IOCP per-file work items to allow bulk runs to continue. +- Diagnosed IOCP bulk-create server crash (`0xC0000005`) to use-after-free of `CommandLineToArgvW` buffers referenced by table-create parameters. +- Retained bulk-create directory `ArgvW` buffers in request state (free on completion) and moved argv adjustment into server dispatch to keep string pointers alive. +- Fixed IOCP bulk work-item callback use-after-free (request freed before work item allocator cleanup). +- Mapped IOCP bulk-create crash to `PrepareCHeaderFileChm01` using `Context->CommandLineW`; crash was NULL due to direct `TableCreate` usage. +- Preserved request command line and seeded per-file contexts with `Context->CommandLineW` to keep header generation stable. +- Updated IOCP stress script to accept `PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED` as a success exit code. +- Ran `scripts/stress-sys32-iocp.ps1` against a single-file keys dir (Release, max concurrency 1); completed successfully. +- Added IOCP-backed file work dispatch hooks: context submit/wait helpers with outstanding-event tracking, per-file context IOCP port wiring, and file-work event signaling for non-threadpool callbacks. +- Added async work framework scaffolding and CHM01 async state machine skeleton (new modules, build integration). +- Added CHM01 async bridge that runs the legacy CHM01 routine on a worker thread and polls completion via IOCP, plus an env-gated async entrypoint. +- Replaced the CHM01 async bridge with IOCP-driven solver slices, graph work items, and a state-machine finalize path for save/close/verify. +- Fixed CHM01 async header macro usage inside a struct (removed extra semicolons) and prevented double-include macro expansion by adding `#pragma once` to `Chm01FileWork.h`. +- Added explicit padding to CHM01 async graph work items and undefed `EXPAND_AS_CHECK_ERRORS` before redefining to satisfy `/Wall` with warnings-as-errors. +- Rebuilt `PerfectHashServerExe` Debug successfully after the async/header fixes. +- Ran `scripts/stress-sys32-iocp.ps1` with `PERFECT_HASH_IOCP_ASYNC_CHM01=1` against `perfecthash-keys\\hard` (Debug, max concurrency 32); run timed out with client hung waiting for completion (server already exited), leaving partial output in `build-win\\iocp-stress-hard`. +- Added IOCP work-item flags and a CHM01 async pump loop so IOCP completions continue while `Chm01AsyncWaitJob()` waits; file work/async/pipe items are dispatched inline, bulk work items are requeued. +- Added a CHM01 async prepare-wait state for the try-larger-table-size path (still returns `PH_E_NOT_IMPLEMENTED` but now waits for prepare file work before closing). +- Stress runs against `perfecthash-keys\\hard` still time out; observed repeated `SetEndOfFile`/`PerfectHashFileTruncate` failures (error 1224: mapped section open) and `PrepareFileChm01` failures during IOCP bulk runs. +- Added a context flag to skip context-level file work for per-file IOCP contexts; CHM01 async prepare/save/close now bypass context file items and signal events directly to avoid `ERROR_USER_MAPPED_FILE` conflicts. +- Added `CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED` and gated `Chm01AsyncFinalizeClose()` on save completion; updated graph completion to use `Job->Context` and the IOCP pump to wait for `Outstanding == 0` before exiting. +- IOCP stress runs still time out within the 120s tool limit, but bulk create AVs in `Chm01AsyncGraphStep` are no longer reproduced. +- Added per-work-item `LastError` tracking in IOCP bulk logging (`PerfectHashServerBulkCreateFailures.log`) to capture `GetLastError()` at the point of failure. +- Added focused two-file IOCP stress run directory (`build-win\\keys-test`) to isolate failures on `shell32-13803.keys`/`CoreUIComponents-7995.keys`. +- `Mulshrolate4X` is not recognized by `PerfectHashClient` (invalid hash function ID); only `Mulshrolate4RX` exists in current hash table list. +- IOCP bulk-create against the two-file `build-win\\keys-test` set with `Mulshrolate4RX` still hangs (client/server left running until manually terminated; no bulk failure log emitted). +- Normal `PerfectHashBulkCreate.exe` run on the same two-file set with `Mulshrolate4RX` crashed with `0xC0000005` after `CoreUIComponents-7995.keys`; no crash dump/log emitted. +- OG `PerfectHashBulkCreate.exe` run against `perfecthash-keys\\test1` (5 files) crashed with `0xC0000005` after `Hydrogen-43013.keys` even with an absolute output directory; `--DisableCsvOutputFile` did not prevent the crash; no crash dump/log emitted. + +## 2026-01-21 +- Created a clean `D:\\src\\perfecthash-main-clean` worktree at `origin/main` (7c83dc5a), built Debug `PerfectHashBulkCreateExe`, and ran `PerfectHashBulkCreate.exe` against `..\\perfecthash-keys\\test1` with `Chm01 Mulshrolate4RX And 4`; completed successfully with five output directories, indicating the crash is specific to `iocp-dev`. +- Noted that `tests/CMakeLists.txt` includes a bulk CLI test that uses `--NoFileIo` and `MultiplyShiftR`, so it doesn't cover the current file-I/O crash scenario. +- Clean rebuilt `build-win` after `PERFECT_HASH_CONTEXT` layout changes; `PerfectHashBulkCreate.exe` no longer AVs on `test1`, indicating a stale PCH/obj mismatch was responsible. +- Added crash-dir env overrides (`PH_BULK_CREATE_CRASH_DIR`, `PH_SERVER_CRASH_DIR`) and server crash test env (`PH_SERVER_CRASH_TEST`), plus DbgHelp preloading and `MiniDumpIgnoreInaccessibleMemory`. +- Reworked crash-test path to generate dumps without exception pointers (avoid `MiniDumpWriteDump` 998) and added `scripts/test-minidump.ps1`; Debug run now produces non-empty minidumps for bulk-create and server. +- Added a `MiniDumpWriteDump` retry path that drops exception pointers on `ERROR_NOACCESS`, keeping real-crash logs while allowing dumps to be written. +- Removed CHM01 async resize/try-larger-table logic (events, state transitions, and initial resize setup) to keep IOCP async solving focused on single-pass solve behavior. +- IOCP bulk stress against `perfecthash-keys\\hard` (Debug) still times out in ~120s; server left running and only ~12/23 output dirs created, indicating completion/hang persists post-resize removal. +- Attached `cdb` to `PerfectHashServer.exe` ~10s into a Debug IOCP bulk run and captured `~* kb` (`cdb-hard-backtrace.log`): main thread blocked in `PerfectHashServerWait`, IOCP worker threads blocked in `GetQueuedCompletionStatus` (33 waits on one IOCP handle, 24 on another), and ~1539 threads waiting in `ZwWaitForWorkViaWorkerFactory`, indicating no active IOCP completions at capture time and unexpected threadpool worker proliferation. +- Added IOCP-native argument extraction helpers (`PerfectHashContextIocpArgs.c`) and wired into build files. + +## 2026-01-25 +- Refactored IOCP file-work buffers to size-class pools (4KB–16MB) plus dynamic oversize pools; buffers tracked via guarded lists for teardown. +- Updated IOCP context/node structs and rundown paths to free pooled buffers and oversize pool lists. +- Reworked CHM01 file-work buffer acquisition to use size-class/oversize pools (no one-off pools) and new acquire/release APIs. +- Updated IOCP buffer pool API to accept `PRTL` for initialization zeroing; removed legacy create/pop/push/destroy calls. +- Updated IOCP buffer pool unit tests and size-class helper coverage. +- Built Release `perfecthash_unit_tests` and ran `build-win\\bin\\Release\\perfecthash_unit_tests.exe` (all 8 tests pass). +- Added TLS flag plumbing for threadpool-less context creation and IOCP helper to create contexts with `CreateContextWithoutThreadpool` set. +- Updated server bulk/table create to avoid legacy context parsing and use IOCP-native arg extraction. +- Added IOCP table-create dispatch in server (context creation, file-work IOCP wiring, hook/RNG init, TLS context) and removed threadpool priority application. +- Routed server BulkCreate requests through the bulk-create directory path to avoid legacy context invocation. +- Clean rebuild (Debug) succeeded; IOCP stress on `perfecthash-keys\\hard` with `Mulshrolate4RX` and concurrency 32 timed out in ~330s; 23/23 output directories were created but the client hung waiting for completion, then was terminated. +- Debugger run with cdb breakpoints (`cdb-server-bp5.log`, `cdb-client-bp5.log`) showed client hits `PerfectHashClientWaitForBulkCreateToken`/`WaitForSingleObject`, server hits `PerfectHashServerBulkCreateWorkItemCallback`, but `PerfectHashServerCompleteBulkRequest` never hit within 120s; only 9 work-item callbacks logged and 5/23 output dirs created before termination. +- Debugger run against `perfecthash-keys\\test1` (5 files) showed `PerfectHashServerBulkCreateWorkItemCallback` hit 3 times, `PerfectHashServerCompleteBulkRequest` never hit, and only 1 output dir created (`cdb-server-bp-test1.log`, `cdb-client-bp-test1.log`). +- Repeated test1 run with server `--MaxConcurrency=1` showed only 1 work-item callback hit, no bulk completion, and 1 output dir created (`cdb-server-bp-test1-1.log`, `cdb-client-bp-test1-1.log`). +- Added cdb request-state logging using struct offsets from `cdb-types.log`. On test1, `PerfectHashServerEnqueueBulkRequest` hit once, and three work-item callbacks logged `TotalWorkItems=5` with `OutstandingWorkItems` at 2/4/6 and `DispatchComplete=0` at callback entry. `PerfectHashServerCompleteBulkRequest` never hit and only 1 output dir was created (`cdb-server-bp-test1-B.log`, `cdb-client-bp-test1-B.log`). +- `--NoFileIo` test (test1) under cdb: enqueue hit once, two work-item callbacks observed, no bulk completion hit, and no output directories created (`cdb-server-bp-test1-nofile.log`, `cdb-client-bp-test1-nofile.log`). +- Diagnosed `TpIsTimerSet` invalid-parameter crash in IOCP worker threads to `SubmitThreadpoolWork(Context->FinishedWork)` from `GraphSolve`; guarded finished-work submissions in `Graph.c`/`GraphCu.c` when threadpool init is skipped. +- Rebuilt Debug and reran IOCP bulk create on `perfecthash-keys\\test1` under cdb with breakpoints on `Chm01AsyncGraphComplete` and `PerfectHashServerCompleteBulkRequest`; graph completion fired repeatedly, bulk completion callback fired once, and all five output directories were created (`cdb-server-graphcomplete.log`). +- Re-ran IOCP bulk create on `perfecthash-keys\\test1` without cdb (`PERFECT_HASH_IOCP_ASYNC_CHM01=1`); completed successfully and emitted all five output directories (`build-win\\iocp-stress-test1-nocdb`). +- Replaced `CHM01_ASYNC_JOB.Events[5]` with an enum-backed count macro and updated initialization to use named event IDs. +- Added client `--WaitForServer`/`--ConnectTimeout=` and ping request/response support to wait for server readiness before submitting work. +- Updated IOCP scripts to pass `--WaitForServer`/`--ConnectTimeout`; ping check against `PerfectHashServer-PingTest` returned 0 for ping/shutdown/server. +- IOCP bulk run against `perfecthash-keys\\hard` (Debug, concurrency 32) still timed out after ~360s; all 23 output directories were created, server exited, but the client remained waiting until terminated (`build-win\\iocp-stress-hard-latest`). +- OG bulk create (Debug) against `perfecthash-keys\\hard` completed in ~0.82s (cmd /c run, concurrency 32), exit 0. +- IOCP bulk run against `perfecthash-keys\\hard` with concurrency 1 under cdb hit `PerfectHashServerEnqueueBulkRequest`/`WorkItem` callbacks, then crashed in `PerfectHash!PhFree` while printing stats (`PrintCurrentContextStatsChm01` -> `Chm01AsyncFinalizeVerify`); client timed out waiting (`cdb-server-hard1.log`). +- Fixed `_dtoa_Allocator` race by resolving allocator from TLS context in `_dtoa.c` (no TLS var in DLL); Debug rebuild succeeded. +- Re-ran IOCP bulk `hard` with concurrency 1 under cdb; no crash, `PerfectHashServerCompleteBulkRequest` hit, client exited cleanly (`cdb-server-hard1b.log`). + +## 2026-01-22 +- Reproduced IOCP bulk-create hang on `perfecthash-keys\\hard` (Debug, concurrency 32); client waited indefinitely while many per-file outputs were created. +- Added optional bulk-count logging (`PH_LOG_BULK_CREATE_COUNTS`) and CHM01 async job logging (`PH_LOG_CHM01_ASYNC_JOB`) to track request completion and async state transitions. +- Identified `Chm01AsyncWaitJob()` pump loop could block in `GetQueuedCompletionStatus()` after job completion, leaving bulk work items stuck mid-callback. +- Added a dummy IOCP wake post in `Chm01AsyncComplete()` and switched the pump loop to a 100ms timeout to re-check job completion state. +- Re-ran IOCP stress on `perfecthash-keys\\hard` with `Mulshrolate4RX`/concurrency 32; run completed successfully and server shutdown cleanly. +- Clean Release rebuild after build dir cleanup and ran IOCP sys32 stress (`Mulshrolate4RX`, concurrency 32); run completed in ~197s but emitted repeated `PerfectHashKeysLoadTableSize` invalid EOF messages (0xe0040264). +- Identified 31 sys32 keys files with zero-length `Chm01_Mulshrolate4RX_And.TableSize` streams causing invalid EOF failures. +- Reproduced the invalid EOF with a single file (`aadauthhelper-189.keys`) via `PerfectHashCreate.exe`. +- Fixed `PerfectHashFileCreate()` to extend existing zero-length files when `NoTruncate` is set; single-file create now succeeds. +- Added `ADS.md` with notes on table-size vs table-info ADS usage and risks of removal. +- Created `..\\perfecthash-keys\\win32.temp50` with 50 random sys32 keys and ran IOCP bulk create (Release, `Mulshrolate4RX`, concurrency 32); run completed with 50 output dirs. +- Attempted full sys32 IOCP run (Release, `Mulshrolate4RX`, concurrency 32) to `build-win\\iocp-stress-sys32-release`; run emitted `SetEndOfFile` failures with `ERROR_DISK_FULL` and was halted; output dir grew to ~193 GB with ~13,637 directories before shutdown. +- Added IOCP server verbosity/no-file-IO controls (`--Verbose`, `--NoFileIo`) and vtbl setters/getters; server now defaults to silent output and only prints console legend when console work is enabled. +- Updated IOCP scripts (`scripts/iocp-smoke.ps1`, `scripts/stress-sys32-iocp.ps1`) to pass the new server flags (NoFileIo default true, optional verbose). +- Ran IOCP sys32 `--NoFileIo` stress (Release, `Mulshrolate4RX`, concurrency 32) via script; client/server did not complete within tool timeout, shutdown timed out, and both processes required termination (follow-up needed). +- Ran IOCP bulk create against a 200-file subset of `sys32` (`G:\\Scratch\\sys32-200`, Release, `Mulshrolate4RX`, concurrency 4, `--NoFileIo`); completed successfully within the 30s timeout. +- Captured server thread samples during the 200-file run (Release, concurrency 4). `PerfectHashServer.exe` showed 68 threads total; initial samples had 31-36 running, then all waiting. Wait reasons were dominated by `EventPairLow` (IOCP), with a few `Unknown`/`UserRequest`. +- Re-ran the 200-file sys32 subset with server `--MaxConcurrency=4` (Release, `Mulshrolate4RX`, `--NoFileIo`). Thread sampling showed 8 total threads (4 running initially, then all waiting), with wait reasons dominated by `EventPairLow`; client/server exited cleanly. +- Added separate IOCP concurrency and worker thread controls: `--IocpConcurrency` (`--MaxConcurrency` alias) and `--MaxThreads` (default `IocpConcurrency * 2` when set). IOCP ports now use per-node concurrency, worker threads default per node; scripts accept `-IocpConcurrency`/`-MaxThreads`. +- 200-file sys32 subset run with `--IocpConcurrency=4` (Release, `Mulshrolate4RX`, `--NoFileIo`) showed 12 total server threads (6 running initially), wait reasons dominated by `EventPairLow`; client/server exited cleanly. +- 1000-file random sys32 subset (`G:\\Scratch\\sys32-1000`, Release, `Mulshrolate4RX`, `--NoFileIo`, per-file concurrency 32, server `--IocpConcurrency=32 --MaxThreads=64`) did not complete within 900s; client/server remained running and required termination. +- OG `PerfectHashBulkCreate.exe` run on the same 1000-file subset (Release, `Mulshrolate4RX`, `--NoFileIo`, concurrency 32, `--Silent`) completed in ~1.95s with exit code 0. +- Added per-file concurrency ramp controls (`InitialPerFileConcurrency`, `MaxPerFileConcurrency`, `IncreaseConcurrencyAfterMilliseconds`) with defaults (1, max, 500ms). +- Wired new table-create params into arg parsing and validation, IOCP arg helpers, and `PERFECT_HASH_CONTEXT`. +- Updated `Chm01Async` to dispatch the initial per-file graph work count and enqueue additional graph work after the configured delay. +- Documented new parameters in `USAGE.txt`. +- IOCP sys32-200 subset run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms) completed successfully. +- IOCP `hard` run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms) crashed with `0xC0000005` (server exit code -1073741819); no minidump found. +- Re-ran `hard` under `cdb` attach (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms); client returned `0x2004000F`, shutdown succeeded, and no AV was captured (`build-win\\logs\\cdb-hard-attach-release.log`). +- IOCP full sys32 run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/4/500ms) completed in ~38.1s. +- OG BulkCreate full sys32 run (Release, `Mulshrolate4RX`, `--NoFileIo`, concurrency 32, full paths) completed in ~20.05s; no errors logged (`build-win\\logs\\og-sys32-conc32.log`). +- IOCP full sys32 run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/16/500ms) completed in ~38.1s. +- OG BulkCreate full sys32 run with file I/O (Release, `Mulshrolate4RX`, concurrency 32) completed in ~2m30s based on `G:\\Scratch\\sys32.og.fileio` directory timestamps (created 19:27:31, last write 19:30:01). +- IOCP full sys32 run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/4/500ms) completed in ~1m27s (`G:\\Scratch\\iocp-sys32-conc32-fileio.log`). +- IOCP full sys32 run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=60`, `--MaxThreads=240`, per-file ramp 1/4/500ms) completed in ~1m49s (`G:\\Scratch\\iocp-sys32-conc60-1-4-500-fileio.log`). +- Added IOCP buffer pool module (`PerfectHashIocpBufferPool.[ch]`) and gtest coverage for create/pop/push and empty-pop behavior. +- Fixed buffer-pool unit test AVs by providing test `RtlFillMemory`/`RtlZeroMemory` helpers (internal `ZeroMemory()` macros call `Rtl->RtlFillMemory`). +- Verified `ctest -C Debug -R PerfectHashIocpBufferPool` passes. +- Added per-node bucketed IOCP file-work buffer pools (bucketed by AlignUpPow2(keys) and indexed by file id); IOCP contexts now capture node pointers from the server for pool selection. +- Switched IOCP file saves to true overlapped writes: issue `WriteFile` with IOCP work item, associate output file handles with per-node ports, and complete saves via IOCP callback (set events, return buffers, record errors). +- IOCP sys32-200 subset run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=4`, `--MaxThreads=8`) hung; only 7/200 output dirs created in `G:\\Scratch\\iocp-sys32-200-conc4-fileio-new`, client stuck waiting, server already exited. +- Attached cdb to `PerfectHashServer.exe` during sys32-200 file-I/O run; captured access violation (`0xC0000005`) in `PerfectHashFileTruncate()` → `PerfectHashFileCreate()` → `PrepareFileChm01()` → `FileWorkItemCallbackChm01()` on an IOCP worker thread, with other threads in `PrepareCHeaderFileChm01()`/`AppendCharBufferToCharBuffer()`; log at `build-win\\logs\\cdb-iocp-fileio-sys32-200-av.log`. + +## 2026-01-23 +- Ran IOCP bulk-create against `G:\\Scratch\\sys32-200` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8 --NoFileIo`); client exited with `0x2004000F` success after manual run. +- Captured server thread stacks post-run: main thread waiting in `PerfectHashServerWait`, IOCP worker threads blocked in `GetQueuedCompletionStatus`, two worker-factory threads idle; log at `build-win\\logs\\cdb-iocp-200-conc4-stacks2.log`. +- Manual `PerfectHashClient` runs require quoting the entire `--BulkCreateDirectory=...` argument; otherwise client reports an invalid argument. +- Ran IOCP bulk-create against `G:\\Scratch\\sys32-1000` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8 --NoFileIo`) and attached cdb ~1s in with `!runaway;~* kb`. Thread sample shows main thread waiting in `PerfectHashServerWait`, IOCP worker threads in `GetQueuedCompletionStatus`, and four worker-factory threads parked in `ZwWaitForWorkViaWorkerFactory` while work is active; log at `build-win\\logs\\cdb-iocp-1000-conc4-stacks.log`. +- Reviewed async/bulk accounting paths (server + CHM01 async + file work) and identified potential outstanding-count leaks on error/requeue paths; see analysis notes for proposed fixes. +- Fixed IOCP async accounting: requeue failures now complete work (decrement outstanding), and graph-work submit failures roll back counts and signal events when needed. +- Quick test (Release, `Mulshrolate4RX`, `G:\\Scratch\\sys32-200`, `--IocpConcurrency=4 --MaxThreads=8 --NoFileIo`) completed in ~1.01s, client exit `0x2004000F`, server exit 0. + +## 2026-01-24 +- Fixed IOCP buffer sizing to index EOF initializers by `FileId` (not `FileWorkId`) to avoid out-of-bounds reads during save-stage sizing; `SavePythonTestFileChm01` no longer fails with `PH_E_INVARIANT_CHECK_FAILED`. +- Rebuilt Release and reran IOCP file-I/O runs on `perfecthash-keys\\test1` and `perfecthash-keys\\hard` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`); both completed without errors. +- Skipped mapping in `PerfectHashFileExtend()` when `SkipMapping` is set (IOCP mode) to prevent `ERROR_USER_MAPPED_FILE` during close. +- Added IOCP one-off buffer fallback and buffer-size validation/cleanup so save stages can request larger payloads without failing; one-off pools are destroyed after use. +- Rebuilt Release and reran IOCP file-I/O on a 200-file sys32 subset (`D:\\Scratch\\sys32-200` → `D:\\Scratch\\iocp-sys32-200-fileio-4`, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`); run completed cleanly with no file-work errors. +- Timed IOCP file-I/O run on `G:\\Scratch\\sys32-1000` (1000 random sys32 keys) with `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8 --MaximumConcurrency=4`; completed in ~26.11s (`G:\\Scratch\\iocp-sys32-1000-fileio-timed2-20260123-120241`). +- Timed OG bulk-create file-I/O run on the same 1000-file set with `Mulshrolate4RX` and concurrency 4; completed in ~11.30s (`G:\\Scratch\\sys32-1000-og-fileio-timed2-20260123-120330`). +- Reset `File->NumberOfBytesWritten` before IOCP prepare callbacks and added IOCP-only bounds checks in `SaveCHeaderFileChm01`/`SaveCppSourceUnityFileChm01` to avoid stale offsets and AVs. +- `Chm01AcquireFileWorkBuffer()` now falls back to one-off buffers when the per-file pool is empty (instead of returning `E_OUTOFMEMORY`). +- IOCP file-I/O run on `G:\\Scratch\\sys32-repro-small` (Release, `Mulshrolate4RX`, `--IocpConcurrency=1 --MaxThreads=2`) completed successfully. +- IOCP file-I/O run on `D:\\src\\perfecthash-keys\\hard` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`) completed successfully. +- Added fail-fast `PH_RAISE` guards for oversized IOCP write requests and reset `NumberOfBytesWritten` for IOCP save-only file work items. +- Full IOCP sys32 file-I/O run completed (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `G:\\Scratch\\iocp-sys32-full` in ~289.6s; largest output file ~2.5MB, no anomalously large files observed. +- OG BulkCreate sys32 file-I/O run completed (Release, `Mulshrolate4RX`, concurrency 32) to `G:\\Scratch\\sys32-og-full` in ~126.6s; largest output file ~21.3MB (CSV), no anomalously large files observed. +- IOCP sys32 file-I/O run with per-file ramp 1/16/500 completed (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `G:\\Scratch\\iocp-sys32-full-max16` in ~289.6s; largest output file ~2.5MB, no anomalously large files observed. +- Created a fresh 1000-file sys32 subset at `E:\\Scratch\\sys32-1000` for ramp tuning. +- OG BulkCreate run on `E:\\Scratch\\sys32-1000` completed in ~22.9s (Release, `Mulshrolate4RX`, concurrency 32) to `E:\\Scratch\\sys32-1000-og`. +- IOCP run on `E:\\Scratch\\sys32-1000` with ramp `Initial=2`, `Max=8`, `After=200ms` completed in ~21.1s (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `E:\\Scratch\\sys32-1000-iocp-2-8-200`. +- IOCP run on `E:\\Scratch\\sys32-1000` with ramp `Initial=1`, `Max=8`, `After=200ms` completed in ~21.1s (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `E:\\Scratch\\sys32-1000-iocp-1-8-200-2`. +- IOCP run on `E:\\Scratch\\sys32-1000` with ramp `Initial=1`, `Max=4`, `After=200ms` completed in ~20.1s (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `E:\\Scratch\\sys32-1000-iocp-1-4-200`. +- IOCP run on `E:\\Scratch\\sys32-1000` with `--IocpConcurrency=64 --MaxThreads=196`, ramp `Initial=1`, `Max=4`, `After=100ms` hit `VirtualAllocEx` failures (error 1455) and `PH_E_OUTOFMEMORY` during save (`SaveCppSourceUnityFile`); run completed in ~21.1s but was not clean (`E:\\Scratch\\sys32-1000-iocp-1-4-100-64`). +- IOCP run on `E:\\Scratch\\sys32-1000` with `--IocpConcurrency=64 --MaxThreads=128 --MaximumConcurrency=32`, ramp `Initial=1`, `Max=2`, `After=100ms` still hit `VirtualAllocEx` failures (error 1455) and `PH_E_OUTOFMEMORY` during save (`SaveCppSourceUnityFile`); run completed in ~20.1s but was not clean (`E:\\Scratch\\sys32-1000-iocp-1-2-100-64`). + +## 2026-01-25 +- Planned IOCP buffer pool redesign based on `D:\\src\\lookaside` NUMA lookaside pattern: + - Replace per-file/bucket pools with per-NUMA size-class pools keyed by payload size (power-of-two). + - Size classes initially 4KB–16MB; oversize allocations use dynamic pools keyed by next power-of-two. + - Pools allocate on demand; buffers are reusable via SLIST free lists and tracked via `GUARDED_LIST` for rundown. + - One-off allocations become reusable buffers owned by a pool; guarded list enables safe teardown. + - Add optional guard-page allocation flag (default on) to choose between guard-page buffers vs raw chunks. + - Add basic pool stats/diagnostics later (depth, alloc failures); ETW deferred. + +## 2026-01-26 +- Added IOCP buffer guard-page toggle: new server flag + CLI (`--IocpBufferGuardPages`), context flag propagation, and pool init wiring. +- Implemented IOCP keys-load path: `PERFECT_HASH_KEYS_LOAD_FLAGS.UseOverlappedIo`, `PERFECT_HASH_FILE_LOAD_FLAGS.SkipMapping`, overlapped `ReadFile` into pooled buffers, and buffer release on file close. +- Added gtest coverage for guard-page buffer pool flag propagation. diff --git a/IOCP-PROMPT.md b/IOCP-PROMPT.md new file mode 100644 index 00000000..b80f523a --- /dev/null +++ b/IOCP-PROMPT.md @@ -0,0 +1,105 @@ +# IOCP Backend Handoff Prompt (Start Here) + +Continue IOCP server/client development for PerfectHash with the current state below. + +## Goal + +Make bulk perfect-hash creation saturate CPU effectively by decoupling work from workers with a NUMA-aware IOCP pipeline, while keeping OG executables/paths intact. + +## What Is Implemented + +- Additive IOCP architecture (new server/client executables; OG untouched). +- Per-NUMA node IOCP runtime: + - one completion port per node + - manually created worker threads + - node/group affinity wiring. +- Named-pipe request/response state machine with: + - ping/pong readiness + - shutdown + - table-create and bulk-directory requests. +- Bulk directory request flow: + - server walks `.keys` files + - dispatches per-file work across configured nodes + - returns a single completion token (event/mapping) to client. +- CHM01 async state-machine path present with per-file ramp controls: + - `--InitialPerFileConcurrency` + - `--MaxPerFileConcurrency` + - `--IncreaseConcurrencyAfterMilliseconds`. +- IOCP file-work path has overlapped save I/O and keys-load overlapped reads. +- IOCP buffer pool infrastructure exists and is being reworked toward NUMA lookaside-like size classes with optional guard pages. + +## Important Fixes Already Landed + +- Lifetime fixes: + - retained command-line/argv buffers for bulk request lifetime + - fixed use-after-free in bulk callback path. +- Async accounting fixes: + - requeue failure now decrements outstanding + - graph-submit failure rollback fixed (`ActiveGraphs`, loop counters, cleanup). +- Crash fixes: + - guarded legacy threadpool callback usage (`TpIsTimerSet` crash source) + - fixed stale `NumberOfBytesWritten`/sizing bugs that produced oversized files + - fixed IOCP file sizing index bug (`FileId` vs `FileWorkId`). +- Added crash diagnostics/minidump improvements and server/client wait-for-ready behavior. + +## Observed Performance Snapshot (Recent) + +- On some file-I/O workloads, IOCP outperformed OG. +- On some no-file-I/O workloads, OG remained faster. +- High concurrency (`IocpConcurrency=64`) exposed memory pressure/over-allocation; pool policy still needs tuning. + +## Current Risks / Gaps + +- Buffer pool policy can overprovision at high concurrency. +- Need stricter invariants around async completion counters to prevent latent hangs. +- Large-payload file writes need explicit chunk/flush strategy validation. +- Need broader repeatable E2E coverage (test1/hard/sys32 subsets/full). + +## Next Execution Order + +1. Lock correctness first: + - counter invariants + - completion-once guarantees + - failure-path decrements. +2. Finish buffer-pool redesign: + - per-NUMA size-class global pools + - oversize reuse pools + - safe guarded-list rundown. +3. Harden overlapped file-I/O: + - large write chunking + - strict bounds/fail-fast behavior + - OG path unchanged. +4. Re-run standardized perf matrix and tune defaults. + +## Key Files + +- IOCP runtime: + - `src/PerfectHash/PerfectHashContextIocp.c` + - `src/PerfectHash/PerfectHashContextIocp.h` +- Server/client core: + - `src/PerfectHash/PerfectHashServer.c` + - `src/PerfectHash/PerfectHashClient.c` +- Async engine: + - `src/PerfectHash/PerfectHashAsync.c` + - `src/PerfectHash/Chm01Async.c` +- IOCP file work/buffer pool: + - `src/PerfectHash/Chm01FileWork.c` + - `src/PerfectHash/Chm01FileWorkIocp.c` + - `src/PerfectHash/PerfectHashIocpBufferPool.c` + - `src/PerfectHash/PerfectHashIocpBufferPool.h` +- Ledgers: + - `IOCP-LOGS.md` + - `IOCP-TOOD.md` + - `IOCP-README.md` + +## Quick Commands + +- Build: + - `cmake --build build-win --config Release --target PerfectHashServerExe PerfectHashClientExe PerfectHashBulkCreateExe` +- IOCP smoke: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\iocp-smoke.ps1 -BuildDir build-win -Config Release` +- IOCP stress: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32-iocp.ps1 -BuildDir build-win -Config Release -IocpConcurrency 32 -MaxThreads 64` +- OG stress: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` + diff --git a/IOCP-README.md b/IOCP-README.md new file mode 100644 index 00000000..688ba7a4 --- /dev/null +++ b/IOCP-README.md @@ -0,0 +1,72 @@ +# IOCP PerfectHash Backend + +This document is the current high-level reference for the additive IOCP implementation. + +## Scope + +- New server/client path for table and bulk create requests. +- Keep OG `CreateExe` / `BulkCreateExe` behavior intact. +- Use manual Windows threads + `GetQueuedCompletionStatus()` loops. +- NUMA-aware design: one IOCP per node, affinity-aware workers. + +## Design Summary + +1. Client sends request over named pipe. +2. Server accepts request via IOCP pipe state machine. +3. For bulk-directory requests: + - enumerate `.keys` files + - dispatch per-file work items (round-robin across selected NUMA nodes) + - track per-request counters + - signal one completion token when all file work is complete. +4. Client waits on token and receives a final bulk status code. + +## Concurrency Controls + +- `--IocpConcurrency`: + - completion-port concurrency level (also aliasable from older `--MaxConcurrency` usage in some paths/scripts). +- `--MaxThreads`: + - max worker threads to create (default: `IocpConcurrency * 2`). +- Per-file async ramp: + - `--InitialPerFileConcurrency` + - `--MaxPerFileConcurrency` + - `--IncreaseConcurrencyAfterMilliseconds`. + +## File I/O Mode + +- IOCP path: + - overlapped writes for generated files + - overlapped keys reads + - pooled intermediate buffers. +- OG path: + - existing memory-mapped behavior preserved. + +## Buffering + +- Direction: NUMA-aware lookaside-style pooling. +- Current state: + - size-class pools (power-of-two) are in place and evolving + - oversize buffer handling exists and needs further hardening/tuning + - optional guard-page behavior is supported for safer debug/development runs. + +## Operational Flags + +- `--WaitForServer` and `--ConnectTimeout=` on client for robust startup coordination. +- `--Verbose` on server gates console output (silent by default). +- `--NoFileIo` supported for rapid compute-only stress loops. + +## Current Status + +- Core IOCP pipeline is functional. +- Small/medium dataset runs are stable. +- Full sys32 runs have succeeded in both `NoFileIo` and file-I/O modes under tested settings. +- Remaining work is mostly: + - correctness hardening for edge/failure paths + - buffer pool policy/memory scaling + - deeper performance tuning and regression automation. + +## See Also + +- Execution log: `IOCP-LOGS.md` +- Active backlog: `IOCP-TOOD.md` +- Session handoff prompt: `IOCP-PROMPT.md` + diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md new file mode 100644 index 00000000..72405c14 --- /dev/null +++ b/IOCP-TOOD.md @@ -0,0 +1,102 @@ +# IOCP TODO + +This is the prioritized, session-ready backlog for the IOCP server/client work. + +## P0: Correctness + Completion Guarantees + +- [ ] Re-verify bulk completion accounting under stress (single token must always signal exactly once): + - `OutstandingWorkItems` + - `PendingCompletions` + - per-node decrement paths + - final request completion transition +- [ ] Add/assert invariants in debug builds for async counters: + - `Job->ActiveGraphs` + - `Context->RemainingSolverLoops` + - `Job->Async.Outstanding` +- [ ] Audit all failure/requeue paths again to ensure every submitted work item has exactly one completion/decrement. +- [ ] Add targeted tests for: + - `PerfectHashAsyncRequeueWork()` failure path + - graph submit failure rollback + - bulk finalization when one node completes last. +- [ ] Confirm server/client connection lifecycle robustness: + - `--WaitForServer` + - `--ConnectTimeout=` + - ping/pong readiness before bulk submission. + +## P0: IOCP-Only Runtime Hygiene + +- [ ] Ensure no legacy threadpool work submission is used by IOCP backend paths (except unavoidable OS-internal TP activity). +- [ ] Keep server silent by default; output only when `--Verbose` is specified. +- [ ] Re-verify `--NoFileIo` on server for fast stress loops and CI-style regression runs. + +## P1: Buffer Pool Redesign Completion (Lookaside/NUMA Style) + +- [ ] Finalize transition to per-NUMA global size-class pools (4KB..16MB default classes). +- [ ] Ensure oversize buffers are pooled/reusable (not leaked one-offs), with guarded-list ownership for teardown. +- [ ] Validate size-class mapping and payload offsets: + - header precedes payload + - `File->BaseAddress` points to payload only + - overrun should fail fast (`PH_RAISE`). +- [ ] Add pool diagnostics (debug-only): + - allocation count + - in-use count + - exhaustion count + - per-class hit/miss. +- [ ] Harden teardown/rundown: + - safe list flush + - safe free after server stop + - no outstanding buffer references. + +## P1: File I/O Pipeline Hardening (Overlapped) + +- [ ] Audit all CHM01 save callbacks for large-write safety with bounded, chunked writes. +- [ ] Add explicit handling for payloads that exceed current buffer capacity: + - predictable flush-and-continue logic + - no undefined pointer arithmetic on overflow. +- [ ] Keep OG memory-mapped path untouched and validated. +- [ ] Re-check ReFS/ADS-related edge cases (table-size stream behavior) and preserve `ADS.md` notes. + +## P1: Tests (Unit + E2E) + +- [ ] Expand unit tests around IOCP buffer pool: + - guard-page mode on/off + - oversize class reuse + - multi-thread pop/push behavior. +- [ ] Add IOCP file-write completion unit tests: + - success path (event signal + buffer return) + - error path (propagated HRESULT + release semantics). +- [ ] Add repeatable E2E matrix scripts: + - `test1`, `hard`, `sys32-200`, `sys32-1000` + - `NoFileIo` and file-I/O modes + - standard ramp presets. + +## P2: Performance Characterization + Tuning + +- [ ] Re-run and capture normalized OG vs IOCP timings (same algo/flags/output drive): + - `Mulshrolate4RX` + - concurrency sets: `4`, `32`, `64`. +- [ ] Sweep per-file ramp knobs: + - `--InitialPerFileConcurrency` + - `--MaxPerFileConcurrency` + - `--IncreaseConcurrencyAfterMilliseconds` +- [ ] Profile system-vs-user overhead and memory footprint at high concurrency. +- [ ] Tune default knobs for balanced throughput and memory use. + +## P2: API/CLI Polish + +- [ ] Keep/confirm naming split: + - `--IocpConcurrency` + - `--MaxThreads` (default `IocpConcurrency * 2`) + - `--MaxPerFileConcurrency` (must be `<= IocpConcurrency`). +- [ ] Validate NUMA targeting options and document examples (all nodes, single node, mask). +- [ ] Decide whether `BulkCreate=` request form should also block on token like `BulkCreateDirectory`. + +## Regression Commands (Baseline) + +- IOCP smoke: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\iocp-smoke.ps1 -BuildDir build-win -Config Release` +- IOCP stress (NoFileIo): + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32-iocp.ps1 -BuildDir build-win -Config Release -IocpConcurrency 32 -MaxThreads 64 -NoFileIo` +- OG stress: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` + diff --git a/USAGE.txt b/USAGE.txt index aa5a9cc4..8caa4a7e 100644 --- a/USAGE.txt +++ b/USAGE.txt @@ -719,6 +719,24 @@ Table Create Parameters: Supplies the maximum number of seconds to try and solve an individual graph. + --InitialPerFileConcurrency=N + + Sets the initial number of graph solver work items to dispatch per + keys file (default: 1). This is useful for IOCP-driven bulk runs + where most key sets solve quickly and only "hard" sets should ramp + up additional concurrency. + + --MaxPerFileConcurrency=N + + Caps the maximum number of graph solver work items per keys file. + Defaults to the maximum concurrency supplied on the command line. + + --IncreaseConcurrencyAfterMilliseconds=N + + When set, and the first solver work item has been running longer than + the supplied interval, an additional solver work item will be queued + (up to --MaxPerFileConcurrency). Defaults to 500ms; use 0 to disable. + --FunctionHookCallbackDllPath= Supplies a fully-qualified path to a .dll file that will be used as the @@ -811,4 +829,3 @@ Mask Functions: ID | Name 2 And - diff --git a/include/PerfectHash.h b/include/PerfectHash.h index 1bfbcdc9..fd3befc5 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -479,6 +479,33 @@ PerfectHashGetCurrentCpuArch( ) \ ) \ \ + ENTRY( \ + ContextIocp, \ + CONTEXT_IOCP, \ + GUID_EX( \ + 0x72bf14d6, 0xa312, 0x4bf5, \ + 0xb0, 0x51, 0x9e, 0xef, 0xa5, 0xfd, 0x8f, 0x59 \ + ) \ + ) \ + \ + ENTRY( \ + Server, \ + SERVER, \ + GUID_EX( \ + 0xbba77b60, 0xb38e, 0x4396, \ + 0xaa, 0x4b, 0xdc, 0xa5, 0x8e, 0xdf, 0xcf, 0x08 \ + ) \ + ) \ + \ + ENTRY( \ + Client, \ + CLIENT, \ + GUID_EX( \ + 0x997e42aa, 0x035f, 0x4d40, \ + 0x82, 0x9f, 0x6c, 0x4c, 0xe2, 0xd2, 0xeb, 0x44 \ + ) \ + ) \ + \ ENTRY( \ Table, \ TABLE, \ @@ -708,7 +735,7 @@ PerfectHashInterfaceGuidToId( for (Index = 1; Index < Count; Index++) { #ifdef __cplusplus const GUID *Entry = PerfectHashInterfaceGuids[Index]; - if (Entry && InlineIsEqualGUID(Guid, Entry)) { + if (Entry && InlineIsEqualGUID(Guid, *Entry)) { Id = (PERFECT_HASH_INTERFACE_ID)Index; break; } @@ -1623,11 +1650,18 @@ typedef union _PERFECT_HASH_FILE_LOAD_FLAGS { ULONG TryLargePagesForFileData:1; + // + // When set, skip mapping the file into memory. The caller is + // responsible for providing a suitable buffer and handling any I/O. + // + + ULONG SkipMapping:1; + // // Unused bits. // - ULONG Unused:31; + ULONG Unused:30; }; LONG AsLong; @@ -1710,11 +1744,19 @@ typedef union _PERFECT_HASH_FILE_CREATE_FLAGS { ULONG EndOfFileIsExtensionSizeIfFileExists:1; + // + // When set, skip creating a memory-mapped view for this file. The + // caller is responsible for providing a suitable buffer and handling + // any I/O. + // + + ULONG SkipMapping:1; + // // Unused bits. // - ULONG Unused:29; + ULONG Unused:28; }; LONG AsLong; @@ -1957,6 +1999,13 @@ typedef union _PERFECT_HASH_KEYS_LOAD_FLAGS { ULONG TryLargePagesForKeysData:1; + // + // When set, keys are loaded using overlapped I/O into a buffer instead + // of memory-mapped views. Intended for IOCP-native contexts. + // + + ULONG UseOverlappedIo:1; + // // When set, skips the verification of keys during loading. // Specifically, skips enumerating all keys and verifying that the @@ -1998,7 +2047,7 @@ typedef union _PERFECT_HASH_KEYS_LOAD_FLAGS { // Unused bits. // - ULONG Unused:28; + ULONG Unused:27; }; LONG AsLong; @@ -2656,7 +2705,20 @@ IsModulusMasking( // PERFECT_HASH_TABLE interface's creation routine. // +// +// Define NUMA node mask handling. A mask of all 1s indicates "all nodes"; +// callers should intersect it with the machine's available NUMA nodes. +// + +typedef ULONGLONG PERFECT_HASH_NUMA_NODE_MASK; +typedef PERFECT_HASH_NUMA_NODE_MASK *PPERFECT_HASH_NUMA_NODE_MASK; + +#define PERFECT_HASH_NUMA_NODE_MASK_ALL ((PERFECT_HASH_NUMA_NODE_MASK)~0ULL) + DECLARE_COMPONENT(Context, PERFECT_HASH_CONTEXT); +DECLARE_COMPONENT(ContextIocp, PERFECT_HASH_CONTEXT_IOCP); +DECLARE_COMPONENT(Server, PERFECT_HASH_SERVER); +DECLARE_COMPONENT(Client, PERFECT_HASH_CLIENT); typedef HRESULT @@ -3723,6 +3785,9 @@ IsValidPerfectHashCuRngId( ENTRY(Seed3Byte1MaskCounts) \ ENTRY(Seed3Byte2MaskCounts) \ ENTRY(MaxSolveTimeInSeconds) \ + ENTRY(InitialPerFileConcurrency) \ + ENTRY(MaxPerFileConcurrency) \ + ENTRY(IncreaseConcurrencyAfterMilliseconds) \ ENTRY(AutoResizeWhenKeysToEdgesRatioExceeds) \ ENTRY(FunctionHookCallbackDllPath) \ ENTRY(FunctionHookCallbackFunctionName) \ @@ -4210,6 +4275,609 @@ typedef struct _PERFECT_HASH_CONTEXT { typedef PERFECT_HASH_CONTEXT *PPERFECT_HASH_CONTEXT; #endif +// +// Define the PERFECT_HASH_CONTEXT_IOCP interface. +// + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG MaximumConcurrency + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Out_ PULONG MaximumConcurrency + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG MaximumThreads + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS + *PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Out_ PULONG MaximumThreads + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS + *PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK + *PPERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Out_ PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK + *PPERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PCUNICODE_STRING BaseOutputDirectory + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY + *PPERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Inout_ PPERFECT_HASH_DIRECTORY *BaseOutputDirectory + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY + *PPERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PCUNICODE_STRING KeysDirectory, + _In_ PCUNICODE_STRING BaseOutputDirectory, + _In_ PERFECT_HASH_ALGORITHM_ID AlgorithmId, + _In_ PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _In_ PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _In_opt_ PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + _In_opt_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _In_opt_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _In_opt_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_opt_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE + *PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW + ); +typedef PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW, + _In_ PUNICODE_STRING KeysDirectory, + _In_ PUNICODE_STRING BaseOutputDirectory, + _Inout_ PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + _Inout_ PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _Inout_ PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _Inout_ PULONG MaximumConcurrency, + _Inout_ PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + _Inout_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _Inout_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _Inout_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PCUNICODE_STRING KeysPath, + _In_ PCUNICODE_STRING BaseOutputDirectory, + _In_ PERFECT_HASH_ALGORITHM_ID AlgorithmId, + _In_ PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _In_ PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _In_opt_ PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS + ContextTableCreateFlags, + _In_opt_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _In_opt_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _In_opt_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE + *PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW + ); +typedef PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE + PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW, + _In_ PUNICODE_STRING KeysPath, + _In_ PUNICODE_STRING BaseOutputDirectory, + _Inout_ PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + _Inout_ PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _Inout_ PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _Inout_ PULONG MaximumConcurrency, + _Inout_ PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS + ContextTableCreateFlags, + _Inout_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _Inout_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _Inout_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPSTR *ArgvA + ); +typedef PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA + *PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPSTR *ArgvA + ); +typedef PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA + *PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA; + +typedef struct _PERFECT_HASH_CONTEXT_IOCP_VTBL { + DECLARE_COMPONENT_VTBL_HEADER(PERFECT_HASH_CONTEXT_IOCP); + + PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY SetMaximumConcurrency; + PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY GetMaximumConcurrency; + PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS SetMaximumThreads; + PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS GetMaximumThreads; + PPERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK SetNumaNodeMask; + PPERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK GetNumaNodeMask; + PPERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY SetBaseOutputDirectory; + PPERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY GetBaseOutputDirectory; + + PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE BulkCreate; + PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW BulkCreateArgvW; + PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + ExtractBulkCreateArgsFromArgvW; + + PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE TableCreate; + + PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW TableCreateArgvW; + PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + ExtractTableCreateArgsFromArgvW; + + // + // N.B. These two routines will only be present on non-Windows platforms. + // + + PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA TableCreateArgvA; + PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA BulkCreateArgvA; + +} PERFECT_HASH_CONTEXT_IOCP_VTBL; +typedef PERFECT_HASH_CONTEXT_IOCP_VTBL *PPERFECT_HASH_CONTEXT_IOCP_VTBL; + +#ifndef _PERFECT_HASH_INTERNAL_BUILD +typedef struct _PERFECT_HASH_CONTEXT_IOCP { + PPERFECT_HASH_CONTEXT_IOCP_VTBL Vtbl; +} PERFECT_HASH_CONTEXT_IOCP; +typedef PERFECT_HASH_CONTEXT_IOCP *PPERFECT_HASH_CONTEXT_IOCP; +#endif + +// +// Define the PERFECT_HASH_SERVER interface. +// + +typedef enum _PERFECT_HASH_SERVER_REQUEST_TYPE { + PerfectHashNullServerRequestType = 0, + PerfectHashTableCreateServerRequestType, + PerfectHashBulkCreateServerRequestType, + PerfectHashBulkCreateDirectoryServerRequestType, + PerfectHashPingServerRequestType, + PerfectHashShutdownServerRequestType, + PerfectHashInvalidServerRequestType +} PERFECT_HASH_SERVER_REQUEST_TYPE; + +FORCEINLINE +BOOLEAN +IsValidPerfectHashServerRequestType( + _In_ PERFECT_HASH_SERVER_REQUEST_TYPE RequestType + ) +{ + return ( + RequestType > PerfectHashNullServerRequestType && + RequestType < PerfectHashInvalidServerRequestType + ); +} + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_SERVER_REQUEST { + ULONG SizeOfStruct; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + ULONG Padding1; + ULONG Padding2; + ULONGLONG RequestId; + UNICODE_STRING CommandLine; +} PERFECT_HASH_SERVER_REQUEST; +typedef PERFECT_HASH_SERVER_REQUEST *PPERFECT_HASH_SERVER_REQUEST; + +// +// Wire protocol structures for client/server IPC. +// + +#define PERFECT_HASH_SERVER_MESSAGE_VERSION 1 +#define PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE (1024 * 1024) + +#define PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE 0x00000001 +#define PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN 0x00000002 +#define PERFECT_HASH_SERVER_RESPONSE_FLAG_PONG 0x00000004 +#define PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT \ + L"EventHandle=%llu ResultHandle=%llu" + +#define PERFECT_HASH_SERVER_BULK_RESULT_VERSION 1 + +typedef struct _Struct_size_bytes_(SizeOfStruct) +_PERFECT_HASH_SERVER_BULK_RESULT { + ULONG SizeOfStruct; + ULONG Version; + HRESULT Result; + ULONG Flags; + ULONG TotalFiles; + ULONG SucceededFiles; + ULONG FailedFiles; + HRESULT FirstFailure; +} PERFECT_HASH_SERVER_BULK_RESULT; +typedef PERFECT_HASH_SERVER_BULK_RESULT *PPERFECT_HASH_SERVER_BULK_RESULT; + +typedef struct _Struct_size_bytes_(SizeOfStruct) +_PERFECT_HASH_SERVER_REQUEST_HEADER { + ULONG SizeOfStruct; + ULONG Version; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + ULONG Flags; + ULONGLONG RequestId; + ULONG PayloadLength; + ULONG Reserved; +} PERFECT_HASH_SERVER_REQUEST_HEADER; +typedef PERFECT_HASH_SERVER_REQUEST_HEADER + *PPERFECT_HASH_SERVER_REQUEST_HEADER; + +typedef struct _Struct_size_bytes_(SizeOfStruct) +_PERFECT_HASH_SERVER_RESPONSE_HEADER { + ULONG SizeOfStruct; + ULONG Version; + HRESULT Result; + ULONG Flags; + ULONGLONG RequestId; + ULONG PayloadLength; + ULONG Reserved; +} PERFECT_HASH_SERVER_RESPONSE_HEADER; +typedef PERFECT_HASH_SERVER_RESPONSE_HEADER + *PPERFECT_HASH_SERVER_RESPONSE_HEADER; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ ULONG MaximumConcurrency + ); +typedef PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PULONG MaximumConcurrency + ); +typedef PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ ULONG MaximumThreads + ); +typedef PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS + *PPERFECT_HASH_SERVER_SET_MAXIMUM_THREADS; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PULONG MaximumThreads + ); +typedef PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS + *PPERFECT_HASH_SERVER_GET_MAXIMUM_THREADS; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK + *PPERFECT_HASH_SERVER_SET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK + *PPERFECT_HASH_SERVER_GET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_ENDPOINT)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ PCUNICODE_STRING Endpoint + ); +typedef PERFECT_HASH_SERVER_SET_ENDPOINT + *PPERFECT_HASH_SERVER_SET_ENDPOINT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_ENDPOINT)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PUNICODE_STRING Endpoint + ); +typedef PERFECT_HASH_SERVER_GET_ENDPOINT + *PPERFECT_HASH_SERVER_GET_ENDPOINT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_LOCAL_ONLY)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN LocalOnly + ); +typedef PERFECT_HASH_SERVER_SET_LOCAL_ONLY + *PPERFECT_HASH_SERVER_SET_LOCAL_ONLY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_LOCAL_ONLY)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN LocalOnly + ); +typedef PERFECT_HASH_SERVER_GET_LOCAL_ONLY + *PPERFECT_HASH_SERVER_GET_LOCAL_ONLY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_VERBOSE)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN Verbose + ); +typedef PERFECT_HASH_SERVER_SET_VERBOSE + *PPERFECT_HASH_SERVER_SET_VERBOSE; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_VERBOSE)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN Verbose + ); +typedef PERFECT_HASH_SERVER_GET_VERBOSE + *PPERFECT_HASH_SERVER_GET_VERBOSE; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_NO_FILE_IO)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN NoFileIo + ); +typedef PERFECT_HASH_SERVER_SET_NO_FILE_IO + *PPERFECT_HASH_SERVER_SET_NO_FILE_IO; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_NO_FILE_IO)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN NoFileIo + ); +typedef PERFECT_HASH_SERVER_GET_NO_FILE_IO + *PPERFECT_HASH_SERVER_GET_NO_FILE_IO; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN GuardPages + ); +typedef PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + *PPERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN GuardPages + ); +typedef PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + *PPERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_START)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_START *PPERFECT_HASH_SERVER_START; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_STOP)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_STOP *PPERFECT_HASH_SERVER_STOP; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_WAIT)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_WAIT *PPERFECT_HASH_SERVER_WAIT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SUBMIT_REQUEST)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ PPERFECT_HASH_SERVER_REQUEST Request + ); +typedef PERFECT_HASH_SERVER_SUBMIT_REQUEST + *PPERFECT_HASH_SERVER_SUBMIT_REQUEST; + +typedef struct _PERFECT_HASH_SERVER_VTBL { + DECLARE_COMPONENT_VTBL_HEADER(PERFECT_HASH_SERVER); + + PPERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY SetMaximumConcurrency; + PPERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY GetMaximumConcurrency; + PPERFECT_HASH_SERVER_SET_MAXIMUM_THREADS SetMaximumThreads; + PPERFECT_HASH_SERVER_GET_MAXIMUM_THREADS GetMaximumThreads; + PPERFECT_HASH_SERVER_SET_NUMA_NODE_MASK SetNumaNodeMask; + PPERFECT_HASH_SERVER_GET_NUMA_NODE_MASK GetNumaNodeMask; + PPERFECT_HASH_SERVER_SET_ENDPOINT SetEndpoint; + PPERFECT_HASH_SERVER_GET_ENDPOINT GetEndpoint; + PPERFECT_HASH_SERVER_SET_LOCAL_ONLY SetLocalOnly; + PPERFECT_HASH_SERVER_GET_LOCAL_ONLY GetLocalOnly; + PPERFECT_HASH_SERVER_SET_VERBOSE SetVerbose; + PPERFECT_HASH_SERVER_GET_VERBOSE GetVerbose; + PPERFECT_HASH_SERVER_SET_NO_FILE_IO SetNoFileIo; + PPERFECT_HASH_SERVER_GET_NO_FILE_IO GetNoFileIo; + PPERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + SetIocpBufferGuardPages; + PPERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + GetIocpBufferGuardPages; + PPERFECT_HASH_SERVER_START Start; + PPERFECT_HASH_SERVER_STOP Stop; + PPERFECT_HASH_SERVER_WAIT Wait; + PPERFECT_HASH_SERVER_SUBMIT_REQUEST SubmitRequest; +} PERFECT_HASH_SERVER_VTBL; +typedef PERFECT_HASH_SERVER_VTBL *PPERFECT_HASH_SERVER_VTBL; + +#ifndef _PERFECT_HASH_INTERNAL_BUILD +typedef struct _PERFECT_HASH_SERVER { + PPERFECT_HASH_SERVER_VTBL Vtbl; +} PERFECT_HASH_SERVER; +typedef PERFECT_HASH_SERVER *PPERFECT_HASH_SERVER; +#endif + +// +// Define the PERFECT_HASH_CLIENT interface. +// + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_CONNECT)( + _In_ PPERFECT_HASH_CLIENT Client, + _In_opt_ PCUNICODE_STRING Endpoint + ); +typedef PERFECT_HASH_CLIENT_CONNECT *PPERFECT_HASH_CLIENT_CONNECT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_DISCONNECT)( + _In_ PPERFECT_HASH_CLIENT Client + ); +typedef PERFECT_HASH_CLIENT_DISCONNECT *PPERFECT_HASH_CLIENT_DISCONNECT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_SUBMIT_REQUEST)( + _In_ PPERFECT_HASH_CLIENT Client, + _In_ PPERFECT_HASH_SERVER_REQUEST Request + ); +typedef PERFECT_HASH_CLIENT_SUBMIT_REQUEST + *PPERFECT_HASH_CLIENT_SUBMIT_REQUEST; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_GET_LAST_RESPONSE)( + _In_ PPERFECT_HASH_CLIENT Client, + _Out_ PUNICODE_STRING ResponsePayload, + _Out_opt_ PULONG ResponseFlags + ); +typedef PERFECT_HASH_CLIENT_GET_LAST_RESPONSE + *PPERFECT_HASH_CLIENT_GET_LAST_RESPONSE; + +typedef struct _PERFECT_HASH_CLIENT_VTBL { + DECLARE_COMPONENT_VTBL_HEADER(PERFECT_HASH_CLIENT); + + PPERFECT_HASH_CLIENT_CONNECT Connect; + PPERFECT_HASH_CLIENT_DISCONNECT Disconnect; + PPERFECT_HASH_CLIENT_SUBMIT_REQUEST SubmitRequest; + PPERFECT_HASH_CLIENT_GET_LAST_RESPONSE GetLastResponse; +} PERFECT_HASH_CLIENT_VTBL; +typedef PERFECT_HASH_CLIENT_VTBL *PPERFECT_HASH_CLIENT_VTBL; + +#ifndef _PERFECT_HASH_INTERNAL_BUILD +typedef struct _PERFECT_HASH_CLIENT { + PPERFECT_HASH_CLIENT_VTBL Vtbl; +} PERFECT_HASH_CLIENT; +typedef PERFECT_HASH_CLIENT *PPERFECT_HASH_CLIENT; +#endif + // // Define the PERFECT_HASH_TABLE interface. // diff --git a/include/PerfectHashErrors.h b/include/PerfectHashErrors.h index 46c5cf90..5a08ca25 100644 --- a/include/PerfectHashErrors.h +++ b/include/PerfectHashErrors.h @@ -189,6 +189,15 @@ Module Name: // #define PH_S_CU_KERNEL_RUNTIME_TARGET_REACHED ((HRESULT)0xE004000EL) +// +// MessageId: PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED +// +// MessageText: +// +// Bulk create request completed successfully. +// +#define PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED ((HRESULT)0x2004000FL) + //////////////////////////////////////////////////////////////////////////////// // PH_SEVERITY_INFORMATIONAL @@ -2538,7 +2547,16 @@ Module Name: #define PH_E_ERROR_DURING_PREPARE_VCPROPS_COMPILED_PERFECT_HASH_FILE ((HRESULT)0xE004028FL) // -// Spare IDs: 0x290, 0x291. +// MessageId: PH_E_SERVER_BULK_CREATE_FAILED +// +// MessageText: +// +// Bulk create request failed. +// +#define PH_E_SERVER_BULK_CREATE_FAILED ((HRESULT)0xE0040290L) + +// +// Spare ID: 0x291. // // // MessageId: PH_E_ERROR_DURING_SAVE_VCPROJECT_DLL_FILE diff --git a/scripts/iocp-smoke.ps1 b/scripts/iocp-smoke.ps1 new file mode 100644 index 00000000..2ea04c9f --- /dev/null +++ b/scripts/iocp-smoke.ps1 @@ -0,0 +1,135 @@ +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$Endpoint = "\\.\\pipe\\PerfectHashServer-Smoke", + [int]$TimeoutSeconds = 10, + [int]$IocpConcurrency = 0, + [int]$MaxThreads = 0, + [bool]$WaitForServer = $true, + [int]$ConnectTimeoutMs = 10000, + [bool]$NoFileIo = $true, + [bool]$VerboseServer = $false +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Remove-Item Env:PH_LOG_BULK_CREATE_COUNTS -ErrorAction SilentlyContinue +Remove-Item Env:PH_LOG_CHM01_ASYNC_JOB -ErrorAction SilentlyContinue + +$serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) +$clientExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashClient.exe" -f $Config) +$workDir = Join-Path $BuildDir "iocp-smoke" +$keysFile = Join-Path $workDir "smoke_keys_16.keys" +$outputDir = Join-Path $workDir "output" + +if (-not (Test-Path $serverExe)) { + throw "Server exe not found: $serverExe" +} + +if (-not (Test-Path $clientExe)) { + throw "Client exe not found: $clientExe" +} + +New-Item -ItemType Directory -Path $workDir -Force | Out-Null +New-Item -ItemType Directory -Path $outputDir -Force | Out-Null + +$keys = 1..128 +$stream = [System.IO.File]::Open($keysFile, + [System.IO.FileMode]::Create, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None) +$writer = New-Object System.IO.BinaryWriter($stream) +foreach ($key in $keys) { + $writer.Write([UInt32]$key) +} +$writer.Flush() +$writer.Dispose() +$stream.Dispose() + +$keysPath = (Resolve-Path $keysFile).Path +$outputPath = (Resolve-Path $outputDir).Path + +$createCommand = "PerfectHashCreate.exe $keysPath $outputPath " + + "Chm01 Mulshrolate1RX And 0 --DisableCsvOutputFile" +$tableArg = "--TableCreate=$createCommand" + +$serverArgs = @("--Endpoint=$Endpoint") +if ($IocpConcurrency -gt 0) { + $serverArgs += ("--IocpConcurrency={0}" -f $IocpConcurrency) +} +if ($MaxThreads -gt 0) { + $serverArgs += ("--MaxThreads={0}" -f $MaxThreads) +} +if ($NoFileIo) { + $serverArgs += "--NoFileIo" +} +if ($VerboseServer) { + $serverArgs += "--Verbose" +} +$clientArgs = @("--Endpoint=$Endpoint", "--Shutdown") +if ($WaitForServer) { + $clientArgs += "--WaitForServer" + if ($ConnectTimeoutMs -gt 0) { + $clientArgs += ("--ConnectTimeout={0}" -f $ConnectTimeoutMs) + } +} + +$server = Start-Process -FilePath $serverExe ` + -ArgumentList $serverArgs ` + -PassThru ` + -NoNewWindow + +Start-Sleep -Milliseconds 500 + +$clientCreateArgs = @("--Endpoint=$Endpoint", "`"$tableArg`"") +if ($WaitForServer) { + $clientCreateArgs += "--WaitForServer" + if ($ConnectTimeoutMs -gt 0) { + $clientCreateArgs += ("--ConnectTimeout={0}" -f $ConnectTimeoutMs) + } +} + +$clientCreate = Start-Process -FilePath $clientExe ` + -ArgumentList $clientCreateArgs ` + -PassThru ` + -NoNewWindow ` + -Wait + +if ($clientCreate.ExitCode -ne 0) { + throw ("Client create exited with code {0}" -f $clientCreate.ExitCode) +} + +$clientShutdown = Start-Process -FilePath $clientExe ` + -ArgumentList $clientArgs ` + -PassThru ` + -NoNewWindow ` + -Wait + +if ($clientShutdown.ExitCode -ne 0) { + throw ("Client shutdown exited with code {0}" -f $clientShutdown.ExitCode) +} + +if (-not $server.WaitForExit($TimeoutSeconds * 1000)) { + $server.Kill() + throw ("Server did not exit within {0}s" -f $TimeoutSeconds) +} + +$server.WaitForExit() +$server.Refresh() + +if ($null -eq $server.ExitCode) { + $serverProcess = Get-Process -Id $server.Id -ErrorAction SilentlyContinue + if ($null -ne $serverProcess) { + throw "Server exit code unavailable after wait." + } + $serverExitCode = 0 +} else { + $serverExitCode = $server.ExitCode +} + +if ($serverExitCode -ne 0) { + throw ("Server exited with code {0}" -f $serverExitCode) +} + +Write-Host "IOCP smoke test succeeded." diff --git a/scripts/stress-sys32-iocp.ps1 b/scripts/stress-sys32-iocp.ps1 new file mode 100644 index 00000000..ee96d349 --- /dev/null +++ b/scripts/stress-sys32-iocp.ps1 @@ -0,0 +1,136 @@ +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$Endpoint = "\\.\\pipe\\PerfectHashServer-StressSys32", + [string]$KeysDir = "..\\perfecthash-keys\\sys32", + [string]$OutputDir = "build-win\\iocp-stress-sys32", + [string]$Algorithm = "Chm01", + [string]$HashFunction = "Mulshrolate4RX", + [string]$MaskFunction = "And", + [int]$MaximumConcurrency = 0, + [int]$IocpConcurrency = 0, + [int]$MaxThreads = 0, + [string[]]$ExtraArgs = @(), + [int]$TimeoutSeconds = 300, + [bool]$WaitForServer = $true, + [int]$ConnectTimeoutMs = 10000, + [bool]$NoFileIo = $true, + [bool]$VerboseServer = $false +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Remove-Item Env:PH_LOG_BULK_CREATE_COUNTS -ErrorAction SilentlyContinue +Remove-Item Env:PH_LOG_CHM01_ASYNC_JOB -ErrorAction SilentlyContinue + +$serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) +$clientExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashClient.exe" -f $Config) + +if (-not (Test-Path $serverExe)) { + throw "Server exe not found: $serverExe" +} + +if (-not (Test-Path $clientExe)) { + throw "Client exe not found: $clientExe" +} + +if (-not (Test-Path $KeysDir)) { + throw "Keys dir not found: $KeysDir" +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +$keysPath = (Resolve-Path $KeysDir).Path +$outputPath = (Resolve-Path $OutputDir).Path + +if ($MaximumConcurrency -le 0) { + $MaximumConcurrency = [Environment]::ProcessorCount +} + +$bulkCommand = ('"{0}" "{1}" {2} {3} {4} {5}' -f $keysPath, + $outputPath, + $Algorithm, + $HashFunction, + $MaskFunction, + $MaximumConcurrency) + +if ($ExtraArgs -and $ExtraArgs.Count -gt 0) { + $bulkCommand += " " + ($ExtraArgs -join " ") +} + +$bulkArg = "--BulkCreateDirectory=$bulkCommand" + +$serverArgs = @("--Endpoint=$Endpoint") +if ($IocpConcurrency -gt 0) { + $serverArgs += ("--IocpConcurrency={0}" -f $IocpConcurrency) +} +if ($MaxThreads -gt 0) { + $serverArgs += ("--MaxThreads={0}" -f $MaxThreads) +} +if ($NoFileIo) { + $serverArgs += "--NoFileIo" +} +if ($VerboseServer) { + $serverArgs += "--Verbose" +} + +$server = Start-Process -FilePath $serverExe ` + -ArgumentList $serverArgs ` + -PassThru ` + -NoNewWindow + +$clientArgs = @("--Endpoint=$Endpoint") +if ($WaitForServer) { + $clientArgs += "--WaitForServer" + if ($ConnectTimeoutMs -gt 0) { + $clientArgs += ("--ConnectTimeout={0}" -f $ConnectTimeoutMs) + } +} + +$clientCreateArgs = @($clientArgs + "`"$bulkArg`"") +$clientCreate = Start-Process -FilePath $clientExe ` + -ArgumentList $clientCreateArgs ` + -PassThru ` + -NoNewWindow ` + -Wait + +$bulkSuccess = 0x2004000F +if ($clientCreate.ExitCode -ne 0 -and $clientCreate.ExitCode -ne $bulkSuccess) { + throw ("Client bulk-create exited with code {0}" -f $clientCreate.ExitCode) +} + +$clientShutdownArgs = @($clientArgs + "--Shutdown") +$clientShutdown = Start-Process -FilePath $clientExe ` + -ArgumentList $clientShutdownArgs ` + -PassThru ` + -NoNewWindow ` + -Wait + +if ($clientShutdown.ExitCode -ne 0) { + throw ("Client shutdown exited with code {0}" -f $clientShutdown.ExitCode) +} + +if (-not $server.WaitForExit($TimeoutSeconds * 1000)) { + $server.Kill() + throw ("Server did not exit within {0}s" -f $TimeoutSeconds) +} + +$server.WaitForExit() +$server.Refresh() + +if ($null -eq $server.ExitCode) { + $serverProcess = Get-Process -Id $server.Id -ErrorAction SilentlyContinue + if ($null -ne $serverProcess) { + throw "Server exit code unavailable after wait." + } + $serverExitCode = 0 +} else { + $serverExitCode = $server.ExitCode +} + +if ($serverExitCode -ne 0) { + throw ("Server exited with code {0}" -f $serverExitCode) +} + +Write-Host "IOCP stress test complete." diff --git a/scripts/stress-sys32.ps1 b/scripts/stress-sys32.ps1 new file mode 100644 index 00000000..2ab7fd84 --- /dev/null +++ b/scripts/stress-sys32.ps1 @@ -0,0 +1,51 @@ +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$KeysDir = "..\\perfecthash-keys\\sys32", + [string]$OutputDir = "build-win\\stress-sys32", + [string]$Algorithm = "Chm01", + [string]$HashFunction = "Mulshrolate4RX", + [string]$MaskFunction = "And", + [int]$MaximumConcurrency = 0, + [string[]]$ExtraArgs = @() +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$exe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashBulkCreate.exe" -f $Config) + +if (-not (Test-Path $exe)) { + throw "Bulk create exe not found: $exe" +} + +if (-not (Test-Path $KeysDir)) { + throw "Keys dir not found: $KeysDir" +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +$keysPath = (Resolve-Path $KeysDir).Path +$outputPath = (Resolve-Path $OutputDir).Path + +if ($MaximumConcurrency -le 0) { + $MaximumConcurrency = [Environment]::ProcessorCount +} + +$args = @($keysPath, + $outputPath, + $Algorithm, + $HashFunction, + $MaskFunction, + $MaximumConcurrency) + +if ($ExtraArgs -and $ExtraArgs.Count -gt 0) { + $args += $ExtraArgs +} + +& $exe @args +if ($LASTEXITCODE -ne 0) { + throw ("Bulk create exited with code {0}" -f $LASTEXITCODE) +} + +Write-Host "Stress test complete." diff --git a/scripts/stress-sys32.sh b/scripts/stress-sys32.sh new file mode 100644 index 00000000..109eb240 --- /dev/null +++ b/scripts/stress-sys32.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUILD_DIR="${BUILD_DIR:-build}" +CONFIG="${CONFIG:-Debug}" +KEYS_DIR="${KEYS_DIR:-../perfecthash-keys/sys32}" +OUTPUT_DIR="${OUTPUT_DIR:-${BUILD_DIR}/stress-sys32}" +ALGORITHM="${ALGORITHM:-Chm01}" +HASH_FUNCTION="${HASH_FUNCTION:-Mulshrolate4RX}" +MASK_FUNCTION="${MASK_FUNCTION:-And}" + +if command -v nproc >/dev/null 2>&1; then + MAX_CONCURRENCY="${MAX_CONCURRENCY:-$(nproc)}" +else + MAX_CONCURRENCY="${MAX_CONCURRENCY:-1}" +fi + +EXE="${BUILD_DIR}/bin/${CONFIG}/PerfectHashBulkCreate" +if [[ ! -x "${EXE}" ]]; then + EXE="${BUILD_DIR}/bin/PerfectHashBulkCreate" +fi + +if [[ ! -x "${EXE}" ]]; then + echo "Bulk create exe not found: ${EXE}" >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" + +"${EXE}" \ + "${KEYS_DIR}" \ + "${OUTPUT_DIR}" \ + "${ALGORITHM}" \ + "${HASH_FUNCTION}" \ + "${MASK_FUNCTION}" \ + "${MAX_CONCURRENCY}" \ + "$@" diff --git a/scripts/test-minidump.ps1 b/scripts/test-minidump.ps1 new file mode 100644 index 00000000..43520eb0 --- /dev/null +++ b/scripts/test-minidump.ps1 @@ -0,0 +1,128 @@ +# Copyright (c) 2026 Trent Nelson + +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$OutputDir = "build-win\\minidump-test", + [switch]$ForceFallback +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$bulkExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashBulkCreate.exe" -f $Config) +$serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) + +if (-not (Test-Path $bulkExe)) { + throw "Bulk-create exe not found: $bulkExe" +} + +if (-not (Test-Path $serverExe)) { + throw "Server exe not found: $serverExe" +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null +$dumpDir = (Resolve-Path $OutputDir).Path + +$bulkDump = Join-Path $dumpDir "PerfectHashBulkCreateCrash.dmp" +$bulkLog = Join-Path $dumpDir "PerfectHashBulkCreateCrash.log" +$serverDump = Join-Path $dumpDir "PerfectHashServerCrash.dmp" +$serverLog = Join-Path $dumpDir "PerfectHashServerCrash.log" + +Remove-Item $bulkDump, $bulkLog, $serverDump, $serverLog -ErrorAction SilentlyContinue + +$envSnapshot = @{ + PH_LOG_BULK_CREATE_CRASH = $env:PH_LOG_BULK_CREATE_CRASH + PH_BULK_CREATE_CRASH_TEST = $env:PH_BULK_CREATE_CRASH_TEST + PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK = $env:PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK + PH_BULK_CREATE_CRASH_DIR = $env:PH_BULK_CREATE_CRASH_DIR + PH_LOG_SERVER_CRASH = $env:PH_LOG_SERVER_CRASH + PH_SERVER_CRASH_TEST = $env:PH_SERVER_CRASH_TEST + PH_SERVER_MINIDUMP_FORCE_FALLBACK = $env:PH_SERVER_MINIDUMP_FORCE_FALLBACK + PH_SERVER_CRASH_DIR = $env:PH_SERVER_CRASH_DIR +} + +try { + $env:PH_BULK_CREATE_CRASH_DIR = $dumpDir + $env:PH_LOG_BULK_CREATE_CRASH = "1" + $env:PH_BULK_CREATE_CRASH_TEST = "AV" + if ($ForceFallback) { + $env:PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK = "1" + } else { + Remove-Item Env:PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK ` + -ErrorAction SilentlyContinue + } + + $bulkProcess = Start-Process -FilePath $bulkExe ` + -PassThru ` + -NoNewWindow ` + -Wait + + if ($bulkProcess.ExitCode -eq 0) { + throw ("Bulk-create crash test exited with code {0}" -f ` + $bulkProcess.ExitCode) + } + + if (-not (Test-Path $bulkDump)) { + throw "Bulk-create minidump not found." + } + + if ((Get-Item $bulkDump).Length -eq 0) { + throw "Bulk-create minidump is empty." + } + + if (-not (Test-Path $bulkLog)) { + throw "Bulk-create crash log not found." + } + + if ((Get-Item $bulkLog).Length -eq 0) { + throw "Bulk-create crash log is empty." + } + + $env:PH_SERVER_CRASH_DIR = $dumpDir + $env:PH_LOG_SERVER_CRASH = "1" + $env:PH_SERVER_CRASH_TEST = "AV" + if ($ForceFallback) { + $env:PH_SERVER_MINIDUMP_FORCE_FALLBACK = "1" + } else { + Remove-Item Env:PH_SERVER_MINIDUMP_FORCE_FALLBACK ` + -ErrorAction SilentlyContinue + } + + $serverProcess = Start-Process -FilePath $serverExe ` + -PassThru ` + -NoNewWindow ` + -Wait + + if ($serverProcess.ExitCode -eq 0) { + throw ("Server crash test exited with code {0}" -f ` + $serverProcess.ExitCode) + } + + if (-not (Test-Path $serverDump)) { + throw "Server minidump not found." + } + + if ((Get-Item $serverDump).Length -eq 0) { + throw "Server minidump is empty." + } + + if (-not (Test-Path $serverLog)) { + throw "Server crash log not found." + } + + if ((Get-Item $serverLog).Length -eq 0) { + throw "Server crash log is empty." + } +} finally { + foreach ($name in $envSnapshot.Keys) { + $value = $envSnapshot[$name] + if ($null -eq $value) { + Remove-Item ("Env:{0}" -f $name) -ErrorAction SilentlyContinue + } else { + Set-Item ("Env:{0}" -f $name) -Value $value + } + } +} + +Write-Host "Minidump test succeeded." diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5d56c1dd..c7c42669 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,6 +27,8 @@ endif() add_subdirectory(PerfectHash) add_subdirectory(PerfectHashCreateExe) add_subdirectory(PerfectHashBulkCreateExe) +add_subdirectory(PerfectHashServerExe) +add_subdirectory(PerfectHashClientExe) if(PERFECTHASH_HAS_CUDA) add_subdirectory(PerfectHashCuda) @@ -37,6 +39,8 @@ if(WIN32) add_dependencies(PerfectHash PerfectHashResources) add_dependencies(PerfectHashCreateExe PerfectHashResources) add_dependencies(PerfectHashBulkCreateExe PerfectHashResources) + add_dependencies(PerfectHashServerExe PerfectHashResources) + add_dependencies(PerfectHashClientExe PerfectHashResources) if(PERFECTHASH_HAS_CUDA) add_dependencies(PerfectHashCuda PerfectHashResources) endif() diff --git a/src/PerfectHash.sln b/src/PerfectHash.sln index 810ca625..a8da928c 100644 --- a/src/PerfectHash.sln +++ b/src/PerfectHash.sln @@ -28,6 +28,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerfectHashCreateExe", "Per {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} = {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerfectHashServerExe", "PerfectHashServerExe\PerfectHashServerExe.vcxproj", "{B269AB7F-EF35-490A-B66D-031E431C4481}" + ProjectSection(ProjectDependencies) = postProject + {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} = {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerfectHashClientExe", "PerfectHashClientExe\PerfectHashClientExe.vcxproj", "{EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}" + ProjectSection(ProjectDependencies) = postProject + {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} = {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FunctionHook", "FunctionHook\FunctionHook.vcxproj", "{B9EF04DA-0AD9-45C4-97B0-857A13AA4850}" EndProject EndProject @@ -70,6 +80,26 @@ Global {244D25EC-A8AA-4AAD-A66C-D64AFA836CAE}.PGUpdate|x64.Build.0 = PGUpdate|x64 {244D25EC-A8AA-4AAD-A66C-D64AFA836CAE}.Release|x64.ActiveCfg = Release|x64 {244D25EC-A8AA-4AAD-A66C-D64AFA836CAE}.Release|x64.Build.0 = Release|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Debug|x64.ActiveCfg = Debug|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Debug|x64.Build.0 = Debug|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGOptimize|x64.ActiveCfg = PGOptimize|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGOptimize|x64.Build.0 = PGOptimize|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Release|x64.ActiveCfg = Release|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Release|x64.Build.0 = Release|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Debug|x64.ActiveCfg = Debug|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Debug|x64.Build.0 = Debug|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGOptimize|x64.ActiveCfg = PGOptimize|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGOptimize|x64.Build.0 = PGOptimize|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Release|x64.ActiveCfg = Release|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Release|x64.Build.0 = Release|x64 {B9EF04DA-0AD9-45C4-97B0-857A13AA4850}.Debug|x64.ActiveCfg = Debug|x64 {B9EF04DA-0AD9-45C4-97B0-857A13AA4850}.Debug|x64.Build.0 = Debug|x64 {B9EF04DA-0AD9-45C4-97B0-857A13AA4850}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 diff --git a/src/PerfectHash/CMakeLists.txt b/src/PerfectHash/CMakeLists.txt index ddb75365..1702a151 100644 --- a/src/PerfectHash/CMakeLists.txt +++ b/src/PerfectHash/CMakeLists.txt @@ -8,6 +8,7 @@ set(Private_Header_Files "BulkCreateBestCsv.h" "BulkCreateCsv.h" "Chm01.h" + "Chm01Async.h" "Chm01FileWork.h" "Chm01Private.h" "Chm02Private.h" @@ -25,7 +26,11 @@ set(Private_Header_Files "PerfectHashAllocator.h" "PerfectHashConstants.h" "PerfectHashContext.h" + "PerfectHashContextIocp.h" + "PerfectHashIocpBufferPool.h" + "PerfectHashAsync.h" "PerfectHashCu.h" + "PerfectHashClient.h" "PerfectHashDirectory.h" "PerfectHashErrorHandling.h" "PerfectHashEventsPrivate.h" @@ -36,6 +41,7 @@ set(Private_Header_Files "PerfectHashPath.h" "PerfectHashPrimes.h" "PerfectHashPrivate.h" + "PerfectHashServer.h" "PerfectHashTable.h" "PerfectHashTimestamp.h" "PerfectHashTls.h" @@ -202,13 +208,18 @@ set(Source_Files "Math.c" "PerfectHashAllocator.c" "PerfectHashContext.c" + "PerfectHashContextIocp.c" + "PerfectHashContextIocpArgs.c" + "PerfectHashIocpBufferPool.c" "PerfectHashContextTableCreate.c" + "PerfectHashClient.c" "PerfectHashKeys.c" "PerfectHashKeysLoad.c" "PerfectHashKeysLoadTableSize.c" "PerfectHashNames.c" "PerfectHashPath.c" "PerfectHashPrimes.c" + "PerfectHashServer.c" "PerfectHashTable.c" "PerfectHashTableCreate.c" "PerfectHashTableDelete.c" @@ -263,7 +274,9 @@ endif() if(WIN32) list( APPEND Source_Files + "Chm01Async.c" "Chm01.c" + "PerfectHashAsync.c" "RtlUuid.c" "Security.c" "RtlRandom.c" diff --git a/src/PerfectHash/Chm01.c b/src/PerfectHash/Chm01.c index 10644e16..5cba535a 100644 --- a/src/PerfectHash/Chm01.c +++ b/src/PerfectHash/Chm01.c @@ -415,24 +415,21 @@ Return Value: TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); - ASSERT(!TlsContext->Flags.DisableGlobalAllocatorComponent); - ASSERT(!TlsContext->Flags.CustomAllocatorDetailsPresent); - ASSERT(!TlsContext->HeapCreateFlags); - ASSERT(!TlsContext->HeapMinimumSize); - TlsContextDisableGlobalAllocator(TlsContext); TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; TlsContext->HeapMinimumSize = (ULONG_PTR)Info.AllocSize; // - // Make sure we haven't overflowed MAX_ULONG. This should be caught - // earlier when preparing the graph info. + // Make sure we haven't overflowed the heap minimum size on 32-bit builds. + // This should be caught earlier when preparing the graph info. // +#if defined(_M_IX86) if ((ULONGLONG)TlsContext->HeapMinimumSize != Info.AllocSize) { PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); } +#endif // // Copy the table create flags into the TLS context as well; they are now @@ -646,7 +643,7 @@ Return Value: ZeroStructInline(Verb##Name); \ Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_PREPARE_FILE_WORK() \ PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) @@ -1057,7 +1054,7 @@ Return Value: // if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); } // @@ -1435,7 +1432,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->ConsoleWork, TRUE); WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); } // @@ -1485,14 +1482,14 @@ Return Value: Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ Verb##Name.EndOfFile = EndOfFile; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_CLOSE_FILE_WORK() \ CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_CLOSE_FILE_WORK) SUBMIT_CLOSE_FILE_WORK(); - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); #define EXPAND_AS_CHECK_CLOSE_ERRORS( \ Verb, VUpper, Name, Upper, \ @@ -2435,7 +2432,9 @@ Return Value: Result = S_OK; } - PerfectHashPrintMessage(PH_MSG_PERFECT_HASH_CONSOLE_KEYS_HELP); + if (Context->ConsoleWork != NULL) { + PerfectHashPrintMessage(PH_MSG_PERFECT_HASH_CONSOLE_KEYS_HELP); + } // // Intentional follow-on to End. diff --git a/src/PerfectHash/Chm01Async.c b/src/PerfectHash/Chm01Async.c new file mode 100644 index 00000000..cfd1d1aa --- /dev/null +++ b/src/PerfectHash/Chm01Async.c @@ -0,0 +1,1997 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + Chm01Async.c + +Abstract: + + This module implements the CHM01 asynchronous table creation state + machine. Work is decomposed into fine-grained IOCP-driven steps that + overlap compute and file I/O. + +--*/ + +#include "stdafx.h" +#include "Chm01.h" +#include "Chm01Private.h" +#include "Chm01FileWork.h" +#include "Chm01Async.h" + +// +// Async job flags. +// + +#define CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES 0x00000001 +#define CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED 0x00000002 +#define CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED 0x00000008 +#define CHM01_ASYNC_IOCP_PUMP_TIMEOUT_MS 100 + +typedef struct _CHM01_ASYNC_GRAPH_WORK { + PERFECT_HASH_ASYNC_WORK Work; + PCHM01_ASYNC_JOB Job; + PGRAPH Graph; + BOOLEAN Started; + BYTE Padding[7]; +} CHM01_ASYNC_GRAPH_WORK; +typedef CHM01_ASYNC_GRAPH_WORK *PCHM01_ASYNC_GRAPH_WORK; + +FORCEINLINE +BOOLEAN +Chm01AsyncShouldSkipContextFileWork( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ FILE_WORK_ID FileWorkId + ) +{ + FILE_ID FileId; + + if (!SkipContextFileWork(Context)) { + return FALSE; + } + + FileId = FileWorkIdToFileId(FileWorkId); + + return IsValidContextFileId((CONTEXT_FILE_ID)FileId); +} + +static +VOID +Chm01AsyncLogJobState( + _In_ PCHM01_ASYNC_JOB Job, + _In_ ULONG Stage + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[512]; + LONG Outstanding = 0; + LONG ActiveGraphs = 0; + ULONG State = 0; + ULONG Attempt = 0; + ULONG Flags = 0; + HRESULT Result = S_OK; + ULONG NameChars = 0; + PCWSTR NameBuffer = L""; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + PPERFECT_HASH_FILE File; + PPERFECT_HASH_PATH Path; + + if (GetEnvironmentVariableW(L"PH_LOG_CHM01_ASYNC_JOB", NULL, 0) == 0) { + return; + } + + if (!ARGUMENT_PRESENT(Job)) { + return; + } + + Outstanding = Job->Async.Outstanding; + ActiveGraphs = Job->ActiveGraphs; + State = (ULONG)Job->State; + Attempt = Job->Attempt; + Flags = Job->Flags; + Result = Job->LastResult; + + Table = Job->Table; + Keys = Table ? Table->Keys : NULL; + File = Keys ? Keys->File : NULL; + Path = File ? GetActivePath(File) : NULL; + + if (Path && Path->FileName.Buffer) { + NameBuffer = Path->FileName.Buffer; + NameChars = Path->FileName.Length / sizeof(WCHAR); + } + + LogHandle = CreateFileW(L"PerfectHashChm01AsyncJob.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s( + Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu State=%lu Outstanding=%ld ActiveGraphs=%ld Attempt=%lu " + "Flags=0x%08lX Result=0x%08lX Name=%.*S\r\n", + Stage, + State, + Outstanding, + ActiveGraphs, + Attempt, + Flags, + Result, + (int)NameChars, + NameBuffer + ); + + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Job); + UNREFERENCED_PARAMETER(Stage); +#endif +} + +static +HRESULT +Chm01AsyncGraphStep( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work + ); + +static +VOID +Chm01AsyncGraphComplete( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ); + +static +HRESULT +Chm01AsyncPrepareGraphs( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncDispatchGraphWork( + _In_ PCHM01_ASYNC_JOB Job, + _In_ ULONG DispatchCount + ); + +static +HRESULT +Chm01AsyncInitializeJob( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncPollSolveEvents( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncMaybeIncreaseConcurrency( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeVerify( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeWaitSave( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeClose( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncReleaseGraphs( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncStep( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work + ); + +static +VOID +Chm01AsyncComplete( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ); + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncGraphStep( + PPERFECT_HASH_ASYNC_WORK Work + ) +{ + PGRAPH Graph; + PGRAPH LockedGraph; + PGRAPH NewGraph; + ULONG OldNumberOfVertices; + HRESULT Result; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_JOB Job; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); + Graph = GraphWork->Graph; + Job = GraphWork->Job; + Context = Job->Context; + + if (!GraphWork->Started) { + GraphWork->Started = TRUE; + InterlockedIncrement(&Context->ActiveSolvingLoops); + if (InterlockedCompareExchange(&Job->FirstGraphStarted, 1, 0) == 0) { + Job->FirstGraphStartMilliseconds = GetTickCount64(); + if (Job->IncreaseConcurrencyAfterMilliseconds > 0) { + Job->NextConcurrencyIncreaseMilliseconds = + Job->FirstGraphStartMilliseconds + + Job->IncreaseConcurrencyAfterMilliseconds; + } + } + } + + LockedGraph = Graph; + AcquireGraphLockExclusive(LockedGraph); + + if (!IsGraphInfoLoaded(LockedGraph)) { + Result = LockedGraph->Vtbl->LoadInfo(LockedGraph); + if (FAILED(Result)) { + if (Result != E_OUTOFMEMORY) { + PH_ERROR(GraphLoadInfo, Result); + Result = PH_E_INVARIANT_CHECK_FAILED; + } else { + if (InterlockedDecrement(&Context->GraphMemoryFailures) == 0) { + Context->State.AllGraphsFailedMemoryAllocation = TRUE; + SetStopSolving(Context); + if (!SetEvent(Context->FailedEvent)) { + SYS_ERROR(SetEvent); + } + } + Result = S_OK; + } + goto End; + } + } + + if (!LockedGraph->Vtbl->ShouldWeContinueTryingToSolve(LockedGraph)) { + Result = S_OK; + goto End; + } + + Result = LockedGraph->Vtbl->Reset(LockedGraph); + if (FAILED(Result)) { + if (Result == PH_E_NO_MORE_SEEDS) { + Result = S_OK; + } + goto End; + } + + if (Result != PH_S_CONTINUE_GRAPH_SOLVING) { + Result = S_OK; + goto End; + } + + Result = LockedGraph->Vtbl->LoadNewSeeds(LockedGraph); + if (FAILED(Result)) { + if (Result == PH_E_NO_MORE_SEEDS) { + Result = S_OK; + } + goto End; + } + + NewGraph = NULL; + Result = LockedGraph->Vtbl->Solve(LockedGraph, &NewGraph); + if (FAILED(Result)) { + goto End; + } + + if (Result == PH_S_STOP_GRAPH_SOLVING || + Result == PH_S_GRAPH_SOLVING_STOPPED) { + Result = S_OK; + goto End; + } + + if (Result == PH_S_USE_NEW_GRAPH_FOR_SOLVING) { + if (!NewGraph) { + Result = PH_E_INVARIANT_CHECK_FAILED; + goto End; + } + + OldNumberOfVertices = LockedGraph->NumberOfVertices; + + AcquireGraphLockExclusive(NewGraph); + ReleaseGraphLockExclusive(LockedGraph); + LockedGraph = NewGraph; + GraphWork->Graph = NewGraph; + + if (!IsGraphInfoLoaded(LockedGraph) || + LockedGraph->LastLoadedNumberOfVertices < OldNumberOfVertices) { + Result = LockedGraph->Vtbl->LoadInfo(LockedGraph); + if (FAILED(Result)) { + PH_ERROR(GraphLoadInfo_NewGraph, Result); + goto End; + } + } + + Result = S_FALSE; + goto End; + } + + ASSERT(Result == PH_S_CONTINUE_GRAPH_SOLVING); + + Result = S_FALSE; + +End: + + if (LockedGraph) { + ReleaseGraphLockExclusive(LockedGraph); + } + + return Result; +} + +_Use_decl_annotations_ +static +VOID +Chm01AsyncGraphComplete( + PPERFECT_HASH_ASYNC_WORK Work, + HRESULT Result + ) +{ + PGRAPH Graph; + PHANDLE Event; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_JOB Job; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + UNREFERENCED_PARAMETER(Result); + + GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); + Job = GraphWork->Job; + Graph = GraphWork->Graph; + Context = Job->Context; + + if (GraphWork->Started) { + InterlockedDecrement(&Context->ActiveSolvingLoops); + } + + if (InterlockedDecrement(&Context->RemainingSolverLoops) == 0) { + if (Context->FinishedCount == 0) { + Event = &Context->FailedEvent; + } else { + Event = &Context->SucceededEvent; + } + if (!SetEvent(*Event)) { + SYS_ERROR(SetEvent); + } + SetStopSolving(Context); + } + + if (InterlockedDecrement(&Job->ActiveGraphs) == 0) { + if (Job->GraphsCompleteEvent) { + SetEvent(Job->GraphsCompleteEvent); + } + } + + if (Job->Allocator) { + Job->Allocator->Vtbl->FreePointer(Job->Allocator, (PVOID *)&GraphWork); + } +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncPrepareGraphs( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONG Index; + PGRAPH Graph; + PGRAPH *Graphs; + PPERFECT_HASH_CONTEXT Context; + + Context = Job->Context; + Graphs = Job->Graphs; + + if (FirstSolvedGraphWins(Context)) { + ASSERT(Job->NumberOfGraphs == Job->Concurrency); + } else { + ASSERT(Job->NumberOfGraphs - 1 == Job->Concurrency); + } + + for (Index = 0; Index < Job->NumberOfGraphs; Index++) { + + Graph = Graphs[Index]; + + ResetSRWLock(&Graph->Lock); + InitializeListHead(&Graph->ListEntry); + + AcquireGraphLockExclusive(Graph); + Result = Graph->Vtbl->SetInfo(Graph, &Job->GraphInfo); + ReleaseGraphLockExclusive(Graph); + + if (FAILED(Result)) { + PH_ERROR(GraphSetInfo, Result); + return Result; + } + + Graph->Flags.IsInfoLoaded = FALSE; + + if (!FirstSolvedGraphWins(Context) && Index == 0) { + Graph->Flags.IsSpare = TRUE; + + _Benign_race_begin_ + Context->SpareGraph = Graph; + _Benign_race_end_ + + continue; + } + + Graph->Flags.IsSpare = FALSE; + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncDispatchGraphWork( + PCHM01_ASYNC_JOB Job, + ULONG DispatchCount + ) +{ + HRESULT Result; + ULONG Index; + ULONG GraphIndex; + ULONG Remaining; + PGRAPH Graph; + PGRAPH *Graphs; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + Context = Job->Context; + Graphs = Job->Graphs; + + Remaining = Job->Concurrency - Job->DispatchedGraphs; + if (DispatchCount == 0 || Remaining == 0) { + return S_OK; + } + + if (DispatchCount > Remaining) { + DispatchCount = Remaining; + } + + if (Job->GraphsCompleteEvent) { + ResetEvent(Job->GraphsCompleteEvent); + } + + if (Job->DispatchedGraphs == 0) { + ASSERT(Context->MainWorkList->Vtbl->IsEmpty(Context->MainWorkList)); + ASSERT(Context->FinishedWorkList->Vtbl->IsEmpty(Context->FinishedWorkList)); + } + + for (Index = 0; Index < DispatchCount; Index++) { + + GraphIndex = Job->DispatchedGraphs; + if (!FirstSolvedGraphWins(Context)) { + GraphIndex++; + } + + if (GraphIndex >= Job->NumberOfGraphs) { + return PH_E_INVARIANT_CHECK_FAILED; + } + + Graph = Graphs[GraphIndex]; + + GraphWork = (PCHM01_ASYNC_GRAPH_WORK)( + Job->Allocator->Vtbl->Calloc( + Job->Allocator, + 1, + sizeof(*GraphWork) + ) + ); + + if (!GraphWork) { + return E_OUTOFMEMORY; + } + + GraphWork->Job = Job; + GraphWork->Graph = Graph; + GraphWork->Work.Step = Chm01AsyncGraphStep; + GraphWork->Work.Complete = Chm01AsyncGraphComplete; + GraphWork->Work.SliceBudget = Job->SliceBudget; + + InterlockedIncrement(&Context->GraphMemoryFailures); + InterlockedIncrement(&Context->RemainingSolverLoops); + InterlockedIncrement(&Job->ActiveGraphs); + Job->DispatchedGraphs++; + + Result = PerfectHashAsyncSubmit(&Job->Async, &GraphWork->Work); + if (FAILED(Result)) { + if (!GraphWork->Work.Flags.InlineDispatch) { + if (InterlockedDecrement(&Context->RemainingSolverLoops) == 0) { + if (Context->FinishedCount == 0) { + if (!SetEvent(Context->FailedEvent)) { + SYS_ERROR(SetEvent); + } + } else { + if (!SetEvent(Context->SucceededEvent)) { + SYS_ERROR(SetEvent); + } + } + SetStopSolving(Context); + } + + if (InterlockedDecrement(&Job->ActiveGraphs) == 0) { + if (Job->GraphsCompleteEvent) { + if (!SetEvent(Job->GraphsCompleteEvent)) { + SYS_ERROR(SetEvent); + } + } + } + + if (Job->DispatchedGraphs > 0) { + Job->DispatchedGraphs--; + } + + if (Job->Allocator) { + Job->Allocator->Vtbl->FreePointer( + Job->Allocator, + (PVOID *)&GraphWork + ); + } + } + + return Result; + } + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncInitializeJob( + PCHM01_ASYNC_JOB Job + ) +{ + USHORT Index; + HRESULT Result; + BOOLEAN LimitConcurrency; + ULONG Concurrency; + ULONG InitialConcurrency; + ULONG NumberOfGraphs; + ULONG NumberOfSeedsRequired; + ULONG NumberOfSeedsAvailable; + PGRAPH Graph; + PGRAPH *Graphs; + PALLOCATOR Allocator; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; + GRAPH_INFO_ON_DISK *GraphInfoOnDisk; + PTABLE_INFO_ON_DISK TableInfoOnDisk; + PHANDLE Event; + ULONG NumberOfEvents; + + Table = Job->Table; + Context = Table->Context; + Allocator = Table->Allocator; + TableCreateFlags.AsULongLong = Table->TableCreateFlags.AsULongLong; + + // + // Initialize event arrays. + // + + Job->Events[Chm01AsyncJobEventSucceeded] = Context->SucceededEvent; + Job->Events[Chm01AsyncJobEventCompleted] = Context->CompletedEvent; + Job->Events[Chm01AsyncJobEventShutdown] = Context->ShutdownEvent; + Job->Events[Chm01AsyncJobEventFailed] = Context->FailedEvent; + Job->Events[Chm01AsyncJobEventLowMemory] = Context->LowMemoryEvent; + + { + PHANDLE SaveEvent = Job->SaveEvents; + PHANDLE PrepareEvent = Job->PrepareEvents; + +#define EXPAND_AS_ASSIGN_EVENT( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + *Verb##Event++ = Context->Verb##d##Name##Event; + + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASSIGN_EVENT); + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASSIGN_EVENT); + +#undef EXPAND_AS_ASSIGN_EVENT + } + + // + // Set up on-disk info buffers. + // + + GraphInfoOnDisk = &Job->GraphInfoOnDisk; + TableInfoOnDisk = &GraphInfoOnDisk->TableInfoOnDisk; + + Job->TableInfoOnDisk = TableInfoOnDisk; + Context->GraphInfoOnDisk = GraphInfoOnDisk; + Table->TableInfoOnDisk = TableInfoOnDisk; + + // + // Verify we have sufficient seeds available for the hash function. + // + + NumberOfSeedsRequired = HashRoutineNumberOfSeeds[Table->HashFunctionId]; + NumberOfSeedsAvailable = (( + FIELD_OFFSET(GRAPH, LastSeed) - + FIELD_OFFSET(GRAPH, FirstSeed) + ) / sizeof(ULONG)) + 1; + + if (NumberOfSeedsAvailable < NumberOfSeedsRequired) { + return PH_E_INVALID_NUMBER_OF_SEEDS; + } + + // + // Determine concurrency. + // + + Concurrency = Context->MaxPerFileConcurrency; + if (Concurrency == 0) { + Concurrency = Context->MaximumConcurrency; + } + if (Concurrency == 0) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + LimitConcurrency = ( + Table->PriorPredictedAttempts > 0 && + TableCreateFlags.TryUsePredictedAttemptsToLimitMaxConcurrency != FALSE + ); + + if (LimitConcurrency) { + Concurrency = min(Concurrency, Table->PriorPredictedAttempts); + } + + if (Concurrency == 0) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + InitialConcurrency = Context->InitialPerFileConcurrency; + if (InitialConcurrency == 0) { + InitialConcurrency = 1; + } + + if (InitialConcurrency > Concurrency) { + InitialConcurrency = Concurrency; + } + + Job->Concurrency = Concurrency; + Job->InitialConcurrency = InitialConcurrency; + Job->DispatchedGraphs = 0; + Job->IncreaseConcurrencyAfterMilliseconds = + Context->IncreaseConcurrencyAfterMilliseconds; + Job->FirstGraphStarted = 0; + Job->FirstGraphStartMilliseconds = 0; + Job->NextConcurrencyIncreaseMilliseconds = 0; + + if (FirstSolvedGraphWins(Context)) { + NumberOfGraphs = Concurrency; + } else { + NumberOfGraphs = Concurrency + 1; + if (NumberOfGraphs == 0) { + return E_INVALIDARG; + } + } + + Job->NumberOfGraphs = NumberOfGraphs; + + Context->InitialResizes = 0; + Context->NumberOfTableResizeEvents = 0; + + Result = PrepareGraphInfoChm01(Table, &Job->GraphInfo, NULL); + if (FAILED(Result)) { + PH_ERROR(CreatePerfectHashTableImplChm01_PrepareFirstGraphInfo, Result); + goto Error; + } + + // + // Allocate graph instances. + // + + Graphs = (PGRAPH *)( + Allocator->Vtbl->Calloc( + Allocator, + NumberOfGraphs, + sizeof(Graph) + ) + ); + + if (!Graphs) { + Result = PH_I_OUT_OF_MEMORY; + goto Error; + } + + Job->Graphs = Graphs; + + // + // Use per-graph allocator instances (TLS-toggled). + // + + TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + + TlsContextDisableGlobalAllocator(TlsContext); + TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; + TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; + TlsContext->HeapMinimumSize = (ULONG_PTR)Job->GraphInfo.AllocSize; + +#if defined(_M_IX86) + if ((ULONGLONG)TlsContext->HeapMinimumSize != Job->GraphInfo.AllocSize) { + PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); + } +#endif + + TlsContext->TableCreateFlags.AsULongLong = TableCreateFlags.AsULongLong; + TlsContext->Table = Table; + + for (Index = 0; Index < NumberOfGraphs; Index++) { + + Result = Table->Vtbl->CreateInstance(Table, + NULL, + &IID_PERFECT_HASH_GRAPH, + (PVOID *)&Graph); + + if (FAILED(Result)) { + if (Result != E_OUTOFMEMORY) { + PH_ERROR(CreatePerfectHashTableImplChm01_CreateGraph, Result); + } + break; + } + +#ifdef PH_WINDOWS + ASSERT(Graph->Allocator != Table->Allocator); + ASSERT(Graph->Allocator->HeapHandle != Table->Allocator->HeapHandle); +#endif + + ASSERT(Graph->Rtl == Table->Rtl); + + Graph->Flags.SkipVerification = ( + TableCreateFlags.SkipGraphVerification != FALSE + ); + + Graph->Flags.WantsWriteCombiningForVertexPairsArray = ( + TableCreateFlags.EnableWriteCombineForVertexPairs != FALSE + ); + + Graph->Flags.RemoveWriteCombineAfterSuccessfulHashKeys = ( + TableCreateFlags.RemoveWriteCombineAfterSuccessfulHashKeys != FALSE + ); + + Graph->Index = Index; + Graphs[Index] = Graph; + } + + TlsContextEnableGlobalAllocator(TlsContext); + TlsContext->Flags.CustomAllocatorDetailsPresent = FALSE; + TlsContext->HeapCreateFlags = 0; + TlsContext->HeapMinimumSize = 0; + TlsContext->Table = NULL; + + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + + if (FAILED(Result)) { + goto Error; + } + + // + // Prepare all graphs for solving. + // + + Result = Chm01AsyncPrepareGraphs(Job); + if (FAILED(Result)) { + goto Error; + } + + // + // Reset all context events. + // + + Event = (PHANDLE)&Context->FirstEvent; + NumberOfEvents = GetNumberOfContextEvents(Context); + + for (Index = 0; Index < NumberOfEvents; Index++, Event++) { + if (!ResetEvent(*Event)) { + SYS_ERROR(ResetEvent); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + } + + Context->LowMemoryObserved = 0; + Context->State.AllGraphsFailedMemoryAllocation = FALSE; + Context->State.SolveTimeoutExpired = FALSE; + Context->State.FixedAttemptsReached = FALSE; + Context->State.MaxAttemptsReached = FALSE; + ClearStopSolving(Context); + + Job->Attempt = 1; + + // + // Set callbacks and algorithm context. + // + + Context->MainWorkCallback = ProcessGraphCallbackChm01; + Context->ConsoleWorkCallback = ProcessConsoleCallbackChm01; + Context->AlgorithmContext = &Job->GraphInfo; + Context->FileWorkCallback = FileWorkCallbackChm01; + + // + // Prepare output directory if required. + // + + if (!NoFileIo(Table)) { + Result = PrepareTableOutputDirectory(Table); + if (FAILED(Result)) { + PH_ERROR(PrepareTableOutputDirectory, Result); + goto Error; + } + } + + // + // Configure solve timeout if requested. + // + + if (Table->MaxSolveTimeInSeconds > 0 && Context->SolveTimeout) { + SetThreadpoolTimer(Context->SolveTimeout, + &Table->RelativeMaxSolveTimeInFiletime.AsFileTime, + 0, + 0); + } + + // + // Submit prepare file work. + // + + ASSERT(Context->FileWorkList->Vtbl->IsEmpty(Context->FileWorkList)); + + if (!NoFileIo(Table)) { + +#define EXPAND_AS_SUBMIT_FILE_WORK( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + ZeroStructInline(Job->Verb##Name); \ + Job->Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ + if (Chm01AsyncShouldSkipContextFileWork( \ + Context, \ + Job->Verb##Name.FileWorkId)) { \ + if (!SetEvent(Context->Verb##d##Name##Event)) { \ + SYS_ERROR(SetEvent); \ + Result = PH_E_SYSTEM_CALL_FAILED; \ + goto Error; \ + } \ + } else { \ + InsertTailFileWork(Context, &Job->Verb##Name.ListEntry); \ + PerfectHashContextSubmitFileWork(Context); \ + } + +#define SUBMIT_PREPARE_FILE_WORK() \ + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) + + SUBMIT_PREPARE_FILE_WORK(); + +#undef SUBMIT_PREPARE_FILE_WORK +#undef EXPAND_AS_SUBMIT_FILE_WORK + } + + // + // Capture initial timing. + // + + QueryPerformanceFrequency(&Context->Frequency); + CONTEXT_START_TIMERS(Solve); + Context->StartMilliseconds = GetTickCount64(); + + // + // Reset counters. + // + + Context->GraphMemoryFailures = 0; + Context->RemainingSolverLoops = 0; + Context->ActiveSolvingLoops = 0; + Context->Attempts = 0; + Context->FailedAttempts = 0; + Context->FinishedCount = 0; + Context->HighestDeletedEdgesCount = 0; + + // + // Dispatch graph work. + // + + Result = Chm01AsyncDispatchGraphWork(Job, Job->InitialConcurrency); + if (FAILED(Result)) { + goto Error; + } + + return S_OK; + +Error: + + if (Result == E_OUTOFMEMORY) { + Result = PH_I_OUT_OF_MEMORY; + } + + if (!NoFileIo(Table)) { + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); + } + + if (Result != S_OK) { + SetEvent(Context->ShutdownEvent); + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncPollSolveEvents( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONG WaitResult; + PPERFECT_HASH_CONTEXT Context; + + Context = Job->Context; + + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->Events), + Job->Events, + FALSE, + 0); + + if (WaitResult == WAIT_TIMEOUT) { + Result = Chm01AsyncMaybeIncreaseConcurrency(Job); + if (FAILED(Result)) { + return Result; + } + return S_FALSE; + } + + SetStopSolving(Context); + + if (CtrlCPressed) { + return PH_E_CTRL_C_PRESSED; + } + + if (WaitResult == WAIT_OBJECT_0 + 4) { + InterlockedIncrement(&Context->LowMemoryObserved); + return PH_I_LOW_MEMORY; + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncMaybeIncreaseConcurrency( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONGLONG Now; + PPERFECT_HASH_CONTEXT Context; + + Context = Job->Context; + + if (Job->IncreaseConcurrencyAfterMilliseconds == 0) { + return S_OK; + } + + if (Job->FirstGraphStarted == 0) { + return S_OK; + } + + if (Job->DispatchedGraphs >= Job->Concurrency) { + return S_OK; + } + + if (StopSolving(Context)) { + return S_OK; + } + + Now = GetTickCount64(); + if (Now < Job->NextConcurrencyIncreaseMilliseconds) { + return S_OK; + } + + Result = Chm01AsyncDispatchGraphWork(Job, 1); + if (FAILED(Result)) { + return Result; + } + + Job->NextConcurrencyIncreaseMilliseconds = + Now + Job->IncreaseConcurrencyAfterMilliseconds; + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeVerify( + PCHM01_ASYNC_JOB Job + ) +{ + PRTL Rtl; + PRNG Rng; + HRESULT Result; + PGRAPH Graph; + ULONG WaitResult; + PLIST_ENTRY ListEntry; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; + BOOLEAN FailedEventSet; + BOOLEAN ShutdownEventSet; + BOOLEAN LowMemoryEventSet = FALSE; + + Context = Job->Context; + Table = Job->Table; + Rtl = Table->Rtl; + TableCreateFlags.AsULongLong = Table->TableCreateFlags.AsULongLong; + + if (Job->GraphsCompleteEvent) { + WaitResult = WaitForSingleObject(Job->GraphsCompleteEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + return S_FALSE; + } + } + + if (CtrlCPressed) { + return PH_E_CTRL_C_PRESSED; + } + + if (Context->FinishedCount == 0) { + + WaitResult = WaitForSingleObject(Context->FailedEvent, 0); + FailedEventSet = (WaitResult == WAIT_OBJECT_0); + + WaitResult = WaitForSingleObject(Context->ShutdownEvent, 0); + ShutdownEventSet = (WaitResult == WAIT_OBJECT_0); + + if ((!FailedEventSet && !ShutdownEventSet) || + Context->LowMemoryObserved > 0) { + LowMemoryEventSet = TRUE; + } + + if (LowMemoryEventSet) { + Result = PH_I_LOW_MEMORY; + } else if (FailedEventSet) { + if (Context->State.AllGraphsFailedMemoryAllocation != FALSE) { + Result = PH_I_FAILED_TO_ALLOCATE_MEMORY_FOR_ALL_GRAPHS; + } else if (Context->State.SolveTimeoutExpired != FALSE) { + Result = PH_I_SOLVE_TIMEOUT_EXPIRED; + } else { + Result = PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION; + } + } else if (ShutdownEventSet) { + Result = PH_I_CREATE_TABLE_ROUTINE_RECEIVED_SHUTDOWN_EVENT; + } else { + Result = PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION; + } + + return Result; + } + + ASSERT(Context->FinishedCount > 0); + + if (!TableCreateFlags.Quiet) { + Result = PrintCurrentContextStatsChm01(Context); + if (FAILED(Result)) { + PH_ERROR(PrintCurrentContextStatsChm01, Result); + return Result; + } + } + + if (FirstSolvedGraphWins(Context)) { + ListEntry = NULL; + if (!RemoveHeadFinishedWork(Context, &ListEntry)) { + Result = PH_E_GUARDED_LIST_EMPTY; + PH_ERROR(PerfectHashCreateChm01Callback_RemoveFinishedWork, Result); + return Result; + } + + Graph = CONTAINING_RECORD(ListEntry, GRAPH, ListEntry); + + } else { + + EnterCriticalSection(&Context->BestGraphCriticalSection); + Graph = Context->BestGraph; + Context->BestGraph = NULL; + LeaveCriticalSection(&Context->BestGraphCriticalSection); + + ASSERT(Graph != NULL); + } + + Table->MaximumGraphTraversalDepth = Graph->MaximumTraversalDepth; + Table->NumberOfEmptyVertices = Graph->NumberOfEmptyVertices; + Table->NumberOfCollisionsDuringAssignment = Graph->Collisions; + + Table->Flags.VertexPairsArrayUsesLargePages = ( + Graph->Flags.VertexPairsArrayUsesLargePages + ); + + Table->Flags.UsedAvx2HashFunction = Graph->Flags.UsedAvx2HashFunction; + Table->Flags.UsedAvx512HashFunction = Graph->Flags.UsedAvx512HashFunction; + Table->Flags.UsedAvx2MemoryCoverageFunction = + Graph->Flags.UsedAvx2MemoryCoverageFunction; + + COPY_GRAPH_COUNTERS_FROM_GRAPH_TO_TABLE(); + + if (Context->RngId != PerfectHashRngSystemId) { + Table->RngSeed = Graph->Rng->Seed; + Table->RngSubsequence = Graph->Rng->Subsequence; + Table->RngOffset = Graph->Rng->Offset; + + Rng = Graph->Rng; + Result = Rng->Vtbl->GetCurrentOffset(Rng, &Table->RngCurrentOffset); + if (FAILED(Result)) { + PH_ERROR(CreatePerfectHashTableImplChm01_RngGetCurrentOffset, + Result); + return Result; + } + } + + Table->SolveDurationInSeconds = (DOUBLE)( + ((DOUBLE)Context->SolveElapsedMicroseconds.QuadPart) / + ((DOUBLE)1e6) + ); + + Table->SolutionsFoundRatio = (DOUBLE)( + ((DOUBLE)Context->FinishedCount) / + ((DOUBLE)Context->Attempts) + ); + + Result = CalculatePredictedAttempts(Table->SolutionsFoundRatio, + &Table->PredictedAttempts); + if (FAILED(Result)) { + PH_ERROR(CreatePerfectHashTableImplChm01_CalculatePredictedAttempts, + Result); + return Result; + } + + Context->SolvedContext = Graph; + + if (!NoFileIo(Table)) { + +#define EXPAND_AS_SUBMIT_FILE_WORK( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + ZeroStructInline(Job->Verb##Name); \ + Job->Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ + if (Chm01AsyncShouldSkipContextFileWork( \ + Context, \ + Job->Verb##Name.FileWorkId)) { \ + if (!SetEvent(Context->Verb##d##Name##Event)) { \ + SYS_ERROR(SetEvent); \ + return PH_E_SYSTEM_CALL_FAILED; \ + } \ + } else { \ + InsertTailFileWork(Context, &Job->Verb##Name.ListEntry); \ + PerfectHashContextSubmitFileWork(Context); \ + } + +#define SUBMIT_SAVE_FILE_WORK() \ + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) + + SUBMIT_SAVE_FILE_WORK(); + +#undef SUBMIT_SAVE_FILE_WORK +#undef EXPAND_AS_SUBMIT_FILE_WORK + Job->Flags |= CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED; + + } else { + + PGRAPH_INFO_ON_DISK NewGraphInfoOnDisk; + PGRAPH_INFO_ON_DISK ExistingGraphInfoOnDisk; + + ExistingGraphInfoOnDisk = Context->GraphInfoOnDisk; + + NewGraphInfoOnDisk = ( + Job->Allocator->Vtbl->Calloc( + Job->Allocator, + 1, + sizeof(*NewGraphInfoOnDisk) + ) + ); + + if (!NewGraphInfoOnDisk) { + return E_OUTOFMEMORY; + } + + CopyMemory(NewGraphInfoOnDisk, + ExistingGraphInfoOnDisk, + sizeof(*NewGraphInfoOnDisk)); + + if (Graph->FirstSeed == 0) { + Result = PH_E_INVARIANT_CHECK_FAILED; + PH_ERROR(CreatePerfectHashTableImplChm01_GraphFirstSeedIs0, Result); + PH_RAISE(Result); + } + + CopyMemory(&NewGraphInfoOnDisk->TableInfoOnDisk.FirstSeed, + &Graph->FirstSeed, + Graph->NumberOfSeeds * sizeof(Graph->FirstSeed)); + + Table->TableInfoOnDisk = &NewGraphInfoOnDisk->TableInfoOnDisk; + Table->State.TableInfoOnDiskWasHeapAllocated = TRUE; + + if (!IsTableCreateOnly(Table)) { + PVOID BaseAddress; + LONGLONG SizeInBytes; + BOOLEAN LargePagesForTableData; + PRTL_TRY_LARGE_PAGE_VIRTUAL_ALLOC TryLargePageVirtualAlloc; + + LargePagesForTableData = ( + Table->TableCreateFlags.TryLargePagesForTableData == TRUE + ); + + SizeInBytes = ( + Job->TableInfoOnDisk->NumberOfTableElements.QuadPart * + Job->TableInfoOnDisk->AssignedElementSizeInBytes + ); + + TryLargePageVirtualAlloc = Rtl->Vtbl->TryLargePageVirtualAlloc; + BaseAddress = TryLargePageVirtualAlloc(Rtl, + NULL, + SizeInBytes, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE, + &LargePagesForTableData); + + Table->TableDataBaseAddress = BaseAddress; + Table->TableDataSizeInBytes = SizeInBytes; + + if (!BaseAddress) { + return E_OUTOFMEMORY; + } + + Table->State.TableDataWasHeapAllocated = TRUE; + Table->Flags.TableDataUsesLargePages = LargePagesForTableData; + + CopyMemory(Table->TableDataBaseAddress, + Graph->Assigned, + SizeInBytes); + } + } + + CONTEXT_START_TIMERS(Verify); + + Result = Graph->Vtbl->Verify(Graph); + + CONTEXT_END_TIMERS(Verify); + + if (!SetEvent(Context->VerifiedTableEvent)) { + SYS_ERROR(SetEvent); + return PH_E_SYSTEM_CALL_FAILED; + } + + if (FAILED(Result)) { + return PH_E_TABLE_VERIFICATION_FAILED; + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeWaitSave( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONG WaitResult; + + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->SaveEvents), + Job->SaveEvents, + TRUE, + 0); + + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + + Result = S_OK; + +#undef EXPAND_AS_CHECK_ERRORS +#define EXPAND_AS_CHECK_ERRORS( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Job->Verb##Name.NumberOfErrors > 0) { \ + Result = Job->Verb##Name.LastResult; \ + if (Result == S_OK || Result == E_UNEXPECTED) { \ + Result = PH_E_ERROR_DURING_##VUpper##_##Upper; \ + } \ + PH_ERROR( \ + CreatePerfectHashTableImplChm01_ErrorDuring##Verb##Name, \ + Result \ + ); \ + goto Error; \ + } + + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CHECK_ERRORS) + +#undef EXPAND_AS_CHECK_ERRORS + + return S_OK; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeClose( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + HRESULT CloseResult; + ULONG CloseFileErrorCount = 0; + ULONG WaitResult; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PLARGE_INTEGER EndOfFile = NULL; + LARGE_INTEGER EmptyEndOfFile = { 0 }; + + Context = Job->Context; + Table = Job->Table; + Result = S_OK; + CloseResult = S_OK; + + if (Job->Attempt == 0 || NoFileIo(Table)) { + return S_OK; + } + + if (Job->GraphsCompleteEvent) { + WaitResult = WaitForSingleObject(Job->GraphsCompleteEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + return S_FALSE; + } + } + + if (Job->Flags & CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED) { + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->SaveEvents), + Job->SaveEvents, + TRUE, + 0); + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + } + + if (!NoFileIo(Table)) { + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->PrepareEvents), + Job->PrepareEvents, + TRUE, + 0); + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + } + + if (Job->Flags & CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES) { + EndOfFile = &EmptyEndOfFile; + } + + if ((Job->Flags & CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED) == 0) { + +#define EXPAND_AS_SUBMIT_CLOSE_FILE_WORK( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + ZeroStructInline(Job->Verb##Name); \ + Job->Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ + Job->Verb##Name.EndOfFile = EndOfFile; \ + if (!Chm01AsyncShouldSkipContextFileWork( \ + Context, \ + Job->Verb##Name.FileWorkId)) { \ + InsertTailFileWork(Context, &Job->Verb##Name.ListEntry); \ + PerfectHashContextSubmitFileWork(Context); \ + } + +#define SUBMIT_CLOSE_FILE_WORK() \ + CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_CLOSE_FILE_WORK) + + SUBMIT_CLOSE_FILE_WORK(); + +#undef SUBMIT_CLOSE_FILE_WORK +#undef EXPAND_AS_SUBMIT_CLOSE_FILE_WORK + + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED; + } + + if (Context->FileWorkOutstandingEvent) { + WaitResult = WaitForSingleObject(Context->FileWorkOutstandingEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + return S_FALSE; + } + } + +#define EXPAND_AS_CHECK_CLOSE_ERRORS( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Job->Verb##Name.NumberOfErrors > 0) { \ + CloseResult = Job->Verb##Name.LastResult; \ + if (CloseResult == S_OK || CloseResult == E_UNEXPECTED) { \ + CloseResult = PH_E_ERROR_DURING_##VUpper##_##Upper; \ + } \ + PH_ERROR( \ + CreatePerfectHashTableImplChm01_ErrorDuring##Verb##Name, \ + Result \ + ); \ + CloseFileErrorCount++; \ + } + + CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CHECK_CLOSE_ERRORS) + +#undef EXPAND_AS_CHECK_CLOSE_ERRORS + + if (CloseFileErrorCount > 0) { + Result = CloseResult; + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncReleaseGraphs( + PCHM01_ASYNC_JOB Job + ) +{ + USHORT Index; + ULONG ReferenceCount; + PGRAPH Graph; + PGRAPH *Graphs; + PALLOCATOR Allocator; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PHANDLE Event; + ULONG NumberOfEvents; + HRESULT Result; + + Result = Job->LastResult; + Context = Job->Context; + Table = Job->Table; + Allocator = Job->Allocator; + Graphs = Job->Graphs; + + if (Result == E_OUTOFMEMORY) { + Result = PH_I_OUT_OF_MEMORY; + } + + if (Graphs) { + for (Index = 0; Index < Job->NumberOfGraphs; Index++) { + + Graph = Graphs[Index]; + if (!Graph) { + continue; + } + + ReferenceCount = Graph->Vtbl->Release(Graph); + + if (ReferenceCount != 0) { + Result = PH_E_INVARIANT_CHECK_FAILED; + PH_ERROR(GraphReferenceCountNotZero, Result); + PH_RAISE(Result); + } + + Graphs[Index] = NULL; + } + + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Graphs); + Job->Graphs = NULL; + } + + Event = (PHANDLE)&Context->FirstEvent; + NumberOfEvents = GetNumberOfContextEvents(Context); + + for (Index = 0; Index < NumberOfEvents; Index++, Event++) { + if (!ResetEvent(*Event)) { + SYS_ERROR(ResetEvent); + if (Result == S_OK) { + Result = PH_E_SYSTEM_CALL_FAILED; + } + } + } + + RELEASE(Table->OutputPath); + + Job->LastResult = Result; + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncStep( + PPERFECT_HASH_ASYNC_WORK Work + ) +{ + HRESULT Result; + PCHM01_ASYNC_JOB Job; + + Job = CONTAINING_RECORD(Work, CHM01_ASYNC_JOB, Work); + + switch (Job->State) { + + case Chm01AsyncStateInitialize: + Result = Chm01AsyncInitializeJob(Job); + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->State = Chm01AsyncStateSolveGraphs; + return S_FALSE; + + case Chm01AsyncStateSolveGraphs: + Result = Chm01AsyncPollSolveEvents(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->State = Chm01AsyncStateFinalizeVerify; + return S_FALSE; + + case Chm01AsyncStateFinalizeVerify: + Result = Chm01AsyncFinalizeVerify(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->LastResult = Result; + if (NoFileIo(Job->Table)) { + Job->State = Chm01AsyncStateReleaseGraphs; + } else { + Job->State = Chm01AsyncStateFinalizeWaitSave; + } + return S_FALSE; + + case Chm01AsyncStateFinalizeWaitSave: + Result = Chm01AsyncFinalizeWaitSave(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->LastResult = Result; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + + case Chm01AsyncStateFinalizeClose: + Result = Chm01AsyncFinalizeClose(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result) && Job->LastResult == S_OK) { + Job->LastResult = Result; + } + Job->State = Chm01AsyncStateReleaseGraphs; + return S_FALSE; + + case Chm01AsyncStateReleaseGraphs: + Result = Chm01AsyncReleaseGraphs(Job); + Job->State = Chm01AsyncStateComplete; + return Result; + + case Chm01AsyncStateComplete: + return Job->LastResult; + + case Chm01AsyncStateError: + default: + Job->LastResult = PH_E_INVARIANT_CHECK_FAILED; + return Job->LastResult; + } +} + +_Use_decl_annotations_ +static +VOID +Chm01AsyncComplete( + PPERFECT_HASH_ASYNC_WORK Work, + HRESULT Result + ) +{ + PCHM01_ASYNC_JOB Job; + + Job = CONTAINING_RECORD(Work, CHM01_ASYNC_JOB, Work); + Job->LastResult = Result; + + Chm01AsyncLogJobState(Job, 1); + + if (Job->CompletionEvent) { + SetEvent(Job->CompletionEvent); + } + + if (Job->Async.IoCompletionPort) { + PostQueuedCompletionStatus(Job->Async.IoCompletionPort, + 0, + 0, + NULL); + } +} + +_Use_decl_annotations_ +HRESULT +Chm01AsyncCreateJob( + PPERFECT_HASH_TABLE Table, + HANDLE IoCompletionPort, + PCHM01_ASYNC_JOB *JobPointer + ) +{ + HRESULT Result; + PCHM01_ASYNC_JOB Job; + PPERFECT_HASH_CONTEXT Context; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(Table) || + !ARGUMENT_PRESENT(JobPointer)) { + return E_POINTER; + } + + Context = Table->Context; + Allocator = Table->Allocator; + + if (!Allocator) { + return E_UNEXPECTED; + } + + Job = (PCHM01_ASYNC_JOB)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*Job) + ); + + if (!Job) { + return E_OUTOFMEMORY; + } + + Job->Table = Table; + Job->Context = Context; + Job->Allocator = Allocator; + Job->Rtl = Table->Rtl; + Job->State = Chm01AsyncStateInitialize; + Job->SliceBudget = 1; + Job->LastResult = S_OK; + + Job->CompletionEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!Job->CompletionEvent) { + SYS_ERROR(CreateEventW_Chm01AsyncCompletionEvent); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Job->GraphsCompleteEvent = CreateEventW(NULL, TRUE, TRUE, NULL); + if (!Job->GraphsCompleteEvent) { + SYS_ERROR(CreateEventW_Chm01AsyncGraphsCompleteEvent); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Result = PerfectHashAsyncInitialize(&Job->Async, Context, IoCompletionPort); + if (FAILED(Result)) { + goto Error; + } + + Job->Work.Step = Chm01AsyncStep; + Job->Work.Complete = Chm01AsyncComplete; + Job->Work.SliceBudget = Job->SliceBudget; + + if (!Context->FileWorkIoCompletionPort) { + Context->FileWorkIoCompletionPort = IoCompletionPort; + } + + *JobPointer = Job; + return S_OK; + +Error: + + if (Job) { + if (Job->CompletionEvent) { + CloseHandle(Job->CompletionEvent); + Job->CompletionEvent = NULL; + } + if (Job->GraphsCompleteEvent) { + CloseHandle(Job->GraphsCompleteEvent); + Job->GraphsCompleteEvent = NULL; + } + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Job); + } + + return Result; +} + +_Use_decl_annotations_ +HRESULT +Chm01AsyncSubmitJob( + PCHM01_ASYNC_JOB Job + ) +{ + if (!ARGUMENT_PRESENT(Job)) { + return E_POINTER; + } + + return PerfectHashAsyncSubmit(&Job->Async, &Job->Work); +} + +#ifdef PH_WINDOWS +static +VOID +Chm01AsyncPumpIoCompletionPort( + _In_ PCHM01_ASYNC_JOB Job + ) +{ + BOOL Success; + DWORD NumberOfBytes; + DWORD LastError; + ULONG_PTR CompletionKey; + LPOVERLAPPED Overlapped; + BOOLEAN LoggedCompletionSignaled = FALSE; + BOOLEAN LoggedOutstandingZero = FALSE; + HANDLE IoCompletionPort; + PPERFECT_HASH_IOCP_WORK WorkItem; + const ULONG AllowedFlags = ( + PH_IOCP_WORK_FLAG_ASYNC | + PH_IOCP_WORK_FLAG_FILE_WORK | + PH_IOCP_WORK_FLAG_PIPE | + PH_IOCP_WORK_FLAG_FILE_IO + ); + + IoCompletionPort = Job->Async.IoCompletionPort; + if (!IoCompletionPort || !Job->CompletionEvent) { + return; + } + + for (;;) { + if (WaitForSingleObject(Job->CompletionEvent, 0) == WAIT_OBJECT_0) { + if (Job->Async.Outstanding == 0) { + break; + } + if (!LoggedCompletionSignaled) { + Chm01AsyncLogJobState(Job, 2); + LoggedCompletionSignaled = TRUE; + } + } else if (Job->Async.Outstanding == 0 && !LoggedOutstandingZero) { + Chm01AsyncLogJobState(Job, 3); + LoggedOutstandingZero = TRUE; + } + + Success = GetQueuedCompletionStatus(IoCompletionPort, + &NumberOfBytes, + &CompletionKey, + &Overlapped, + CHM01_ASYNC_IOCP_PUMP_TIMEOUT_MS); + + if (!Success && !Overlapped) { + LastError = GetLastError(); + if (LastError == WAIT_TIMEOUT) { + continue; + } + } + + if (CompletionKey == PERFECT_HASH_IOCP_SHUTDOWN_KEY) { + PostQueuedCompletionStatus(IoCompletionPort, + NumberOfBytes, + CompletionKey, + Overlapped); + continue; + } + + if (!Overlapped) { + continue; + } + + WorkItem = (PPERFECT_HASH_IOCP_WORK)Overlapped; + + if (WorkItem->Signature != PH_IOCP_WORK_SIGNATURE || + !WorkItem->CompletionCallback) { + continue; + } + + if ((WorkItem->Flags & AllowedFlags) == 0) { + PostQueuedCompletionStatus(IoCompletionPort, + NumberOfBytes, + CompletionKey, + Overlapped); + continue; + } + + WorkItem->CompletionCallback(NULL, + CompletionKey, + Overlapped, + NumberOfBytes, + Success); + } +} +#endif + +_Use_decl_annotations_ +VOID +Chm01AsyncWaitJob( + PCHM01_ASYNC_JOB Job + ) +{ + if (!ARGUMENT_PRESENT(Job)) { + return; + } + + if (Job->CompletionEvent) { +#ifdef PH_WINDOWS + if (Job->Async.IoCompletionPort) { + Chm01AsyncPumpIoCompletionPort(Job); + return; + } +#endif + WaitForSingleObject(Job->CompletionEvent, INFINITE); + } +} + +_Use_decl_annotations_ +VOID +Chm01AsyncDestroyJob( + PCHM01_ASYNC_JOB *JobPointer + ) +{ + PCHM01_ASYNC_JOB Job; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(JobPointer)) { + return; + } + + Job = *JobPointer; + if (!Job) { + return; + } + + Allocator = Job->Allocator; + + if (Job->GraphsCompleteEvent) { + CloseHandle(Job->GraphsCompleteEvent); + Job->GraphsCompleteEvent = NULL; + } + + if (Job->CompletionEvent) { + CloseHandle(Job->CompletionEvent); + Job->CompletionEvent = NULL; + } + + PerfectHashAsyncRundown(&Job->Async); + + if (Job->Graphs) { + Chm01AsyncReleaseGraphs(Job); + } + + if (Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)JobPointer); + } else { + *JobPointer = NULL; + } +} + +_Use_decl_annotations_ +HRESULT +CreatePerfectHashTableImplChm01Async( + PPERFECT_HASH_TABLE Table, + HANDLE IoCompletionPort + ) +{ + HRESULT Result; + PCHM01_ASYNC_JOB Job; + + Result = Chm01AsyncCreateJob(Table, IoCompletionPort, &Job); + if (FAILED(Result)) { + return Result; + } + + Result = Chm01AsyncSubmitJob(Job); + if (FAILED(Result)) { + Chm01AsyncDestroyJob(&Job); + return Result; + } + + Chm01AsyncWaitJob(Job); + Result = Job->LastResult; + + Chm01AsyncDestroyJob(&Job); + return Result; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01Async.h b/src/PerfectHash/Chm01Async.h new file mode 100644 index 00000000..1acbdb74 --- /dev/null +++ b/src/PerfectHash/Chm01Async.h @@ -0,0 +1,153 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + Chm01Async.h + +Abstract: + + This module defines the CHM01 asynchronous table creation state machine. + It decomposes the CHM01 algorithm into small, IOCP-driven work steps. + +--*/ + +#pragma once + +#include "PerfectHashAsync.h" + +typedef enum _CHM01_ASYNC_STATE { + Chm01AsyncStateInitialize = 0, + Chm01AsyncStateSolveGraphs, + Chm01AsyncStateFinalizeVerify, + Chm01AsyncStateFinalizeWaitSave, + Chm01AsyncStateFinalizeClose, + Chm01AsyncStateReleaseGraphs, + Chm01AsyncStateComplete, + Chm01AsyncStateError +} CHM01_ASYNC_STATE; + +typedef enum _CHM01_ASYNC_JOB_EVENT_ID { + Chm01AsyncJobEventSucceeded = 0, + Chm01AsyncJobEventCompleted, + Chm01AsyncJobEventShutdown, + Chm01AsyncJobEventFailed, + Chm01AsyncJobEventLowMemory, + Chm01AsyncJobEventInvalid +} CHM01_ASYNC_JOB_EVENT_ID; + +#define NUMBER_OF_CHM01_ASYNC_JOB_EVENTS (Chm01AsyncJobEventInvalid) + +typedef struct _CHM01_ASYNC_GRAPH_WORK CHM01_ASYNC_GRAPH_WORK; +typedef CHM01_ASYNC_GRAPH_WORK *PCHM01_ASYNC_GRAPH_WORK; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4820) +#endif + +typedef struct _CHM01_ASYNC_JOB { + PERFECT_HASH_ASYNC_CONTEXT Async; + PERFECT_HASH_ASYNC_WORK Work; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_CONTEXT Context; + PALLOCATOR Allocator; + PRTL Rtl; + PGRAPH *Graphs; + ULONG NumberOfGraphs; + ULONG Concurrency; + ULONG InitialConcurrency; + ULONG DispatchedGraphs; + ULONG IncreaseConcurrencyAfterMilliseconds; + ULONG Attempt; + volatile LONG FirstGraphStarted; + ULONG Padding1; + ULONGLONG FirstGraphStartMilliseconds; + ULONGLONG NextConcurrencyIncreaseMilliseconds; + GRAPH_INFO GraphInfo; + GRAPH_INFO PrevGraphInfo; + GRAPH_INFO_ON_DISK GraphInfoOnDisk; + PTABLE_INFO_ON_DISK TableInfoOnDisk; + HANDLE Events[NUMBER_OF_CHM01_ASYNC_JOB_EVENTS]; + HANDLE SaveEvents[NUMBER_OF_SAVE_FILE_EVENTS]; + HANDLE PrepareEvents[NUMBER_OF_PREPARE_FILE_EVENTS]; + PCHM01_ASYNC_GRAPH_WORK *GraphWorkItems; + volatile LONG ActiveGraphs; + ULONG Padding2; + HANDLE GraphsCompleteEvent; + +#define EXPAND_AS_ASYNC_FILE_WORK_ITEM( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + FILE_WORK_ITEM Verb##Name; + + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASYNC_FILE_WORK_ITEM) + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASYNC_FILE_WORK_ITEM) + CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASYNC_FILE_WORK_ITEM) + +#undef EXPAND_AS_ASYNC_FILE_WORK_ITEM + + CHM01_ASYNC_STATE State; + ULONG SliceBudget; + ULONG Flags; + HRESULT LastResult; + HANDLE CompletionEvent; +} CHM01_ASYNC_JOB; +typedef CHM01_ASYNC_JOB *PCHM01_ASYNC_JOB; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI CHM01_ASYNC_CREATE_JOB)( + _In_ PPERFECT_HASH_TABLE Table, + _In_ HANDLE IoCompletionPort, + _Outptr_ PCHM01_ASYNC_JOB *Job + ); +typedef CHM01_ASYNC_CREATE_JOB *PCHM01_ASYNC_CREATE_JOB; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI CHM01_ASYNC_SUBMIT_JOB)( + _In_ PCHM01_ASYNC_JOB Job + ); +typedef CHM01_ASYNC_SUBMIT_JOB *PCHM01_ASYNC_SUBMIT_JOB; + +typedef +VOID +(NTAPI CHM01_ASYNC_WAIT_JOB)( + _In_ PCHM01_ASYNC_JOB Job + ); +typedef CHM01_ASYNC_WAIT_JOB *PCHM01_ASYNC_WAIT_JOB; + +typedef +VOID +(NTAPI CHM01_ASYNC_DESTROY_JOB)( + _Inout_ PCHM01_ASYNC_JOB *Job + ); +typedef CHM01_ASYNC_DESTROY_JOB *PCHM01_ASYNC_DESTROY_JOB; + +extern CHM01_ASYNC_CREATE_JOB Chm01AsyncCreateJob; +extern CHM01_ASYNC_SUBMIT_JOB Chm01AsyncSubmitJob; +extern CHM01_ASYNC_WAIT_JOB Chm01AsyncWaitJob; +extern CHM01_ASYNC_DESTROY_JOB Chm01AsyncDestroyJob; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI CHM01_ASYNC_CREATE_TABLE)( + _In_ PPERFECT_HASH_TABLE Table, + _In_ HANDLE IoCompletionPort + ); +typedef CHM01_ASYNC_CREATE_TABLE *PCHM01_ASYNC_CREATE_TABLE; + +extern CHM01_ASYNC_CREATE_TABLE CreatePerfectHashTableImplChm01Async; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01Compat.c b/src/PerfectHash/Chm01Compat.c index 51910227..5926a893 100644 --- a/src/PerfectHash/Chm01Compat.c +++ b/src/PerfectHash/Chm01Compat.c @@ -428,11 +428,6 @@ Return Value: TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); - ASSERT(!TlsContext->Flags.DisableGlobalAllocatorComponent); - ASSERT(!TlsContext->Flags.CustomAllocatorDetailsPresent); - ASSERT(!TlsContext->HeapCreateFlags); - ASSERT(!TlsContext->HeapMinimumSize); - TlsContextDisableGlobalAllocator(TlsContext); TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; diff --git a/src/PerfectHash/Chm01FileWork.c b/src/PerfectHash/Chm01FileWork.c index 3df4d7b9..9d4a6e97 100644 --- a/src/PerfectHash/Chm01FileWork.c +++ b/src/PerfectHash/Chm01FileWork.c @@ -21,6 +21,8 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashContextIocp.h" +#include "PerfectHashIocpBufferPool.h" // // File work callback array. @@ -40,6 +42,805 @@ FILE_WORK_CALLBACK_IMPL *FileCallbacks[] = { NULL }; +static +VOID +Chm01GetFileWorkBufferPoolState( + _In_ PPERFECT_HASH_CONTEXT Context, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL **PoolsPointer, + _Out_ PULONG *PoolCountPointer, + _Outptr_ PGUARDED_LIST *BufferListPointer, + _Outptr_ PLIST_ENTRY *OversizeListPointer, + _Outptr_ PULONG *OversizeCountPointer, + _Outptr_ PSRWLOCK *PoolLockPointer, + _Out_ PUSHORT NumaNodePointer + ) +{ + PPERFECT_HASH_IOCP_NODE Node; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer) || + !ARGUMENT_PRESENT(BufferListPointer) || + !ARGUMENT_PRESENT(OversizeListPointer) || + !ARGUMENT_PRESENT(OversizeCountPointer) || + !ARGUMENT_PRESENT(PoolLockPointer) || + !ARGUMENT_PRESENT(NumaNodePointer)) { + return; + } + + Node = Context->IocpNode; + if (Node) { + *PoolsPointer = &Node->FileWorkBufferPools; + *PoolCountPointer = &Node->FileWorkBufferPoolCount; + *BufferListPointer = Node->FileWorkBufferList; + *OversizeListPointer = &Node->FileWorkOversizePools; + *OversizeCountPointer = &Node->FileWorkOversizePoolCount; + *PoolLockPointer = &Node->FileWorkBufferPoolLock; + *NumaNodePointer = (USHORT)Node->NodeId; + } else { + *PoolsPointer = &Context->FileWorkBufferPools; + *PoolCountPointer = &Context->FileWorkBufferPoolCount; + *BufferListPointer = Context->FileWorkBufferList; + *OversizeListPointer = &Context->FileWorkOversizePools; + *OversizeCountPointer = &Context->FileWorkOversizePoolCount; + *PoolLockPointer = &Context->Lock; + *NumaNodePointer = 0; + } +} + +static +HRESULT +Chm01EnsureFileWorkBufferPools( + _In_ PPERFECT_HASH_CONTEXT Context, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolsPointer, + _Out_ PULONG PoolCountPointer, + _Outptr_ PGUARDED_LIST *BufferListPointer, + _Out_ PUSHORT NumaNodePointer + ) +{ + HRESULT Result; + ULONG Flags; + ULONG PoolIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer) || + !ARGUMENT_PRESENT(BufferListPointer) || + !ARGUMENT_PRESENT(NumaNodePointer)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + Chm01GetFileWorkBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!PoolArrayPointer || !PoolCountField || !BufferList) { + return E_UNEXPECTED; + } + + Pools = *PoolArrayPointer; + if (!Pools) { + AcquireSRWLockExclusive(PoolLock); + Pools = *PoolArrayPointer; + if (!Pools) { + PoolCount = PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT; + Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + PoolCount, + sizeof(*Pools) + ); + if (!Pools) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; + } + + Flags = UseIocpBufferGuardPages(Context) ? + PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES : + 0; + for (PoolIndex = 0; PoolIndex < PoolCount; PoolIndex++) { + PayloadSize = + PerfectHashIocpBufferGetPayloadSizeFromClassIndex( + (LONG)PoolIndex + ); + + Result = PerfectHashIocpBufferPoolInitialize( + Rtl, + &Pools[PoolIndex], + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList + ); + + if (FAILED(Result)) { + ReleaseSRWLockExclusive(PoolLock); + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Pools + ); + return Result; + } + } + + *PoolArrayPointer = Pools; + *PoolCountField = PoolCount; + } + ReleaseSRWLockExclusive(PoolLock); + } + + *PoolsPointer = Pools; + *PoolCountPointer = *PoolCountField; + *BufferListPointer = BufferList; + *NumaNodePointer = NumaNode; + return S_OK; +} + +static +HRESULT +Chm01GetOversizeFileWorkBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG PayloadSize, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + ULONG Flags; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PLIST_ENTRY Entry; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + Chm01GetFileWorkBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!OversizeList || !OversizeCount || !PoolLock) { + return E_UNEXPECTED; + } + + AcquireSRWLockExclusive(PoolLock); + + Entry = OversizeList->Flink; + while (Entry != OversizeList) { + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + if (Pool->PayloadSize == PayloadSize) { + ReleaseSRWLockExclusive(PoolLock); + *PoolPointer = Pool; + return S_OK; + } + + Entry = Entry->Flink; + } + + Pool = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*Pool) + ); + if (!Pool) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; + } + + Flags = PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE; + if (UseIocpBufferGuardPages(Context)) { + Flags |= PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES; + } + Result = PerfectHashIocpBufferPoolInitialize(Rtl, + Pool, + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList); + + if (FAILED(Result)) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + ReleaseSRWLockExclusive(PoolLock); + return Result; + } + + InsertTailList(OversizeList, &Pool->ListEntry); + (*OversizeCount)++; + ReleaseSRWLockExclusive(PoolLock); + + *PoolPointer = Pool; + return S_OK; +} + +static +HRESULT +Chm01GetFileWorkBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + LONG ClassIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + PGUARDED_LIST BufferList; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Rtl = Context->Rtl; + if (!Rtl) { + return E_UNEXPECTED; + } + + ClassIndex = PerfectHashIocpBufferGetClassIndex(Rtl, + RequiredPayloadBytes); + if (ClassIndex >= 0) { + Result = Chm01EnsureFileWorkBufferPools(Context, + &Pools, + &PoolCount, + &BufferList, + &NumaNode); + if (FAILED(Result)) { + return Result; + } + + if ((ULONG)ClassIndex >= PoolCount) { + return E_INVALIDARG; + } + + *PoolPointer = &Pools[ClassIndex]; + return S_OK; + } + + PayloadSize = Rtl->RoundUpPowerOfTwo64(RequiredPayloadBytes); + if (PayloadSize == 0) { + return E_INVALIDARG; + } + + return Chm01GetOversizeFileWorkBufferPool(Context, + PayloadSize, + PoolPointer); +} + +static +HRESULT +Chm01AcquireFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONG FileIndex, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + HRESULT Result; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER Buffer; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + UNREFERENCED_PARAMETER(FileIndex); + + if (!ARGUMENT_PRESENT(BufferPointer) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + *BufferPointer = NULL; + *PoolPointer = NULL; + + if (!ARGUMENT_PRESENT(Context)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + Result = Chm01GetFileWorkBufferPool(Context, + RequiredPayloadBytes, + &Pool); + if (FAILED(Result)) { + return Result; + } + + Result = PerfectHashIocpBufferPoolAcquire(Rtl, + Allocator, + Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + if (Buffer->PayloadSize < RequiredPayloadBytes) { + PerfectHashIocpBufferPoolRelease(Pool, Buffer); + return E_FAIL; + } + + *PoolPointer = Pool; + *BufferPointer = Buffer; + return S_OK; +} + +static +VOID +Chm01ClearFileWorkBuffer( + _In_ PPERFECT_HASH_FILE File + ) +{ + if (!ARGUMENT_PRESENT(File)) { + return; + } + + File->IocpBufferPool = NULL; + File->IocpBuffer = NULL; + File->BaseAddress = NULL; +} + +static +VOID +Chm01ReleaseIocpBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + UNREFERENCED_PARAMETER(Context); + + if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + PerfectHashIocpBufferPoolRelease(Pool, Buffer); +} + +static +VOID +Chm01ReleaseFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PPERFECT_HASH_FILE File + ) +{ + if (!ARGUMENT_PRESENT(File)) { + return; + } + + if (File->IocpBufferPool && File->IocpBuffer) { + Chm01ReleaseIocpBuffer(Context, + File->IocpBufferPool, + File->IocpBuffer); + } + + Chm01ClearFileWorkBuffer(File); +} + +static +VOID +Chm01RequeueFileWorkItem( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PFILE_WORK_ITEM Item + ) +{ + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(Item)) { + return; + } + + InsertTailFileWork(Context, &Item->ListEntry); + PerfectHashContextSubmitFileWork(Context); +} + +static +HRESULT +Chm01EnsureFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONG FileIndex, + _In_ ULONGLONG RequiredPayloadBytes, + _In_ PPERFECT_HASH_FILE File + ) +{ + HRESULT Result; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(File)) { + return E_POINTER; + } + + if (File->IocpBuffer) { + if (File->IocpBuffer->PayloadSize < RequiredPayloadBytes) { + Chm01ReleaseFileWorkBuffer(Context, File); + } else { + File->BaseAddress = PerfectHashIocpBufferPayload(File->IocpBuffer); + return S_OK; + } + } + + Result = Chm01AcquireFileWorkBuffer(Context, + FileIndex, + RequiredPayloadBytes, + &Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + File->IocpBufferPool = Pool; + File->IocpBuffer = Buffer; + File->BaseAddress = PerfectHashIocpBufferPayload(Buffer); + + return S_OK; +} + +static +ULONGLONG +Chm01AdjustRequiredPayloadBytes( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ FILE_ID FileId, + _In_ ULONGLONG RequiredPayloadBytes + ) +{ + ULONGLONG Adjusted; + ULONGLONG BucketCount; + PRTL Rtl; + PCEOF_INIT Eof; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + + if (!ARGUMENT_PRESENT(Context)) { + return RequiredPayloadBytes; + } + + Rtl = Context->Rtl; + Table = Context->Table; + Keys = Table ? Table->Keys : NULL; + + if (!Rtl || !Table || !Keys) { + return RequiredPayloadBytes; + } + + if (!IsValidFileId(FileId)) { + return RequiredPayloadBytes; + } + + Eof = &EofInits[FileId]; + switch (Eof->Type) { + case EofInitTypeNumberOfKeysMultiplier: + BucketCount = Rtl->RoundUpPowerOfTwo64(Keys->NumberOfKeys.QuadPart); + if (BucketCount == 0 || Eof->Multiplier == 0) { + return RequiredPayloadBytes; + } + if (BucketCount > ((ULONGLONG)-1) / Eof->Multiplier) { + return RequiredPayloadBytes; + } + Adjusted = BucketCount * Eof->Multiplier; + break; + + case EofInitTypeNumberOfTableElementsMultiplier: + if (!Table->TableInfoOnDisk) { + return RequiredPayloadBytes; + } + BucketCount = Rtl->RoundUpPowerOfTwo64( + Table->TableInfoOnDisk->NumberOfTableElements.QuadPart + ); + if (BucketCount == 0 || Eof->Multiplier == 0) { + return RequiredPayloadBytes; + } + if (BucketCount > ((ULONGLONG)-1) / Eof->Multiplier) { + return RequiredPayloadBytes; + } + Adjusted = BucketCount * Eof->Multiplier; + break; + + case EofInitTypeNull: + case EofInitTypeDefault: + case EofInitTypeAssignedSize: + case EofInitTypeFixed: + case EofInitTypeNumberOfPages: + case EofInitTypeInvalid: + default: + return RequiredPayloadBytes; + } + + if (Adjusted < Context->SystemAllocationGranularity) { + Adjusted = Context->SystemAllocationGranularity; + } + + if (Adjusted < RequiredPayloadBytes) { + Adjusted = RequiredPayloadBytes; + } + + return Adjusted; +} + +typedef struct _CHM01_FILE_IOCP_WRITE { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_CONTEXT Context; + PFILE_WORK_ITEM Item; + PPERFECT_HASH_FILE File; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + ULONG BytesToWrite; + ULONG Padding1; +} CHM01_FILE_IOCP_WRITE; +typedef CHM01_FILE_IOCP_WRITE *PCHM01_FILE_IOCP_WRITE; + +static +HRESULT +Chm01FileWriteIocpCompletionCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ) +{ + ULONG EventIndex; + ULONG LastError; + HANDLE Event; + HRESULT Result; + PFILE_WORK_ITEM Item; + PPERFECT_HASH_CONTEXT Context; + PCHM01_FILE_IOCP_WRITE WorkItem; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(CompletionKey); + + WorkItem = (PCHM01_FILE_IOCP_WRITE)Overlapped; + if (!WorkItem) { + return E_POINTER; + } + + Context = WorkItem->Context; + Item = WorkItem->Item; + Event = NULL; + Result = S_OK; + LastError = ERROR_SUCCESS; + + if (!Success) { + LastError = GetLastError(); + SYS_ERROR(WriteFile_IocpCompletion); + Result = PH_E_SYSTEM_CALL_FAILED; + } else if (NumberOfBytesTransferred != WorkItem->BytesToWrite) { + LastError = ERROR_WRITE_FAULT; + SetLastError(LastError); + Result = PH_E_SYSTEM_CALL_FAILED; + } + + if (WorkItem->Buffer) { + WorkItem->Buffer->BytesWritten = NumberOfBytesTransferred; + } + + if (WorkItem->Pool && WorkItem->Buffer) { + Chm01ReleaseIocpBuffer(Context, WorkItem->Pool, WorkItem->Buffer); + } + + if (WorkItem->File) { + Chm01ClearFileWorkBuffer(WorkItem->File); + } + + if (Item) { + Item->LastResult = Result; + Item->LastError = (LONG)LastError; + if (FAILED(Result)) { + InterlockedIncrement(&Item->NumberOfErrors); + } + } + + if (Context && Item && !IsCloseFileWorkId(Item->FileWorkId)) { + EventIndex = FileWorkIdToEventIndex(Item->FileWorkId); + Event = *(&Context->FirstPreparedEvent + EventIndex); + if (Event) { + SetEvent(Event); + } + } + + if (Context && Context->Allocator) { + Context->Allocator->Vtbl->FreePointer( + Context->Allocator, + (PVOID *)&WorkItem + ); + } + + return S_OK; +} + +static +HRESULT +Chm01WriteFileFromBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PFILE_WORK_ITEM Item, + _In_ PPERFECT_HASH_FILE File, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + BOOL Success; + DWORD BytesWritten; + DWORD BytesToWrite; + ULONG LastError; + HRESULT Result = S_OK; + LARGE_INTEGER BytesRemaining; + PVOID BaseAddress; + PALLOCATOR Allocator; + PCHM01_FILE_IOCP_WRITE WorkItem; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(File) || + !ARGUMENT_PRESENT(Item) || + !ARGUMENT_PRESENT(Pool) || + !ARGUMENT_PRESENT(Buffer)) { + return E_POINTER; + } + + BytesRemaining.QuadPart = File->NumberOfBytesWritten.QuadPart; + if (BytesRemaining.QuadPart <= 0) { + PerfectHashIocpBufferPoolRelease(Pool, Buffer); + return S_OK; + } + +#ifdef PH_WINDOWS + if (BytesRemaining.QuadPart > (LONGLONG)Buffer->PayloadSize || + BytesRemaining.QuadPart >= (LONGLONG)(1ULL << 30)) { + BOOL LogLargeWrite; + WCHAR LogBuffer[512]; + int CharCount; + HANDLE LogHandle; + DWORD BytesWrittenLog; + + LogLargeWrite = (GetEnvironmentVariableW( + L"PH_LOG_LARGE_FILE_WRITE", NULL, 0) != 0); + + if (LogLargeWrite) { + CharCount = _snwprintf_s( + LogBuffer, + ARRAYSIZE(LogBuffer), + _TRUNCATE, + L"PH_LARGE_WRITE: FileId=%lu FileWorkId=%lu " + L"BytesRemaining=%lld PayloadSize=%llu " + L"EndOfFile=%lld Path=%wZ\n", + (ULONG)Item->FileId, + (ULONG)Item->FileWorkId, + BytesRemaining.QuadPart, + Buffer->PayloadSize, + File->FileInfo.EndOfFile.QuadPart, + (File->Path ? &File->Path->FullPath : &NullUnicodeString) + ); + + if (CharCount > 0) { + OutputDebugStringW(LogBuffer); + LogHandle = CreateFileW(L"PerfectHashLargeWrite.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (IsValidHandle(LogHandle)) { + WriteFile(LogHandle, + LogBuffer, + (DWORD)(CharCount * sizeof(WCHAR)), + &BytesWrittenLog, + NULL); + CloseHandle(LogHandle); + } + } + } + + PH_RAISE(PH_E_INVALID_END_OF_FILE); + } +#endif + + if (BytesRemaining.HighPart != 0) { + PH_RAISE(PH_E_INVALID_END_OF_FILE); + } + + BaseAddress = PerfectHashIocpBufferPayload(Buffer); + BytesToWrite = BytesRemaining.LowPart; + BytesWritten = 0; + + Allocator = Context->Allocator; + if (!Allocator) { + Result = E_UNEXPECTED; + goto End; + } + + WorkItem = (PCHM01_FILE_IOCP_WRITE)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*WorkItem) + ); + if (!WorkItem) { + Result = E_OUTOFMEMORY; + goto End; + } + + WorkItem->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + WorkItem->Iocp.Flags = PH_IOCP_WORK_FLAG_FILE_IO; + WorkItem->Iocp.CompletionCallback = + Chm01FileWriteIocpCompletionCallback; + WorkItem->Context = Context; + WorkItem->Item = Item; + WorkItem->File = File; + WorkItem->Pool = Pool; + WorkItem->Buffer = Buffer; + WorkItem->BytesToWrite = BytesToWrite; + + Success = WriteFile(File->FileHandle, + BaseAddress, + BytesToWrite, + &BytesWritten, + &WorkItem->Iocp.Overlapped); + + if (!Success) { + LastError = GetLastError(); + if (LastError != ERROR_IO_PENDING) { + SetLastError(LastError); + SYS_ERROR(WriteFile); + Result = PH_E_SYSTEM_CALL_FAILED; + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + goto End; + } + } + + return S_FALSE; + +End: + + Chm01ReleaseIocpBuffer(Context, Pool, Buffer); + Chm01ClearFileWorkBuffer(File); + return Result; +} + // // Forward decls. // @@ -127,6 +928,7 @@ Return Value: PGRAPH_INFO Info; FILE_ID FileId; FILE_WORK_ID FileWorkId; + FILE_WORK_ID PrepareWorkId; CONTEXT_FILE_ID ContextFileId; PFILE_WORK_CALLBACK_IMPL Impl = NULL; PPERFECT_HASH_TABLE Table; @@ -140,6 +942,8 @@ Return Value: PPERFECT_HASH_CONTEXT Context; BOOLEAN IsContextFile = FALSE; BOOLEAN IsMakefileOrMainMkOrCMakeListsTextFile = FALSE; + BOOLEAN HasSaveCallback = FALSE; + BOOLEAN HasPrepareCallback = FALSE; // // Initialize aliases. @@ -179,6 +983,25 @@ Return Value: Item->FileId = FileId = FileWorkIdToFileId(FileWorkId); + if (SkipContextFileWork(Context) && + IsValidContextFileId((CONTEXT_FILE_ID)FileId)) { + Item->Flags.IsContextFile = TRUE; + Item->LastResult = S_OK; + Item->LastError = 0; + if (Event) { +#ifdef PH_WINDOWS + if (Item->Instance) { + SetEventWhenCallbackReturns(Item->Instance, Event); + } else { + SetEvent(Event); + } +#else + SetEvent(Event); +#endif + } + return; + } + // // Determine if this is a context file. Context files get treated // differently to normal table output files in that they are only @@ -400,17 +1223,25 @@ Return Value: break; case EofInitTypeNumberOfKeysMultiplier: - EndOfFile.QuadPart = ( - (LONGLONG)Keys->NumberOfKeys.QuadPart * - Eof->Multiplier - ); + { + LONGLONG Computed = (LONGLONG)( + Keys->NumberOfKeys.QuadPart * Eof->Multiplier + ); + if (Computed > EndOfFile.QuadPart) { + EndOfFile.QuadPart = Computed; + } + } break; case EofInitTypeNumberOfTableElementsMultiplier: - EndOfFile.QuadPart = ( - NumberOfTableElements.QuadPart * - Eof->Multiplier - ); + { + LONGLONG Computed = (LONGLONG)( + NumberOfTableElements.QuadPart * Eof->Multiplier + ); + if (Computed > EndOfFile.QuadPart) { + EndOfFile.QuadPart = Computed; + } + } break; case EofInitTypeNumberOfPages: @@ -576,19 +1407,147 @@ Return Value: PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); } +#ifdef PH_WINDOWS + if (EndOfFile.QuadPart >= (LONGLONG)(1ULL << 30)) { + BOOL LogLargeEof; + BOOL FailLargeEof; + ULONGLONG NumberOfKeys; + ULONGLONG LargeEofTableElements; + ULONGLONG AssignedSizeInBytes; + ULONG AllocationGranularity; + PCUNICODE_STRING Extension; + PCUNICODE_STRING Suffix; + WCHAR LogBuffer[512]; + int CharCount; + HANDLE LogHandle; + DWORD BytesWritten; + + LogLargeEof = (GetEnvironmentVariableW( + L"PH_LOG_LARGE_EOF", NULL, 0) != 0); + FailLargeEof = (GetEnvironmentVariableW( + L"PH_FAIL_LARGE_EOF", NULL, 0) != 0); + + if (LogLargeEof || FailLargeEof) { + NumberOfKeys = Keys ? Keys->NumberOfKeys.QuadPart : 0; + LargeEofTableElements = ( + TableInfo ? + TableInfo->NumberOfTableElements.QuadPart : + 0 + ); + AssignedSizeInBytes = ( + Info ? Info->AssignedSizeInBytes : 0 + ); + AllocationGranularity = ( + Context ? Context->SystemAllocationGranularity : 0 + ); + Extension = GetFileWorkItemExtension(FileWorkId); + Suffix = GetFileWorkItemSuffix(FileWorkId); + + CharCount = _snwprintf_s( + LogBuffer, + ARRAYSIZE(LogBuffer), + _TRUNCATE, + L"PH_LARGE_EOF: FileWorkId=%lu FileId=%lu " + L"EofType=%lu Multiplier=%llu EndOfFile=%lld " + L"Keys=%llu TableElements=%llu AssignedSize=%llu " + L"Granularity=%lu Suffix=%wZ Ext=%wZ Path=%wZ\n", + (ULONG)FileWorkId, + (ULONG)FileId, + (ULONG)Eof->Type, + (ULONGLONG)Eof->Multiplier, + EndOfFile.QuadPart, + NumberOfKeys, + LargeEofTableElements, + AssignedSizeInBytes, + AllocationGranularity, + Suffix ? Suffix : &NullUnicodeString, + Extension ? Extension : &NullUnicodeString, + Path ? &Path->FullPath : &NullUnicodeString + ); + + if (CharCount > 0) { + OutputDebugStringW(LogBuffer); + + LogHandle = CreateFileW(L"PerfectHashLargeEof.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (IsValidHandle(LogHandle)) { + WriteFile(LogHandle, + LogBuffer, + (DWORD)(CharCount * sizeof(WCHAR)), + &BytesWritten, + NULL); + CloseHandle(LogHandle); + } + } + } + + if (FailLargeEof) { + Result = PH_E_INVALID_END_OF_FILE; + PH_ERROR(PrepareFileChm01_LargeEof, Result); + goto End; + } + } +#endif + Result = PrepareFileChm01(Table, Item, Path, &EndOfFile, DependentEvent); + if (Result == S_FALSE) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } + if (FAILED(Result)) { PH_ERROR(PrepareFileChm01, Result); goto End; } + if (UseOverlappedIo(Context) && Impl) { + ULONGLONG BufferPayloadBytes; + + if (!*File) { + Result = E_UNEXPECTED; + goto End; + } + + BufferPayloadBytes = Chm01AdjustRequiredPayloadBytes( + Context, + FileId, + (ULONGLONG)EndOfFile.QuadPart + ); + + Result = Chm01EnsureFileWorkBuffer( + Context, + FileIndex, + BufferPayloadBytes, + *File + ); + if (FAILED(Result)) { + goto End; + } + + // + // Reset bytes written for IOCP buffers. Prepare routines must + // explicitly advance this as they generate output. + // + + (*File)->NumberOfBytesWritten.QuadPart = 0; + } + if (Impl) { Result = Impl(Context, Item); + if (Result == S_FALSE) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } if (FAILED(Result)) { // @@ -600,6 +1559,31 @@ Return Value: } } + if (UseOverlappedIo(Context) && Impl) { + FILE_WORK_ID SaveWorkId; + + SaveWorkId = FileWorkIdToDependentId(FileWorkId); + HasSaveCallback = (FileCallbacks[SaveWorkId] != NULL); + + if (!HasSaveCallback) { + Result = Chm01WriteFileFromBuffer( + Context, + Item, + *File, + (*File)->IocpBufferPool, + (*File)->IocpBuffer + ); + if (Result == S_FALSE) { + Chm01ClearFileWorkBuffer(*File); + return; + } + Chm01ClearFileWorkBuffer(*File); + if (FAILED(Result)) { + goto End; + } + } + } + } else if (IsSaveFileWorkId(FileWorkId)) { ULONG WaitResult; @@ -611,7 +1595,14 @@ Return Value: DependentEventIndex = FileWorkIdToDependentEventIndex(FileWorkId); DependentEvent = *(&Context->FirstPreparedEvent + DependentEventIndex); - WaitResult = WaitForSingleObject(DependentEvent, INFINITE); + WaitResult = WaitForSingleObject( + DependentEvent, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForSingleObject); Result = PH_E_SYSTEM_CALL_FAILED; @@ -632,8 +1623,57 @@ Return Value: goto End; } + PrepareWorkId = FileWorkIdToDependentId(FileWorkId); + HasPrepareCallback = (FileCallbacks[PrepareWorkId] != NULL); + + if (UseOverlappedIo(Context)) { + if (Impl || (*File)->IocpBuffer) { + ULONGLONG BufferPayloadBytes; + + BufferPayloadBytes = Chm01AdjustRequiredPayloadBytes( + Context, + FileId, + (ULONGLONG)(*File)->FileInfo.EndOfFile.QuadPart + ); + + Result = Chm01EnsureFileWorkBuffer( + Context, + FileIndex, + BufferPayloadBytes, + *File + ); + if (FAILED(Result)) { + goto End; + } + } + } + + if (UseOverlappedIo(Context) && Impl && !HasPrepareCallback) { + (*File)->NumberOfBytesWritten.QuadPart = 0; + } + if (Impl) { Result = Impl(Context, Item); + if (Result == S_FALSE) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } + if (FAILED(Result)) { + goto End; + } + } + + if (UseOverlappedIo(Context) && (*File)->IocpBuffer) { + Result = Chm01WriteFileFromBuffer(Context, + Item, + *File, + (*File)->IocpBufferPool, + (*File)->IocpBuffer); + if (Result == S_FALSE) { + Chm01ClearFileWorkBuffer(*File); + return; + } + Chm01ClearFileWorkBuffer(*File); if (FAILED(Result)) { goto End; } @@ -646,15 +1686,17 @@ Return Value: // has to do when the close work items are submitted in parallel. // - Result = UnmapFileChm01(Table, Item); - if (FAILED(Result)) { + if (!UseOverlappedIo(Context)) { + Result = UnmapFileChm01(Table, Item); + if (FAILED(Result)) { - // - // Nothing needs doing here. The Result will bubble back up - // via the normal mechanisms. - // + // + // Nothing needs doing here. The Result will bubble back up + // via the normal mechanisms. + // - NOTHING; + NOTHING; + } } } else { @@ -698,6 +1740,15 @@ Return Value: RELEASE(Path); + if (UseOverlappedIo(Context) && + File && + *File && + (*File)->IocpBuffer) { + if (FAILED(Result) || IsCloseFileWorkId(FileWorkId)) { + Chm01ReleaseFileWorkBuffer(Context, *File); + } + } + // // If the item's UUID string buffer is non-NULL here, the downstream routine // did not successfully take ownership of it, and thus, we're responsbile @@ -727,7 +1778,11 @@ Return Value: if (Event) { #ifdef PH_WINDOWS - SetEventWhenCallbackReturns(Item->Instance, Event); + if (Item->Instance) { + SetEventWhenCallbackReturns(Item->Instance, Event); + } else { + SetEvent(Event); + } #else SetEvent(Event); #endif @@ -790,6 +1845,21 @@ Return Value: HRESULT Result = S_OK; PPERFECT_HASH_FILE File = NULL; PPERFECT_HASH_DIRECTORY Directory = NULL; + PPERFECT_HASH_CONTEXT Context = NULL; + PERFECT_HASH_FILE_CREATE_FLAGS CreateFlags = { 0 }; + PPERFECT_HASH_FILE_CREATE_FLAGS CreateFlagsPointer = NULL; + + // + // Dereference the file pointer provided by the caller. If NULL, this + // is the first preparation request for the given file instance. Otherwise, + // a table resize event has occurred, which means a file rename needs to be + // scheduled (as we include the number of table elements in the file name), + // and the mapping size needs to be extended (as a larger table size means + // larger files are required to capture table data). + // + + File = *Item->FilePointer; + Context = Table->Context; // // If a dependent event has been provided, wait for this object to become @@ -798,8 +1868,14 @@ Return Value: if (IsValidHandle(DependentEvent)) { ULONG WaitResult; + DWORD Timeout; + + Timeout = (Context && UseOverlappedIo(Context)) ? 0 : INFINITE; - WaitResult = WaitForSingleObject(DependentEvent, INFINITE); + WaitResult = WaitForSingleObject(DependentEvent, Timeout); + if (WaitResult == WAIT_TIMEOUT && Timeout == 0) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForSingleObject); Result = PH_E_SYSTEM_CALL_FAILED; @@ -809,14 +1885,6 @@ Return Value: // // Dereference the file pointer provided by the caller. If NULL, this - // is the first preparation request for the given file instance. Otherwise, - // a table resize event has occurred, which means a file rename needs to be - // scheduled (as we include the number of table elements in the file name), - // and the mapping size needs to be extended (as a larger table size means - // larger files are required to capture table data). - // - - File = *Item->FilePointer; if (!File) { @@ -847,11 +1915,16 @@ Return Value: Directory = Table->OutputDirectory; } + if (Context && UseOverlappedIo(Context)) { + CreateFlags.SkipMapping = TRUE; + CreateFlagsPointer = &CreateFlags; + } + Result = File->Vtbl->Create(File, Path, EndOfFile, Directory, - NULL); + CreateFlagsPointer); if (FAILED(Result)) { PH_ERROR(PerfectHashFileCreate, Result); @@ -859,6 +1932,26 @@ Return Value: goto Error; } + if (Context && + UseOverlappedIo(Context) && + Context->FileWorkIoCompletionPort) { + HANDLE PortHandle; + + PortHandle = CreateIoCompletionPort( + File->FileHandle, + Context->FileWorkIoCompletionPort, + (ULONG_PTR)File, + 0 + ); + + if (!PortHandle) { + SYS_ERROR(CreateIoCompletionPort_FileWork); + Result = PH_E_SYSTEM_CALL_FAILED; + RELEASE(File); + goto Error; + } + } + // // Set the file ID and then update the file pointer. // diff --git a/src/PerfectHash/Chm01FileWork.h b/src/PerfectHash/Chm01FileWork.h index 799a3df5..1899f660 100644 --- a/src/PerfectHash/Chm01FileWork.h +++ b/src/PerfectHash/Chm01FileWork.h @@ -13,6 +13,8 @@ Module Name: --*/ +#pragma once + #include "stdafx.h" #define EXPAND_AS_CALLBACK_DECL( \ @@ -37,7 +39,6 @@ SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CALLBACK_DECL); #define PrepareCSourceTableDataFileChm01 NULL #define PreparePythonFileChm01 NULL #define PreparePythonTestFileChm01 NULL -#define PrepareCppHeaderOnlyFileChm01 NULL #define PrepareCppHeaderOnlyTestFileChm01 NULL #define PrepareRustCargoTomlFileChm01 NULL #define PrepareRustLibFileChm01 NULL diff --git a/src/PerfectHash/Chm01FileWorkCHeaderFile.c b/src/PerfectHash/Chm01FileWorkCHeaderFile.c index b4a0762e..e6089748 100644 --- a/src/PerfectHash/Chm01FileWorkCHeaderFile.c +++ b/src/PerfectHash/Chm01FileWorkCHeaderFile.c @@ -56,6 +56,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" extern const STRING CompiledPerfectHashTableRoutinesPreCSourceRawCString; extern const STRING CompiledPerfectHashTableRoutinesCSourceRawCString; @@ -335,6 +336,15 @@ SaveCHeaderFileChm01( ); Algo.MaximumLength = Algo.Length; + if (UseOverlappedIo(Context)) { + if (!File->IocpBuffer || + File->NumberOfBytesWritten.QuadPart <= 0 || + (ULONGLONG)File->NumberOfBytesWritten.QuadPart >= + File->IocpBuffer->PayloadSize) { + return PH_E_INVALID_END_OF_FILE; + } + } + // // Pick up the offset from where we left off. // diff --git a/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c b/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c index 994d5c02..b0550161 100644 --- a/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c +++ b/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c @@ -16,6 +16,76 @@ Module Name: #include "stdafx.h" +_Use_decl_annotations_ +HRESULT +PrepareCppHeaderOnlyFileChm01( + PPERFECT_HASH_CONTEXT Context, + PFILE_WORK_ITEM Item + ) +{ + HRESULT Result = S_OK; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + PPERFECT_HASH_FILE File; + PTABLE_INFO_ON_DISK TableInfo; + ULONGLONG NumberOfKeys; + ULONGLONG NumberOfTableElements; + ULONGLONG KeyElementBytes; + ULONGLONG TableElementBytes; + ULONGLONG BaseSize; + ULONGLONG RequiredSize; + ULONGLONG AlignedSize; + LARGE_INTEGER EndOfFile; + + // + // Initialize aliases. + // + + Table = Context->Table; + Keys = Table->Keys; + File = *Item->FilePointer; + TableInfo = Table->TableInfoOnDisk; + + if (!File) { + return E_UNEXPECTED; + } + + NumberOfKeys = Keys->NumberOfKeys.QuadPart; + NumberOfTableElements = TableInfo->NumberOfTableElements.QuadPart; + + // + // Estimate required output size. Use fixed-width hex output estimates + // plus a base allowance for headers and function bodies. + // + + BaseSize = Context->SystemAllocationGranularity; + TableElementBytes = 16; + KeyElementBytes = ( + Keys->OriginalKeySizeType == LongLongType ? 24 : 16 + ); + + RequiredSize = ( + BaseSize + + (NumberOfTableElements * TableElementBytes) + + (NumberOfKeys * KeyElementBytes) + ); + + AlignedSize = ALIGN_UP(RequiredSize, + Context->SystemAllocationGranularity); + + if (AlignedSize > (ULONGLONG)File->FileInfo.EndOfFile.QuadPart) { + EndOfFile.QuadPart = (LONGLONG)AlignedSize; + AcquirePerfectHashFileLockExclusive(File); + Result = File->Vtbl->Extend(File, &EndOfFile); + ReleasePerfectHashFileLockExclusive(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileExtend, Result); + } + } + + return Result; +} + _Use_decl_annotations_ HRESULT SaveCppHeaderOnlyFileChm01( diff --git a/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c b/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c index 6b15b745..7a3234de 100644 --- a/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c +++ b/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c @@ -61,6 +61,7 @@ Module Name: #include "stdafx.h" #include "CompiledPerfectHashTableBenchmarkIndexInline_CSource_RawCString.h" +#include "PerfectHashIocpBufferPool.h" extern const STRING no_sal2CHeaderRawCString; extern const STRING CompiledPerfectHashCHeaderRawCString; @@ -507,6 +508,15 @@ SaveCppSourceUnityFileChm01( ); Algo.MaximumLength = Algo.Length; + if (UseOverlappedIo(Context)) { + if (!File->IocpBuffer || + File->NumberOfBytesWritten.QuadPart <= 0 || + (ULONGLONG)File->NumberOfBytesWritten.QuadPart >= + File->IocpBuffer->PayloadSize) { + return PH_E_INVALID_END_OF_FILE; + } + } + // // Pick up the offset from where we left off. // diff --git a/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c b/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c index 297b4692..b1c56eca 100644 --- a/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c +++ b/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c @@ -63,10 +63,16 @@ PrepareMakefileMainMkFileChm01( ASSERT(ARRAYSIZE(PrepareEvents) == (SIZE_T)NumberOfEvents); - WaitResult = WaitForMultipleObjects(NumberOfEvents, - PrepareEvents, - WaitForAllEvents, - INFINITE); + WaitResult = WaitForMultipleObjects( + NumberOfEvents, + PrepareEvents, + WaitForAllEvents, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForMultipleObjects); diff --git a/src/PerfectHash/Chm01FileWorkTableInfoStream.c b/src/PerfectHash/Chm01FileWorkTableInfoStream.c index 06448ed7..90d67474 100644 --- a/src/PerfectHash/Chm01FileWorkTableInfoStream.c +++ b/src/PerfectHash/Chm01FileWorkTableInfoStream.c @@ -82,7 +82,14 @@ SaveTableInfoStreamChm01( // to save the timings for verification to the header. // - WaitResult = WaitForSingleObject(Context->VerifiedTableEvent, INFINITE); + WaitResult = WaitForSingleObject( + Context->VerifiedTableEvent, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForSingleObject); Result = PH_E_SYSTEM_CALL_FAILED; diff --git a/src/PerfectHash/Chm01FileWorkVSSolutionFile.c b/src/PerfectHash/Chm01FileWorkVSSolutionFile.c index c484db70..8bee86e9 100644 --- a/src/PerfectHash/Chm01FileWorkVSSolutionFile.c +++ b/src/PerfectHash/Chm01FileWorkVSSolutionFile.c @@ -123,10 +123,16 @@ PrepareVSSolutionFileChm01( ASSERT(ARRAYSIZE(PrepareEvents) == (SIZE_T)NumberOfEvents); - WaitResult = WaitForMultipleObjects(NumberOfEvents, - PrepareEvents, - WaitForAllEvents, - INFINITE); + WaitResult = WaitForMultipleObjects( + NumberOfEvents, + PrepareEvents, + WaitForAllEvents, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForMultipleObjects); diff --git a/src/PerfectHash/Chm01Shared.c b/src/PerfectHash/Chm01Shared.c index 488fe238..7ae3ab96 100644 --- a/src/PerfectHash/Chm01Shared.c +++ b/src/PerfectHash/Chm01Shared.c @@ -884,6 +884,66 @@ Return Value: CopyInline(&GraphInfoOnDisk->Dimensions, Dim, sizeof(*Dim)); +#ifdef PH_WINDOWS + if (GetEnvironmentVariableW(L"PH_LOG_GRAPH_INFO", NULL, 0) != 0) { + if (TableInfoOnDisk->NumberOfTableElements.QuadPart >= (1ULL << 20) || + AssignedSizeInBytes >= (1ULL << 30)) { + WCHAR LogBuffer[512]; + int CharCount; + HANDLE LogHandle; + DWORD BytesWritten; + PCUNICODE_STRING KeysPath = NULL; + + if (Table->Keys && + Table->Keys->File && + Table->Keys->File->Path && + Table->Keys->File->Path->FullPath.Length > 0) { + KeysPath = &Table->Keys->File->Path->FullPath; + } + + CharCount = _snwprintf_s( + LogBuffer, + ARRAYSIZE(LogBuffer), + _TRUNCATE, + L"PH_GRAPH_INFO: Keys=%llu TableElems=%llu " + L"AssignedBytes=%llu Vertices=%llu Edges=%llu " + L"HashSize=%lu IndexSize=%lu Requested=%llu " + L"HashFunc=%lu MaskFunc=%lu Path=%wZ\n", + (Table->Keys ? Table->Keys->NumberOfKeys.QuadPart : 0), + TableInfoOnDisk->NumberOfTableElements.QuadPart, + AssignedSizeInBytes, + NumberOfVertices.QuadPart, + NumberOfEdges.QuadPart, + (ULONG)Table->HashSize, + (ULONG)Table->IndexSize, + Table->RequestedNumberOfTableElements.QuadPart, + (ULONG)Table->HashFunctionId, + (ULONG)Table->MaskFunctionId, + KeysPath ? KeysPath : &NullUnicodeString + ); + + if (CharCount > 0) { + OutputDebugStringW(LogBuffer); + LogHandle = CreateFileW(L"PerfectHashGraphInfo.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (IsValidHandle(LogHandle)) { + WriteFile(LogHandle, + LogBuffer, + (DWORD)(CharCount * sizeof(WCHAR)), + &BytesWritten, + NULL); + CloseHandle(LogHandle); + } + } + } + } +#endif + // // Capture ratios. // diff --git a/src/PerfectHash/Chm02.c b/src/PerfectHash/Chm02.c index 53f3ff42..56f61022 100644 --- a/src/PerfectHash/Chm02.c +++ b/src/PerfectHash/Chm02.c @@ -357,7 +357,7 @@ Return Value: ZeroStructInline(Verb##Name); \ Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_PREPARE_FILE_WORK() \ PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) @@ -588,7 +588,7 @@ Return Value: // if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, CancelPending); + PerfectHashContextWaitForFileWorkCallbacks(Context, CancelPending); } // @@ -899,7 +899,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->MainWork, TRUE); WaitForThreadpoolWorkCallbacks(Context->ConsoleWork, TRUE); if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, TRUE); + PerfectHashContextWaitForFileWorkCallbacks(Context, TRUE); } // @@ -944,14 +944,14 @@ Return Value: Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ Verb##Name.EndOfFile = EndOfFile; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_CLOSE_FILE_WORK() \ CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_CLOSE_FILE_WORK) SUBMIT_CLOSE_FILE_WORK(); - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); #define EXPAND_AS_CHECK_CLOSE_ERRORS( \ Verb, VUpper, Name, Upper, \ diff --git a/src/PerfectHash/Chm02Shared.c b/src/PerfectHash/Chm02Shared.c index 02bfbf1e..56723cd7 100644 --- a/src/PerfectHash/Chm02Shared.c +++ b/src/PerfectHash/Chm02Shared.c @@ -725,11 +725,6 @@ Return Value: TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); - ASSERT(!TlsContext->Flags.DisableGlobalAllocatorComponent); - ASSERT(!TlsContext->Flags.CustomAllocatorDetailsPresent); - ASSERT(!TlsContext->HeapCreateFlags); - ASSERT(!TlsContext->HeapMinimumSize); - TlsContextDisableGlobalAllocator(TlsContext); TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; diff --git a/src/PerfectHash/ExtractArg.c b/src/PerfectHash/ExtractArg.c index c501a832..5949df16 100644 --- a/src/PerfectHash/ExtractArg.c +++ b/src/PerfectHash/ExtractArg.c @@ -1040,6 +1040,14 @@ Return Value: ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(MaxSolveTimeInSeconds); + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(InitialPerFileConcurrency); + + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(MaxPerFileConcurrency); + + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER( + IncreaseConcurrencyAfterMilliseconds + ); + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(FunctionHookCallbackIgnoreRip); #define IS_VALUE_EQUAL(ValueName) \ diff --git a/src/PerfectHash/Graph.c b/src/PerfectHash/Graph.c index 574e11e5..d20a1a79 100644 --- a/src/PerfectHash/Graph.c +++ b/src/PerfectHash/Graph.c @@ -632,7 +632,9 @@ Return Value: } } InsertHeadFinishedWork(Context, &Graph->ListEntry); - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } return PH_S_STOP_GRAPH_SOLVING; } @@ -3175,7 +3177,9 @@ Return Value: // the graph contexts to finish and then assessing the context. // - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } // // Clear the caller's NewGraphPointer, as we're not going to be doing @@ -4261,7 +4265,9 @@ Return Value: if (Result != S_OK) { CONTEXT_END_TIMERS(Solve); SetStopSolving(Context); - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } return Result; } @@ -6276,7 +6282,9 @@ Return Value: // the graph contexts to finish and then assessing the context. // - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } // // Clear the caller's NewGraphPointer, as we're not going to be doing diff --git a/src/PerfectHash/GraphCu.c b/src/PerfectHash/GraphCu.c index b689a6c5..338dace5 100644 --- a/src/PerfectHash/GraphCu.c +++ b/src/PerfectHash/GraphCu.c @@ -967,7 +967,9 @@ Return Value: if (Result != S_OK) { CONTEXT_END_TIMERS(Solve); SetStopSolving(Context); - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } return Result; } diff --git a/src/PerfectHash/PerfectHash.vcxproj b/src/PerfectHash/PerfectHash.vcxproj index d2cff653..df270e9c 100644 --- a/src/PerfectHash/PerfectHash.vcxproj +++ b/src/PerfectHash/PerfectHash.vcxproj @@ -69,6 +69,7 @@ + @@ -148,6 +149,10 @@ + + + + @@ -158,6 +163,7 @@ + @@ -228,6 +234,7 @@ + @@ -247,6 +254,11 @@ + + + + + @@ -254,6 +266,7 @@ + diff --git a/src/PerfectHash/PerfectHash.vcxproj.filters b/src/PerfectHash/PerfectHash.vcxproj.filters index 2be77246..68b914a1 100644 --- a/src/PerfectHash/PerfectHash.vcxproj.filters +++ b/src/PerfectHash/PerfectHash.vcxproj.filters @@ -43,6 +43,9 @@ Source Files + + Source Files + Source Files @@ -58,6 +61,21 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + Source Files @@ -127,6 +145,9 @@ Source Files + + Source Files + Source Files @@ -393,6 +414,18 @@ Private Header Files + + Private Header Files + + + Private Header Files + + + Private Header Files + + + Private Header Files + Private Header Files @@ -417,6 +450,9 @@ Private Header Files + + Private Header Files + Private Header Files @@ -432,6 +468,9 @@ Private Header Files + + Private Header Files + Private Header Files diff --git a/src/PerfectHash/PerfectHashAsync.c b/src/PerfectHash/PerfectHashAsync.c new file mode 100644 index 00000000..2a1d4978 --- /dev/null +++ b/src/PerfectHash/PerfectHashAsync.c @@ -0,0 +1,300 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashAsync.c + +Abstract: + + This module implements the asynchronous work framework used by IOCP-based + execution paths. Work items are driven by per-item completion callbacks + and can cooperatively yield by re-posting to the completion port. + +--*/ + +#include "stdafx.h" + +static +HRESULT +PerfectHashAsyncRequeueWork( + _In_ PPERFECT_HASH_ASYNC_WORK Work + ) +{ + BOOL Success; + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext; + + AsyncContext = Work->AsyncContext; + if (!AsyncContext || !AsyncContext->IoCompletionPort) { + return E_UNEXPECTED; + } + + Success = PostQueuedCompletionStatus(AsyncContext->IoCompletionPort, + 0, + 0, + &Work->Iocp.Overlapped); + if (!Success) { + SYS_ERROR(PostQueuedCompletionStatus_AsyncRequeue); + return PH_E_SYSTEM_CALL_FAILED; + } + + return S_OK; +} + +static +HRESULT +PerfectHashAsyncCompleteWork( + _In_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ) +{ + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext; + + AsyncContext = Work->AsyncContext; + Work->LastResult = Result; + + if (Work->Complete) { + Work->Complete(Work, Result); + } + + if (AsyncContext) { + if (InterlockedDecrement(&AsyncContext->Outstanding) == 0) { + if (AsyncContext->OutstandingEvent) { + SetEvent(AsyncContext->OutstandingEvent); + } + } + } + + return Result; +} + +static +HRESULT +PerfectHashAsyncIocpCompletionCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ) +{ + HRESULT Result; + PPERFECT_HASH_ASYNC_WORK Work; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(CompletionKey); + UNREFERENCED_PARAMETER(NumberOfBytesTransferred); + UNREFERENCED_PARAMETER(Success); + + Work = (PPERFECT_HASH_ASYNC_WORK)Overlapped; + if (!Work || !Work->Step) { + return E_POINTER; + } + + Result = Work->Step(Work); + + if (Result == S_FALSE) { + Result = PerfectHashAsyncRequeueWork(Work); + if (FAILED(Result)) { + return PerfectHashAsyncCompleteWork(Work, Result); + } + return S_OK; + } + + return PerfectHashAsyncCompleteWork(Work, Result); +} + +_Use_decl_annotations_ +HRESULT +PerfectHashAsyncInitialize( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + PPERFECT_HASH_CONTEXT Context, + HANDLE IoCompletionPort + ) +/*++ + +Routine Description: + + Initializes an async context for IOCP work submission. + +Arguments: + + AsyncContext - Supplies a pointer to the async context to initialize. + + Context - Supplies a pointer to the owning perfect hash context. + + IoCompletionPort - Supplies the IOCP handle to post work items to. + +Return Value: + + S_OK on success, or an appropriate error code on failure. + +--*/ +{ + if (!ARGUMENT_PRESENT(AsyncContext) || + !ARGUMENT_PRESENT(Context) || + !IoCompletionPort) { + return E_POINTER; + } + + ZeroStructPointerInline(AsyncContext); + AsyncContext->Context = Context; + AsyncContext->Allocator = Context->Allocator; + AsyncContext->IoCompletionPort = IoCompletionPort; + AsyncContext->Outstanding = 0; + AsyncContext->OutstandingEvent = NULL; + + return S_OK; +} + +_Use_decl_annotations_ +VOID +PerfectHashAsyncRundown( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ) +/*++ + +Routine Description: + + Releases resources associated with an async context. + +Arguments: + + AsyncContext - Supplies a pointer to the async context to rundown. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(AsyncContext)) { + return; + } + + if (AsyncContext->OutstandingEvent) { + if (!CloseEvent(AsyncContext->OutstandingEvent)) { + SYS_ERROR(CloseHandle); + } + AsyncContext->OutstandingEvent = NULL; + } + + AsyncContext->Outstanding = 0; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashAsyncSubmit( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + PPERFECT_HASH_ASYNC_WORK Work + ) +/*++ + +Routine Description: + + Submits an async work item to the IOCP dispatch queue. If submission + fails, the work is executed inline. + +Arguments: + + AsyncContext - Supplies a pointer to the async context. + + Work - Supplies a pointer to the async work item. + +Return Value: + + S_OK if the work was queued, otherwise the result of inline execution. + +--*/ +{ + BOOL Success; + LONG Outstanding; + HRESULT Result; + + if (!ARGUMENT_PRESENT(AsyncContext) || + !ARGUMENT_PRESENT(Work) || + !Work->Step) { + return E_POINTER; + } + + if (!AsyncContext->IoCompletionPort) { + return E_UNEXPECTED; + } + + Work->AsyncContext = AsyncContext; + Work->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + Work->Iocp.Flags = PH_IOCP_WORK_FLAG_ASYNC; + Work->Iocp.CompletionCallback = PerfectHashAsyncIocpCompletionCallback; + + if (!AsyncContext->OutstandingEvent) { + AsyncContext->OutstandingEvent = CreateEventW(NULL, TRUE, TRUE, NULL); + if (!AsyncContext->OutstandingEvent) { + SYS_ERROR(CreateEventW_AsyncOutstandingEvent); + goto InlineWork; + } + } + + Outstanding = InterlockedIncrement(&AsyncContext->Outstanding); + if (Outstanding == 1 && AsyncContext->OutstandingEvent) { + ResetEvent(AsyncContext->OutstandingEvent); + } + + Success = PostQueuedCompletionStatus(AsyncContext->IoCompletionPort, + 0, + 0, + &Work->Iocp.Overlapped); + if (!Success) { + SYS_ERROR(PostQueuedCompletionStatus_AsyncSubmit); + goto InlineWork; + } + + return S_OK; + +InlineWork: + + Work->Flags.InlineDispatch = TRUE; + Result = Work->Step(Work); + + if (Result == S_FALSE) { + Result = PerfectHashAsyncRequeueWork(Work); + if (FAILED(Result)) { + return PerfectHashAsyncCompleteWork(Work, Result); + } + return S_OK; + } + + return PerfectHashAsyncCompleteWork(Work, Result); +} + +_Use_decl_annotations_ +VOID +PerfectHashAsyncWait( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ) +/*++ + +Routine Description: + + Waits for all outstanding async work to complete. + +Arguments: + + AsyncContext - Supplies a pointer to the async context. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(AsyncContext)) { + return; + } + + if (AsyncContext->OutstandingEvent) { + WaitForSingleObject(AsyncContext->OutstandingEvent, INFINITE); + } +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashAsync.h b/src/PerfectHash/PerfectHashAsync.h new file mode 100644 index 00000000..47cc62ce --- /dev/null +++ b/src/PerfectHash/PerfectHashAsync.h @@ -0,0 +1,122 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashAsync.h + +Abstract: + + This module defines the asynchronous work framework used by the IOCP + execution model. Work items are scheduled via PostQueuedCompletionStatus + and dispatched through per-item completion callbacks, providing a minimal + continuation-style engine for finer-grained algorithm execution. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef struct _PERFECT_HASH_ASYNC_CONTEXT PERFECT_HASH_ASYNC_CONTEXT; +typedef struct _PERFECT_HASH_ASYNC_WORK PERFECT_HASH_ASYNC_WORK; +typedef PERFECT_HASH_ASYNC_CONTEXT *PPERFECT_HASH_ASYNC_CONTEXT; +typedef PERFECT_HASH_ASYNC_WORK *PPERFECT_HASH_ASYNC_WORK; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_ASYNC_STEP_ROUTINE)( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work + ); +typedef PERFECT_HASH_ASYNC_STEP_ROUTINE *PPERFECT_HASH_ASYNC_STEP_ROUTINE; + +typedef +VOID +(NTAPI PERFECT_HASH_ASYNC_COMPLETE_ROUTINE)( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ); +typedef PERFECT_HASH_ASYNC_COMPLETE_ROUTINE + *PPERFECT_HASH_ASYNC_COMPLETE_ROUTINE; + +typedef union _PERFECT_HASH_ASYNC_WORK_FLAGS { + struct { + ULONG InlineDispatch:1; + ULONG Unused:31; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_ASYNC_WORK_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_ASYNC_WORK_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_ASYNC_WORK_FLAGS *PPERFECT_HASH_ASYNC_WORK_FLAGS; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4820) +#endif + +typedef struct _PERFECT_HASH_ASYNC_CONTEXT { + PPERFECT_HASH_CONTEXT Context; + PALLOCATOR Allocator; + HANDLE IoCompletionPort; + HANDLE OutstandingEvent; + volatile LONG Outstanding; + ULONG Padding; +} PERFECT_HASH_ASYNC_CONTEXT; + +typedef struct _PERFECT_HASH_ASYNC_WORK { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext; + PPERFECT_HASH_ASYNC_STEP_ROUTINE Step; + PPERFECT_HASH_ASYNC_COMPLETE_ROUTINE Complete; + PERFECT_HASH_ASYNC_WORK_FLAGS Flags; + ULONG SliceBudget; + HRESULT LastResult; + ULONG Padding; +} PERFECT_HASH_ASYNC_WORK; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_ASYNC_INITIALIZE)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ HANDLE IoCompletionPort + ); +typedef PERFECT_HASH_ASYNC_INITIALIZE *PPERFECT_HASH_ASYNC_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_ASYNC_RUNDOWN)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ); +typedef PERFECT_HASH_ASYNC_RUNDOWN *PPERFECT_HASH_ASYNC_RUNDOWN; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_ASYNC_SUBMIT)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + _In_ PPERFECT_HASH_ASYNC_WORK Work + ); +typedef PERFECT_HASH_ASYNC_SUBMIT *PPERFECT_HASH_ASYNC_SUBMIT; + +typedef +VOID +(NTAPI PERFECT_HASH_ASYNC_WAIT)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ); +typedef PERFECT_HASH_ASYNC_WAIT *PPERFECT_HASH_ASYNC_WAIT; + +extern PERFECT_HASH_ASYNC_INITIALIZE PerfectHashAsyncInitialize; +extern PERFECT_HASH_ASYNC_RUNDOWN PerfectHashAsyncRundown; +extern PERFECT_HASH_ASYNC_SUBMIT PerfectHashAsyncSubmit; +extern PERFECT_HASH_ASYNC_WAIT PerfectHashAsyncWait; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashClient.c b/src/PerfectHash/PerfectHashClient.c new file mode 100644 index 00000000..8c919ea7 --- /dev/null +++ b/src/PerfectHash/PerfectHashClient.c @@ -0,0 +1,514 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashClient.c + +Abstract: + + This module implements the PERFECT_HASH_CLIENT component. The initial + implementation focuses on wiring the COM interface and reserving slots + for connection and request submission logic. + +--*/ + +#include "stdafx.h" +#include + +#ifdef PH_WINDOWS +static const UNICODE_STRING PerfectHashClientDefaultPipeName = + RTL_CONSTANT_STRING(L"\\\\.\\pipe\\PerfectHashServer"); +#endif + +static +HRESULT +PerfectHashClientWriteAll( + _In_ HANDLE Handle, + _In_reads_bytes_(Length) PVOID Buffer, + _In_ ULONG Length + ) +{ +#ifdef PH_WINDOWS + ULONG Offset; + + Offset = 0; + while (Offset < Length) { + DWORD Written = 0; + BOOL Success; + ULONG Remaining = Length - Offset; + + Success = WriteFile(Handle, + (PUCHAR)Buffer + Offset, + Remaining, + &Written, + NULL); + if (!Success) { + return HRESULT_FROM_WIN32(GetLastError()); + } + + if (Written == 0) { + return HRESULT_FROM_WIN32(ERROR_BROKEN_PIPE); + } + + Offset += Written; + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(Handle); + UNREFERENCED_PARAMETER(Buffer); + UNREFERENCED_PARAMETER(Length); + return E_NOTIMPL; +#endif +} + +static +HRESULT +PerfectHashClientReadAll( + _In_ HANDLE Handle, + _Out_writes_bytes_(Length) PVOID Buffer, + _In_ ULONG Length + ) +{ +#ifdef PH_WINDOWS + ULONG Offset; + + Offset = 0; + while (Offset < Length) { + DWORD Read = 0; + BOOL Success; + ULONG Remaining = Length - Offset; + + Success = ReadFile(Handle, + (PUCHAR)Buffer + Offset, + Remaining, + &Read, + NULL); + if (!Success) { + return HRESULT_FROM_WIN32(GetLastError()); + } + + if (Read == 0) { + return HRESULT_FROM_WIN32(ERROR_BROKEN_PIPE); + } + + Offset += Read; + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(Handle); + UNREFERENCED_PARAMETER(Buffer); + UNREFERENCED_PARAMETER(Length); + return E_NOTIMPL; +#endif +} + +PERFECT_HASH_CLIENT_INITIALIZE PerfectHashClientInitialize; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientInitialize( + PPERFECT_HASH_CLIENT Client + ) +/*++ + +Routine Description: + + Initializes a PERFECT_HASH_CLIENT instance. + +Arguments: + + Client - Supplies a pointer to a PERFECT_HASH_CLIENT structure for which + initialization is to be performed. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ + HRESULT Result; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + Result = Client->Vtbl->CreateInstance(Client, + NULL, + &IID_PERFECT_HASH_RTL, + &Client->Rtl); + if (FAILED(Result)) { + return Result; + } + + Result = Client->Vtbl->CreateInstance(Client, + NULL, + &IID_PERFECT_HASH_ALLOCATOR, + &Client->Allocator); + if (FAILED(Result)) { + return Result; + } + + Client->ResponsePayload.Buffer = NULL; + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + Client->State.Initialized = TRUE; + + return S_OK; +} + +PERFECT_HASH_CLIENT_RUNDOWN PerfectHashClientRundown; + +_Use_decl_annotations_ +VOID +PerfectHashClientRundown( + PPERFECT_HASH_CLIENT Client + ) +/*++ + +Routine Description: + + Releases resources associated with a PERFECT_HASH_CLIENT instance. + +Arguments: + + Client - Supplies a pointer to a PERFECT_HASH_CLIENT structure for which + rundown is to be performed. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(Client)) { + return; + } + +#ifdef PH_WINDOWS + if (Client->ConnectionHandle) { + CloseHandle(Client->ConnectionHandle); + Client->ConnectionHandle = NULL; + } +#endif + + if (Client->ResponsePayload.Buffer && Client->Allocator) { + Client->Allocator->Vtbl->FreePointer( + Client->Allocator, + (PVOID *)&Client->ResponsePayload.Buffer + ); + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + } + + RELEASE(Client->Allocator); + RELEASE(Client->Rtl); +} + +PERFECT_HASH_CLIENT_CONNECT PerfectHashClientConnect; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientConnect( + PPERFECT_HASH_CLIENT Client, + PCUNICODE_STRING Endpoint + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Client); + UNREFERENCED_PARAMETER(Endpoint); + return E_NOTIMPL; +#else + BOOL Success; + HANDLE PipeHandle; + DWORD Mode; + DWORD LastError; + PCUNICODE_STRING TargetEndpoint; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (Client->ConnectionHandle) { + CloseHandle(Client->ConnectionHandle); + Client->ConnectionHandle = NULL; + } + + TargetEndpoint = Endpoint ? Endpoint : &PerfectHashClientDefaultPipeName; + + while (TRUE) { + PipeHandle = CreateFileW(TargetEndpoint->Buffer, + GENERIC_READ | GENERIC_WRITE, + 0, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (PipeHandle != INVALID_HANDLE_VALUE) { + break; + } + + LastError = GetLastError(); + if (LastError != ERROR_PIPE_BUSY) { + return HRESULT_FROM_WIN32(LastError); + } + + Success = WaitNamedPipeW(TargetEndpoint->Buffer, + NMPWAIT_WAIT_FOREVER); + if (!Success) { + return HRESULT_FROM_WIN32(GetLastError()); + } + } + + Mode = PIPE_READMODE_BYTE; + Success = SetNamedPipeHandleState(PipeHandle, + &Mode, + NULL, + NULL); + if (!Success) { + CloseHandle(PipeHandle); + return HRESULT_FROM_WIN32(GetLastError()); + } + + Client->ConnectionHandle = PipeHandle; + Client->Endpoint = *TargetEndpoint; + Client->State.Connected = TRUE; + Client->State.Disconnected = FALSE; + + return S_OK; +#endif +} + +PERFECT_HASH_CLIENT_DISCONNECT PerfectHashClientDisconnect; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientDisconnect( + PPERFECT_HASH_CLIENT Client + ) +{ + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + +#ifdef PH_WINDOWS + if (Client->ConnectionHandle) { + CloseHandle(Client->ConnectionHandle); + Client->ConnectionHandle = NULL; + } +#endif + + if (Client->ResponsePayload.Buffer && Client->Allocator) { + Client->Allocator->Vtbl->FreePointer( + Client->Allocator, + (PVOID *)&Client->ResponsePayload.Buffer + ); + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + } + + Client->State.Disconnected = TRUE; + + return S_OK; +} + +PERFECT_HASH_CLIENT_SUBMIT_REQUEST PerfectHashClientSubmitRequest; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientSubmitRequest( + PPERFECT_HASH_CLIENT Client, + PPERFECT_HASH_SERVER_REQUEST Request + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Client); + UNREFERENCED_PARAMETER(Request); + return E_NOTIMPL; +#else + HRESULT Result; + ULONG PayloadLength; + HANDLE Handle; + WCHAR Terminator; + PRTL Rtl; + PALLOCATOR Allocator; + PERFECT_HASH_SERVER_REQUEST_HEADER RequestHeader; + PERFECT_HASH_SERVER_RESPONSE_HEADER ResponseHeader; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Request)) { + return E_POINTER; + } + + if (!IsValidPerfectHashServerRequestType(Request->RequestType)) { + return E_INVALIDARG; + } + + Handle = Client->ConnectionHandle; + if (!Handle) { + return E_UNEXPECTED; + } + + Rtl = Client->Rtl; + if (!Rtl) { + return E_UNEXPECTED; + } + + Allocator = Client->Allocator; + if (!Allocator) { + return E_UNEXPECTED; + } + + if (Client->ResponsePayload.Buffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Client->ResponsePayload.Buffer + ); + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + } + + PayloadLength = 0; + if (Request->CommandLine.Length > 0) { + if (!Request->CommandLine.Buffer) { + return E_INVALIDARG; + } + if (Request->CommandLine.Length > + (PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE - sizeof(WCHAR))) { + return E_INVALIDARG; + } + PayloadLength = Request->CommandLine.Length + sizeof(WCHAR); + } + + ZeroMemory(&RequestHeader, sizeof(RequestHeader)); + RequestHeader.SizeOfStruct = sizeof(RequestHeader); + RequestHeader.Version = PERFECT_HASH_SERVER_MESSAGE_VERSION; + RequestHeader.RequestType = Request->RequestType; + RequestHeader.RequestId = Request->RequestId; + RequestHeader.PayloadLength = PayloadLength; + + Result = PerfectHashClientWriteAll(Handle, + &RequestHeader, + sizeof(RequestHeader)); + if (FAILED(Result)) { + return Result; + } + + if (PayloadLength > 0) { + Result = PerfectHashClientWriteAll(Handle, + Request->CommandLine.Buffer, + Request->CommandLine.Length); + if (FAILED(Result)) { + return Result; + } + + Terminator = L'\0'; + Result = PerfectHashClientWriteAll(Handle, + &Terminator, + sizeof(Terminator)); + if (FAILED(Result)) { + return Result; + } + } + + Result = PerfectHashClientReadAll(Handle, + &ResponseHeader, + sizeof(ResponseHeader)); + if (FAILED(Result)) { + return Result; + } + + if (ResponseHeader.SizeOfStruct != sizeof(ResponseHeader) || + ResponseHeader.Version != PERFECT_HASH_SERVER_MESSAGE_VERSION) { + return E_UNEXPECTED; + } + + Client->ResponseFlags = ResponseHeader.Flags; + + if (ResponseHeader.PayloadLength > 0) { + PVOID PayloadBuffer; + + PayloadLength = ResponseHeader.PayloadLength; + + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + return E_INVALIDARG; + } + + if (PayloadLength > (ULONG)USHRT_MAX) { + return E_INVALIDARG; + } + + PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + PayloadLength); + if (!PayloadBuffer) { + return E_OUTOFMEMORY; + } + + Result = PerfectHashClientReadAll(Handle, + PayloadBuffer, + PayloadLength); + if (FAILED(Result)) { + Allocator->Vtbl->FreePointer(Allocator, &PayloadBuffer); + return Result; + } + + Client->ResponsePayload.Buffer = (PWSTR)PayloadBuffer; + Client->ResponsePayload.Length = (USHORT)( + PayloadLength >= sizeof(WCHAR) ? + PayloadLength - sizeof(WCHAR) : + 0 + ); + Client->ResponsePayload.MaximumLength = (USHORT)PayloadLength; + Client->ResponsePayloadBufferSize = PayloadLength; + } + + return ResponseHeader.Result; +#endif +} + +PERFECT_HASH_CLIENT_GET_LAST_RESPONSE PerfectHashClientGetLastResponse; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientGetLastResponse( + PPERFECT_HASH_CLIENT Client, + PUNICODE_STRING ResponsePayload, + PULONG ResponseFlags + ) +{ + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ResponsePayload)) { + return E_POINTER; + } + + ResponsePayload->Buffer = Client->ResponsePayload.Buffer; + ResponsePayload->Length = Client->ResponsePayload.Length; + ResponsePayload->MaximumLength = Client->ResponsePayload.MaximumLength; + + if (ARGUMENT_PRESENT(ResponseFlags)) { + *ResponseFlags = Client->ResponseFlags; + } + + return S_OK; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashClient.h b/src/PerfectHash/PerfectHashClient.h new file mode 100644 index 00000000..958a7067 --- /dev/null +++ b/src/PerfectHash/PerfectHashClient.h @@ -0,0 +1,89 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashClient.h + +Abstract: + + This is the private header file for the PERFECT_HASH_CLIENT component of + the perfect hash library. It defines the structure, and function pointer + typedefs for private non-vtbl members. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef union _PERFECT_HASH_CLIENT_STATE { + struct { + + ULONG Initialized:1; + ULONG Connected:1; + ULONG Disconnected:1; + + ULONG Unused:29; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CLIENT_STATE; +C_ASSERT(sizeof(PERFECT_HASH_CLIENT_STATE) == sizeof(ULONG)); +typedef PERFECT_HASH_CLIENT_STATE *PPERFECT_HASH_CLIENT_STATE; + +typedef union _PERFECT_HASH_CLIENT_FLAGS { + struct { + + ULONG Unused:32; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CLIENT_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_CLIENT_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_CLIENT_FLAGS *PPERFECT_HASH_CLIENT_FLAGS; + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CLIENT { + + COMMON_COMPONENT_HEADER(PERFECT_HASH_CLIENT); + + HANDLE ConnectionHandle; + UNICODE_STRING Endpoint; + UNICODE_STRING ResponsePayload; + ULONG ResponsePayloadBufferSize; + ULONG ResponseFlags; + + // + // Backing vtbl. + // + + PERFECT_HASH_CLIENT_VTBL Interface; + +} PERFECT_HASH_CLIENT; +typedef PERFECT_HASH_CLIENT *PPERFECT_HASH_CLIENT; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CLIENT_INITIALIZE)( + _In_ PPERFECT_HASH_CLIENT Client + ); +typedef PERFECT_HASH_CLIENT_INITIALIZE *PPERFECT_HASH_CLIENT_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_CLIENT_RUNDOWN)( + _In_ _Post_ptr_invalid_ PPERFECT_HASH_CLIENT Client + ); +typedef PERFECT_HASH_CLIENT_RUNDOWN *PPERFECT_HASH_CLIENT_RUNDOWN; + +extern PERFECT_HASH_CLIENT_INITIALIZE PerfectHashClientInitialize; +extern PERFECT_HASH_CLIENT_RUNDOWN PerfectHashClientRundown; + +extern PERFECT_HASH_CLIENT_CONNECT PerfectHashClientConnect; +extern PERFECT_HASH_CLIENT_DISCONNECT PerfectHashClientDisconnect; +extern PERFECT_HASH_CLIENT_SUBMIT_REQUEST PerfectHashClientSubmitRequest; +extern PERFECT_HASH_CLIENT_GET_LAST_RESPONSE PerfectHashClientGetLastResponse; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashConstants.c b/src/PerfectHash/PerfectHashConstants.c index 97f2169a..b180e94d 100644 --- a/src/PerfectHash/PerfectHashConstants.c +++ b/src/PerfectHash/PerfectHashConstants.c @@ -900,7 +900,7 @@ const ULONG IndexMaskPlaceholder = 0xbbbbbbbb; // -#define NUMBER_OF_INTERFACES 15 +#define NUMBER_OF_INTERFACES 18 #define EXPECTED_ARRAY_SIZE NUMBER_OF_INTERFACES+2 #define VERIFY_ARRAY_SIZE(Name) C_ASSERT(ARRAYSIZE(Name) == EXPECTED_ARRAY_SIZE) @@ -988,6 +988,9 @@ const SHORT ComponentInterfaceTlsContextOffsets[] = { -1, // IClassFactory (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Keys), (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Context), + -1, // ContextIocp + -1, // Server + -1, // Client (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Table), (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Rtl), (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Allocator), @@ -1011,6 +1014,9 @@ const SHORT GlobalComponentsInterfaceOffsets[] = { -1, // IClassFactory -1, // Keys -1, // Context + -1, // ContextIocp + -1, // Server + -1, // Client -1, // Table (SHORT)FIELD_OFFSET(GLOBAL_COMPONENTS, Rtl), (SHORT)FIELD_OFFSET(GLOBAL_COMPONENTS, Allocator), @@ -1118,6 +1124,90 @@ const PERFECT_HASH_CONTEXT_VTBL PerfectHashContextInterface = { }; VERIFY_VTBL_SIZE(PERFECT_HASH_CONTEXT, 12); +// +// PerfectHashContextIocp +// + +const PERFECT_HASH_CONTEXT_IOCP_VTBL PerfectHashContextIocpInterface = { + (PPERFECT_HASH_CONTEXT_IOCP_QUERY_INTERFACE)&ComponentQueryInterface, + (PPERFECT_HASH_CONTEXT_IOCP_ADD_REF)&ComponentAddRef, + (PPERFECT_HASH_CONTEXT_IOCP_RELEASE)&ComponentRelease, + (PPERFECT_HASH_CONTEXT_IOCP_CREATE_INSTANCE)&ComponentCreateInstance, + (PPERFECT_HASH_CONTEXT_IOCP_LOCK_SERVER)&ComponentLockServer, + &PerfectHashContextIocpSetMaximumConcurrency, + &PerfectHashContextIocpGetMaximumConcurrency, + &PerfectHashContextIocpSetMaximumThreads, + &PerfectHashContextIocpGetMaximumThreads, + &PerfectHashContextIocpSetNumaNodeMask, + &PerfectHashContextIocpGetNumaNodeMask, + &PerfectHashContextIocpSetBaseOutputDirectory, + &PerfectHashContextIocpGetBaseOutputDirectory, + &PerfectHashContextIocpBulkCreate, + &PerfectHashContextIocpBulkCreateArgvW, + &PerfectHashContextIocpExtractBulkCreateArgsFromArgvW, + &PerfectHashContextIocpTableCreate, + &PerfectHashContextIocpTableCreateArgvW, + &PerfectHashContextIocpExtractTableCreateArgsFromArgvW, +#ifdef PH_COMPAT + &PerfectHashContextIocpTableCreateArgvA, + &PerfectHashContextIocpBulkCreateArgvA, +#else + NULL, + NULL, +#endif +}; +VERIFY_VTBL_SIZE(PERFECT_HASH_CONTEXT_IOCP, 16); + +// +// PerfectHashServer +// + +const PERFECT_HASH_SERVER_VTBL PerfectHashServerInterface = { + (PPERFECT_HASH_SERVER_QUERY_INTERFACE)&ComponentQueryInterface, + (PPERFECT_HASH_SERVER_ADD_REF)&ComponentAddRef, + (PPERFECT_HASH_SERVER_RELEASE)&ComponentRelease, + (PPERFECT_HASH_SERVER_CREATE_INSTANCE)&ComponentCreateInstance, + (PPERFECT_HASH_SERVER_LOCK_SERVER)&ComponentLockServer, + &PerfectHashServerSetMaximumConcurrency, + &PerfectHashServerGetMaximumConcurrency, + &PerfectHashServerSetMaximumThreads, + &PerfectHashServerGetMaximumThreads, + &PerfectHashServerSetNumaNodeMask, + &PerfectHashServerGetNumaNodeMask, + &PerfectHashServerSetEndpoint, + &PerfectHashServerGetEndpoint, + &PerfectHashServerSetLocalOnly, + &PerfectHashServerGetLocalOnly, + &PerfectHashServerSetVerbose, + &PerfectHashServerGetVerbose, + &PerfectHashServerSetNoFileIo, + &PerfectHashServerGetNoFileIo, + &PerfectHashServerSetIocpBufferGuardPages, + &PerfectHashServerGetIocpBufferGuardPages, + &PerfectHashServerStart, + &PerfectHashServerStop, + &PerfectHashServerWait, + &PerfectHashServerSubmitRequest, +}; +VERIFY_VTBL_SIZE(PERFECT_HASH_SERVER, 20); + +// +// PerfectHashClient +// + +const PERFECT_HASH_CLIENT_VTBL PerfectHashClientInterface = { + (PPERFECT_HASH_CLIENT_QUERY_INTERFACE)&ComponentQueryInterface, + (PPERFECT_HASH_CLIENT_ADD_REF)&ComponentAddRef, + (PPERFECT_HASH_CLIENT_RELEASE)&ComponentRelease, + (PPERFECT_HASH_CLIENT_CREATE_INSTANCE)&ComponentCreateInstance, + (PPERFECT_HASH_CLIENT_LOCK_SERVER)&ComponentLockServer, + &PerfectHashClientConnect, + &PerfectHashClientDisconnect, + &PerfectHashClientSubmitRequest, + &PerfectHashClientGetLastResponse, +}; +VERIFY_VTBL_SIZE(PERFECT_HASH_CLIENT, 4); + // // PerfectHashTable // @@ -1472,6 +1562,9 @@ const VOID *ComponentInterfaces[] = { &IClassFactoryInterface, &PerfectHashKeysInterface, &PerfectHashContextInterface, + &PerfectHashContextIocpInterface, + &PerfectHashServerInterface, + &PerfectHashClientInterface, &PerfectHashTableInterface, &RtlInterface, &AllocatorInterface, @@ -1500,6 +1593,9 @@ const PCOMPONENT_INITIALIZE ComponentInitializeRoutines[] = { (PCOMPONENT_INITIALIZE)&PerfectHashKeysInitialize, (PCOMPONENT_INITIALIZE)&PerfectHashContextInitialize, + (PCOMPONENT_INITIALIZE)&PerfectHashContextIocpInitialize, + (PCOMPONENT_INITIALIZE)&PerfectHashServerInitialize, + (PCOMPONENT_INITIALIZE)&PerfectHashClientInitialize, (PCOMPONENT_INITIALIZE)&PerfectHashTableInitialize, (PCOMPONENT_INITIALIZE)&RtlInitialize, (PCOMPONENT_INITIALIZE)&AllocatorInitialize, @@ -1529,6 +1625,9 @@ const PCOMPONENT_RUNDOWN ComponentRundownRoutines[] = { (PCOMPONENT_RUNDOWN)&PerfectHashKeysRundown, (PCOMPONENT_RUNDOWN)&PerfectHashContextRundown, + (PCOMPONENT_RUNDOWN)&PerfectHashContextIocpRundown, + (PCOMPONENT_RUNDOWN)&PerfectHashServerRundown, + (PCOMPONENT_RUNDOWN)&PerfectHashClientRundown, (PCOMPONENT_RUNDOWN)&PerfectHashTableRundown, (PCOMPONENT_RUNDOWN)&RtlRundown, (PCOMPONENT_RUNDOWN)&AllocatorRundown, diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index 165fe470..ff5f9bbb 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -21,6 +21,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" #include "bsthreadpool.h" // @@ -124,6 +125,110 @@ Return Value: return MaximumConcurrency + 2; } +static +VOID +PerfectHashIocpFreeBuffer( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + BOOL Success; + HRESULT Result; + HANDLE ProcessHandle; + PVOID BaseAddress; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + ProcessHandle = NULL; + if (Buffer->OwnerPool) { + ProcessHandle = Buffer->OwnerPool->ProcessHandle; + } + + if (!ProcessHandle) { + ProcessHandle = GetCurrentProcess(); + } + + if (Buffer->Flags & PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES) { + BaseAddress = Buffer; + Result = Rtl->Vtbl->DestroyBuffer(Rtl, + ProcessHandle, + &BaseAddress, + Buffer->AllocationSize); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpFreeBuffer_DestroyBuffer, Result); + } + return; + } + + Success = VirtualFreeEx(ProcessHandle, Buffer, 0, MEM_RELEASE); + if (!Success) { + SYS_ERROR(VirtualFreeEx); + } +} + +static +VOID +PerfectHashIocpRundownBufferList( + _In_ PRTL Rtl, + _In_ PGUARDED_LIST BufferList + ) +{ + BOOLEAN NotEmpty; + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(BufferList)) { + return; + } + + while (TRUE) { + Entry = NULL; + NotEmpty = BufferList->Vtbl->RemoveHeadEx(BufferList, &Entry); + if (!NotEmpty) { + break; + } + + Buffer = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER, + ListEntry); + + PerfectHashIocpFreeBuffer(Rtl, Buffer); + } + + BufferList->Vtbl->Reset(BufferList); +} + +static +VOID +PerfectHashIocpRundownOversizePools( + _In_ PALLOCATOR Allocator, + _Inout_ PLIST_ENTRY ListHead, + _Inout_opt_ PULONG PoolCount + ) +{ + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + if (!ARGUMENT_PRESENT(Allocator) || !ARGUMENT_PRESENT(ListHead)) { + return; + } + + while (!IsListEmpty(ListHead)) { + Entry = RemoveHeadList(ListHead); + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + } + + InitializeListHead(ListHead); + if (PoolCount) { + *PoolCount = 0; + } +} + // // Main context creation routine. @@ -154,12 +259,13 @@ Return Value: --*/ { + HRESULT Result = S_OK; PRTL Rtl; + PALLOCATOR Allocator; PACL Acl = NULL; BYTE Index; BYTE NumberOfEvents; BOOL Success; - HRESULT Result = S_OK; ULONG LastError; PHANDLE Event; PCHAR Buffer; @@ -172,7 +278,6 @@ Return Value: PSTRING ComputerName; DWORD ComputerNameLength; PTP_POOL Threadpool; - PALLOCATOR Allocator; ULARGE_INTEGER AllocSize; ULONG ThreadpoolConcurrency; ULARGE_INTEGER ObjectNameArraySize; @@ -248,6 +353,18 @@ Return Value: goto Error; } + Result = Context->Vtbl->CreateInstance(Context, + NULL, + &IID_PERFECT_HASH_GUARDED_LIST, + &Context->FileWorkBufferList); + + if (FAILED(Result)) { + goto Error; + } + + InitializeListHead(&Context->FileWorkOversizePools); + Context->FileWorkOversizePoolCount = 0; + // // Initialize aliases. // @@ -339,6 +456,21 @@ Return Value: Context->State.AsULong = 0; Context->Flags.AsULong = 0; + // + // Honor the TLS flag that requests IOCP-native contexts without + // threadpool initialization. + // + + { + PPERFECT_HASH_TLS_CONTEXT TlsContext; + + TlsContext = PerfectHashTlsGetContext(); + if (TlsContext && + TlsContext->Flags.CreateContextWithoutThreadpool) { + Context->Flags.SkipThreadpoolInitialization = TRUE; + } + } + InitializeSRWLock(&Context->Lock); if (!InitializeCriticalSectionAndSpinCount( @@ -473,9 +605,14 @@ Return Value: Context->MinimumConcurrency = MaximumConcurrency; Context->MaximumConcurrency = MaximumConcurrency; + Context->InitialPerFileConcurrency = 1; + Context->MaxPerFileConcurrency = MaximumConcurrency; + Context->IncreaseConcurrencyAfterMilliseconds = 500; ThreadpoolConcurrency = GetMainThreadpoolConcurrency(MaximumConcurrency); + if (!Context->Flags.SkipThreadpoolInitialization) { + #ifdef PH_WINDOWS // @@ -492,9 +629,13 @@ Return Value: } if (!SetThreadpoolThreadMinimum(Threadpool, ThreadpoolConcurrency)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Threadpool, ThreadpoolConcurrency); @@ -593,9 +734,13 @@ Return Value: SetThreadpoolThreadMaximum(Threadpool, MaximumConcurrency); if (!SetThreadpoolThreadMinimum(Threadpool, MaximumConcurrency)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Threadpool, MaximumConcurrency); @@ -656,9 +801,13 @@ Return Value: Context->FinishedThreadpool); if (!SetThreadpoolThreadMinimum(Context->FinishedThreadpool, 1)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Context->FinishedThreadpool, 1); @@ -689,9 +838,13 @@ Return Value: if (!SetThreadpoolThreadMinimum(Context->ErrorThreadpool, 1)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Context->ErrorThreadpool, 1); @@ -723,6 +876,8 @@ Return Value: #endif + } + // // Initialize the timestamp string. // @@ -853,12 +1008,12 @@ Return Value: --*/ { + HRESULT Result; PRTL Rtl; + PALLOCATOR Allocator; BYTE Index; BYTE NumberOfEvents; - PALLOCATOR Allocator; PHANDLE Event; - HRESULT Result; // // Validate arguments. @@ -870,6 +1025,7 @@ Return Value: Rtl = Context->Rtl; Allocator = Context->Allocator; + Allocator = Context->Allocator; if (!Rtl || !Allocator) { return; @@ -951,6 +1107,13 @@ Return Value: #ifdef PH_WINDOWS + if (Context->FileWorkOutstandingEvent) { + if (!CloseEvent(Context->FileWorkOutstandingEvent)) { + SYS_ERROR(CloseHandle); + } + Context->FileWorkOutstandingEvent = NULL; + } + if (Context->MainCleanupGroup) { CloseThreadpoolCleanupGroupMembers(Context->MainCleanupGroup, @@ -1016,6 +1179,27 @@ Return Value: #endif + if (Context->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, Context->FileWorkBufferList); + } + + if (Allocator) { + if (Context->FileWorkBufferPools) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Context->FileWorkBufferPools + ); + } + + PerfectHashIocpRundownOversizePools( + Allocator, + &Context->FileWorkOversizePools, + &Context->FileWorkOversizePoolCount + ); + } + + Context->FileWorkBufferPoolCount = 0; + if (Context->ObjectNames) { Allocator->Vtbl->FreePointer(Allocator, &Context->ObjectNames); @@ -1039,6 +1223,7 @@ Return Value: RELEASE(Context->MainWorkList); RELEASE(Context->FileWorkList); + RELEASE(Context->FileWorkBufferList); RELEASE(Context->FinishedWorkList); RELEASE(Context->BulkCreateCsvFile); RELEASE(Context->BaseOutputDirectory); @@ -1079,12 +1264,14 @@ Return Value: --*/ { PRTL Rtl; + PALLOCATOR Allocator; if (!ARGUMENT_PRESENT(Context)) { return E_POINTER; } Rtl = Context->Rtl; + Allocator = Context->Allocator; Context->AlgorithmId = PerfectHashNullAlgorithmId; Context->HashFunctionId = PerfectHashNullHashFunctionId; @@ -1150,6 +1337,34 @@ Return Value: Context->FileWorkList->Vtbl->Reset(Context->FileWorkList); Context->FinishedWorkList->Vtbl->Reset(Context->FinishedWorkList); +#ifdef PH_WINDOWS + Context->FileWorkOutstanding = 0; + if (Context->FileWorkOutstandingEvent) { + SetEvent(Context->FileWorkOutstandingEvent); + } +#endif + + if (Context->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, Context->FileWorkBufferList); + } + + if (Allocator) { + if (Context->FileWorkBufferPools) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Context->FileWorkBufferPools + ); + } + + PerfectHashIocpRundownOversizePools( + Allocator, + &Context->FileWorkOversizePools, + &Context->FileWorkOversizePoolCount + ); + } + + Context->FileWorkBufferPoolCount = 0; + Context->KeysSubset = NULL; Context->UserSeeds = NULL; Context->SeedMasks = NULL; @@ -1421,6 +1636,252 @@ Return Value: return; } +/*++ + +Structure Description: + + This structure wraps an IOCP work header and the target context pointer + for IOCP-dispatched file work callbacks. + +--*/ +typedef struct _PERFECT_HASH_FILE_WORK_IOCP { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_CONTEXT Context; +} PERFECT_HASH_FILE_WORK_IOCP; +typedef PERFECT_HASH_FILE_WORK_IOCP *PPERFECT_HASH_FILE_WORK_IOCP; + +static +HRESULT +PerfectHashContextFileWorkIocpCompletionCallback( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG_PTR CompletionKey, + LPOVERLAPPED Overlapped, + DWORD NumberOfBytesTransferred, + BOOL Success + ) +/*++ + +Routine Description: + + Completion callback for IOCP-dispatched file work. This routine removes + one file work item from the context list, invokes the registered file work + callback, decrements the outstanding count, and signals the outstanding + event when all work items have completed. + +Arguments: + + ContextIocp - Supplies the owning IOCP context. + + CompletionKey - Supplies the IOCP completion key. + + Overlapped - Supplies a pointer to the IOCP work header for this request. + + NumberOfBytesTransferred - Supplies the number of bytes transferred. + + Success - Supplies the success status of the IOCP completion. + +Return Value: + + S_OK on success, or E_POINTER for invalid arguments. + +--*/ +{ + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_FILE_WORK_IOCP WorkItem; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(CompletionKey); + UNREFERENCED_PARAMETER(NumberOfBytesTransferred); + UNREFERENCED_PARAMETER(Success); + + WorkItem = (PPERFECT_HASH_FILE_WORK_IOCP)Overlapped; + if (!WorkItem) { + return E_POINTER; + } + + Context = WorkItem->Context; + if (!Context) { + return E_POINTER; + } + + FileWorkCallback(NULL, Context, NULL); + + if (InterlockedDecrement(&Context->FileWorkOutstanding) == 0) { + if (Context->FileWorkOutstandingEvent) { + SetEvent(Context->FileWorkOutstandingEvent); + } + } + + if (Context->Allocator) { + Context->Allocator->Vtbl->FreePointer(Context->Allocator, + (PVOID *)&WorkItem); + } + + return S_OK; +} + +PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK PerfectHashContextSubmitFileWork; + +_Use_decl_annotations_ +VOID +PerfectHashContextSubmitFileWork( + PPERFECT_HASH_CONTEXT Context + ) +/*++ + +Routine Description: + + Submits a file work item. If an IOCP port has been configured, this + routine posts an IOCP completion and tracks outstanding file work so that + callers can wait on a single event. Otherwise, the file work is submitted + to the file threadpool. + +Arguments: + + Context - Supplies a pointer to the PERFECT_HASH_CONTEXT. + +Return Value: + + None. + +--*/ +{ + BOOL Success; + BOOLEAN Incremented = FALSE; + LONG Outstanding; + PALLOCATOR Allocator; + PPERFECT_HASH_FILE_WORK_IOCP WorkItem = NULL; + + if (!ARGUMENT_PRESENT(Context)) { + return; + } + +#ifdef PH_WINDOWS + if (Context->FileWorkIoCompletionPort) { + + if (!Context->FileWorkOutstandingEvent) { + Context->FileWorkOutstandingEvent = + CreateEventW(NULL, TRUE, TRUE, NULL); + if (!Context->FileWorkOutstandingEvent) { + SYS_ERROR(CreateEventW_FileWorkOutstandingEvent); + goto InlineWork; + } + } + + Allocator = Context->Allocator; + if (!Allocator) { + goto InlineWork; + } + + WorkItem = (PPERFECT_HASH_FILE_WORK_IOCP)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*WorkItem) + ); + if (!WorkItem) { + goto InlineWork; + } + + WorkItem->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + WorkItem->Iocp.Flags = PH_IOCP_WORK_FLAG_FILE_WORK; + WorkItem->Iocp.CompletionCallback = + PerfectHashContextFileWorkIocpCompletionCallback; + WorkItem->Context = Context; + + Outstanding = InterlockedIncrement(&Context->FileWorkOutstanding); + Incremented = TRUE; + if (Outstanding == 1 && Context->FileWorkOutstandingEvent) { + ResetEvent(Context->FileWorkOutstandingEvent); + } + + Success = PostQueuedCompletionStatus( + Context->FileWorkIoCompletionPort, + 0, + 0, + &WorkItem->Iocp.Overlapped + ); + + if (!Success) { + SYS_ERROR(PostQueuedCompletionStatus_FileWork); + goto InlineWork; + } + + return; + } +#endif + + SubmitThreadpoolWork(Context->FileWork); + return; + +InlineWork: + + // + // Fall back to synchronous execution if IOCP submission fails. + // + + FileWorkCallback(NULL, Context, NULL); + + if (Context->FileWorkIoCompletionPort && Incremented) { + if (InterlockedDecrement(&Context->FileWorkOutstanding) == 0) { + if (Context->FileWorkOutstandingEvent) { + SetEvent(Context->FileWorkOutstandingEvent); + } + } + } + + Allocator = Context->Allocator; + if (Allocator && WorkItem) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + } + + return; +} + +PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS + PerfectHashContextWaitForFileWorkCallbacks; + +_Use_decl_annotations_ +VOID +PerfectHashContextWaitForFileWorkCallbacks( + PPERFECT_HASH_CONTEXT Context, + BOOL CancelPending + ) +/*++ + +Routine Description: + + Waits for all file work callbacks to complete. When IOCP dispatch is in + effect, this waits on the outstanding file work event; otherwise, it + waits on the file threadpool work callbacks. + +Arguments: + + Context - Supplies a pointer to the PERFECT_HASH_CONTEXT. + + CancelPending - Supplies the cancellation behavior for threadpool work. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(Context)) { + return; + } + +#ifdef PH_WINDOWS + if (Context->FileWorkIoCompletionPort) { + if (Context->FileWorkOutstandingEvent) { + WaitForSingleObject(Context->FileWorkOutstandingEvent, INFINITE); + } + return; + } +#endif + + WaitForThreadpoolWorkCallbacks(Context->FileWork, CancelPending); +} + _Use_decl_annotations_ VOID FinishedWorkCallback( @@ -1688,6 +2149,7 @@ Return Value: PRTL Rtl; PTP_POOL Threadpool; HRESULT Result = S_OK; + ULONG LastError; ULONG ThreadpoolConcurrency; if (!ARGUMENT_PRESENT(Context)) { @@ -1705,6 +2167,11 @@ Return Value: #ifdef PH_WINDOWS if (!Context->MainThreadpool) { + if (Context->Flags.SkipThreadpoolInitialization) { + ReleasePerfectHashContextLockExclusive(Context); + return S_OK; + } + ReleasePerfectHashContextLockExclusive(Context); return E_UNEXPECTED; } @@ -1714,8 +2181,12 @@ Return Value: SetThreadpoolThreadMaximum(Threadpool, ThreadpoolConcurrency); if (!SetThreadpoolThreadMinimum(Threadpool, ThreadpoolConcurrency)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SET_MAXIMUM_CONCURRENCY_FAILED; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SET_MAXIMUM_CONCURRENCY_FAILED; + } } #endif diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index e264525f..647819d2 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -21,6 +21,11 @@ Module Name: #include "bsthreadpool.h" #endif +typedef struct _PERFECT_HASH_IOCP_BUFFER_POOL PERFECT_HASH_IOCP_BUFFER_POOL; +typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; +struct _PERFECT_HASH_IOCP_NODE; +typedef struct _PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; + // // Algorithms are required to register a callback routine with the perfect hash // table context that matches the following signature. This routine will be @@ -126,6 +131,12 @@ typedef union _PERFECT_HASH_CONTEXT_STATE { ULONG IsBulkCreate:1; + // + // When set, indicates context-level file work should be skipped. + // + + ULONG SkipContextFileWork:1; + // // When set, indicates the graph solving failed because every worker // thread failed to allocate sufficient memory to even attempt solving @@ -200,7 +211,7 @@ typedef union _PERFECT_HASH_CONTEXT_STATE { // Unused bits. // - ULONG Unused:16; + ULONG Unused:15; }; LONG AsLong; ULONG AsULong; @@ -245,6 +256,13 @@ typedef PERFECT_HASH_CONTEXT_STATE *PPERFECT_HASH_CONTEXT_STATE; #define SetContextBulkCreate(Context) ((Context)->State.IsBulkCreate = TRUE) #define ClearContextBulkCreate(Context) ((Context)->State.IsBulkCreate = FALSE) +#define SkipContextFileWork(Context) \ + ((Context)->State.SkipContextFileWork == TRUE) +#define SetContextSkipContextFileWork(Context) \ + ((Context)->State.SkipContextFileWork = TRUE) +#define ClearContextSkipContextFileWork(Context) \ + ((Context)->State.SkipContextFileWork = FALSE) + #define IsContextTableCreate(Context) ((Context)->State.IsTableCreate == TRUE) #define SetContextTableCreate(Context) ((Context)->State.IsTableCreate = TRUE) #define ClearContextTableCreate(Context) \ @@ -253,7 +271,60 @@ typedef PERFECT_HASH_CONTEXT_STATE *PPERFECT_HASH_CONTEXT_STATE; #define IsSystemRngId(Id) (Id == PerfectHashRngSystemId) #define IsSystemRng(Context) ((Context)->RngId == PerfectHashRngSystemId) -DEFINE_UNUSED_FLAGS(PERFECT_HASH_CONTEXT); +typedef union _PERFECT_HASH_CONTEXT_FLAGS { + struct { + + // + // When set, skip threadpool initialization for this context. + // + + ULONG SkipThreadpoolInitialization:1; + + // + // When set, indicates file I/O should use overlapped buffers instead + // of memory-mapped views. Intended for IOCP-native contexts. + // + + ULONG UseOverlappedIo:1; + + // + // When set, IOCP file I/O buffers use guard pages. + // + + ULONG UseIocpBufferGuardPages:1; + + // + // Unused bits. + // + + ULONG Unused:29; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CONTEXT_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_CONTEXT_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_CONTEXT_FLAGS *PPERFECT_HASH_CONTEXT_FLAGS; + +#define SkipThreadpoolInitialization(Context) \ + ((Context)->Flags.SkipThreadpoolInitialization != FALSE) + +#define UseOverlappedIo(Context) \ + ((Context)->Flags.UseOverlappedIo != FALSE) + +#define SetContextUseOverlappedIo(Context) \ + ((Context)->Flags.UseOverlappedIo = TRUE) + +#define ClearContextUseOverlappedIo(Context) \ + ((Context)->Flags.UseOverlappedIo = FALSE) + +#define UseIocpBufferGuardPages(Context) \ + ((Context)->Flags.UseIocpBufferGuardPages != FALSE) + +#define SetContextUseIocpBufferGuardPages(Context) \ + ((Context)->Flags.UseIocpBufferGuardPages = TRUE) + +#define ClearContextUseIocpBufferGuardPages(Context) \ + ((Context)->Flags.UseIocpBufferGuardPages = FALSE) typedef struct _BEST_GRAPH_INFO { @@ -1000,6 +1071,10 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { PTP_TIMER SolveTimeout; ULONG MinimumConcurrency; ULONG MaximumConcurrency; + ULONG InitialPerFileConcurrency; + ULONG MaxPerFileConcurrency; + ULONG IncreaseConcurrencyAfterMilliseconds; + ULONG Padding10; union { ULONG CuConcurrency; ULONG NumberOfGpuThreads; @@ -1071,6 +1146,23 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { #endif PTP_WORK FileWork; + // + // Optional IOCP integration for file work dispatch. + // + + HANDLE FileWorkIoCompletionPort; + HANDLE FileWorkOutstandingEvent; + volatile LONG FileWorkOutstanding; + ULONG FileWorkOutstandingPadding; + PPERFECT_HASH_IOCP_NODE IocpNode; + ULONG IocpNodeIndex; + ULONG IocpNodePadding; + PPERFECT_HASH_IOCP_BUFFER_POOL FileWorkBufferPools; + ULONG FileWorkBufferPoolCount; + ULONG FileWorkOversizePoolCount; + LIST_ENTRY FileWorkOversizePools; + PGUARDED_LIST FileWorkBufferList; + volatile LONG GraphRegisterSolvedTsxSuccess; ULONG Padding4; @@ -1393,6 +1485,23 @@ HRESULT ); typedef PERFECT_HASH_CONTEXT_RESET *PPERFECT_HASH_CONTEXT_RESET; +typedef +VOID +(NTAPI PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK)( + _In_ PPERFECT_HASH_CONTEXT Context + ); +typedef PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK + *PPERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK; + +typedef +VOID +(NTAPI PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS)( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ BOOL CancelPending + ); +typedef PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS + *PPERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS; + typedef VOID (NTAPI PERFECT_HASH_CONTEXT_APPLY_THREADPOOL_PRIORITIES)( @@ -1477,6 +1586,9 @@ PerfectHashContextInitializeLowMemoryMonitor( extern PERFECT_HASH_CONTEXT_INITIALIZE PerfectHashContextInitialize; extern PERFECT_HASH_CONTEXT_RUNDOWN PerfectHashContextRundown; extern PERFECT_HASH_CONTEXT_RESET PerfectHashContextReset; +extern PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK PerfectHashContextSubmitFileWork; +extern PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS + PerfectHashContextWaitForFileWorkCallbacks; extern PERFECT_HASH_CONTEXT_INITIALIZE_KEY_SIZE PerfectHashContextInitializeKeySize; extern PERFECT_HASH_CONTEXT_SET_MAXIMUM_CONCURRENCY diff --git a/src/PerfectHash/PerfectHashContextBulkCreate.c b/src/PerfectHash/PerfectHashContextBulkCreate.c index 327dcf5c..79419c62 100644 --- a/src/PerfectHash/PerfectHashContextBulkCreate.c +++ b/src/PerfectHash/PerfectHashContextBulkCreate.c @@ -206,6 +206,10 @@ Return Value: VALIDATE_FLAGS(KeysLoad, KEYS_LOAD, ULong); VALIDATE_FLAGS(TableCompile, TABLE_COMPILE, ULong); + if (UseOverlappedIo(Context)) { + KeysLoadFlags.UseOverlappedIo = TRUE; + } + // // IsValidTableCreateFlags() returns a more specific error code than the // other validation routines invoked above (which would be converted into @@ -346,15 +350,18 @@ Return Value: // Get a reference to the stdout handle. // - if (!Context->OutputHandle) { - Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + if (!Silent) { if (!Context->OutputHandle) { - SYS_ERROR(GetStdHandle); - goto Error; + Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + if (!Context->OutputHandle || + Context->OutputHandle == INVALID_HANDLE_VALUE) { + Context->OutputHandle = NULL; + Silent = TRUE; + } } - } - OutputHandle = Context->OutputHandle; + OutputHandle = Context->OutputHandle; + } // // Calculate the size required for a new concatenated wide string buffer @@ -1615,6 +1622,27 @@ Return Value: break; } + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractBulkCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + // // If we failed, clean up the table create parameters. If that fails, // report the error, then replace our return value error code with that diff --git a/src/PerfectHash/PerfectHashContextIocp.c b/src/PerfectHash/PerfectHashContextIocp.c new file mode 100644 index 00000000..db97e3da --- /dev/null +++ b/src/PerfectHash/PerfectHashContextIocp.c @@ -0,0 +1,1515 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + PerfectHashContextIocp.c + +Abstract: + + This module implements the IOCP-backed perfect hash context component. + The implementation is currently scaffolded to allow incremental bring-up + of the IOCP runtime, NUMA-aware worker pools, and bulk/table create + workflows. + +--*/ + +#include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" + +// +// Forward decls for internal helpers. +// + +static +HRESULT +PerfectHashContextIocpEnumerateNumaNodes( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); + +static +HRESULT +PerfectHashContextIocpCreateIoCompletionPorts( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); + +static +HRESULT +PerfectHashContextIocpCreateWorkerThreads( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); + +static +VOID +PerfectHashIocpApplyNodeAffinity( + _In_ PPERFECT_HASH_IOCP_NODE Node + ); + +static +VOID +PerfectHashIocpFreeBuffer( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ); + +static +VOID +PerfectHashIocpRundownBufferList( + _In_ PRTL Rtl, + _In_ PGUARDED_LIST BufferList + ); + +static +VOID +PerfectHashIocpRundownOversizePools( + _In_ PALLOCATOR Allocator, + _Inout_ PLIST_ENTRY ListHead, + _Inout_opt_ PULONG PoolCount + ); + +static +#ifdef PH_WINDOWS +DWORD +WINAPI +PerfectHashIocpWorkerThreadProc( + _In_ PVOID Parameter + ); +#else +VOID +PerfectHashIocpWorkerThreadProc( + _In_ PVOID Parameter + ); +#endif + +static +VOID +PerfectHashIocpFreeBuffer( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + BOOL Success; + HRESULT Result; + HANDLE ProcessHandle; + PVOID BaseAddress; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + ProcessHandle = NULL; + if (Buffer->OwnerPool) { + ProcessHandle = Buffer->OwnerPool->ProcessHandle; + } + + if (!ProcessHandle) { + ProcessHandle = GetCurrentProcess(); + } + + if (Buffer->Flags & PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES) { + BaseAddress = Buffer; + Result = Rtl->Vtbl->DestroyBuffer(Rtl, + ProcessHandle, + &BaseAddress, + Buffer->AllocationSize); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpFreeBuffer_DestroyBuffer, Result); + } + return; + } + + Success = VirtualFreeEx(ProcessHandle, Buffer, 0, MEM_RELEASE); + if (!Success) { + SYS_ERROR(VirtualFreeEx); + } +} + +static +VOID +PerfectHashIocpRundownBufferList( + _In_ PRTL Rtl, + _In_ PGUARDED_LIST BufferList + ) +{ + BOOLEAN NotEmpty; + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(BufferList)) { + return; + } + + while (TRUE) { + Entry = NULL; + NotEmpty = BufferList->Vtbl->RemoveHeadEx(BufferList, &Entry); + if (!NotEmpty) { + break; + } + + Buffer = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER, + ListEntry); + + PerfectHashIocpFreeBuffer(Rtl, Buffer); + } + + BufferList->Vtbl->Reset(BufferList); +} + +static +VOID +PerfectHashIocpRundownOversizePools( + _In_ PALLOCATOR Allocator, + _Inout_ PLIST_ENTRY ListHead, + _Inout_opt_ PULONG PoolCount + ) +{ + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + if (!ARGUMENT_PRESENT(Allocator) || !ARGUMENT_PRESENT(ListHead)) { + return; + } + + while (!IsListEmpty(ListHead)) { + Entry = RemoveHeadList(ListHead); + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + } + + InitializeListHead(ListHead); + if (PoolCount) { + *PoolCount = 0; + } +} + +PERFECT_HASH_CONTEXT_IOCP_INITIALIZE PerfectHashContextIocpInitialize; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpInitialize( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Initializes an IOCP-backed perfect hash table context. This routine + creates the Rtl and Allocator components and seeds default configuration + values. IOCP resources are brought online in later steps. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + for which initialization is to be performed. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ + HRESULT Result; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_RTL, + &ContextIocp->Rtl); + + if (FAILED(Result)) { + return Result; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_ALLOCATOR, + &ContextIocp->Allocator); + + if (FAILED(Result)) { + return Result; + } + + Rtl = ContextIocp->Rtl; + + ContextIocp->IocpConcurrency = 0; + ContextIocp->MaxWorkerThreads = 0; + + ContextIocp->NumaNodeCount = Rtl->CpuFeatures.NumaNodeCount; + if (ContextIocp->NumaNodeCount == 0) { + ContextIocp->NumaNodeCount = 1; + } + + ContextIocp->NumaNodeMask = PERFECT_HASH_NUMA_NODE_MASK_ALL; + ContextIocp->Flags.UseNumaNodeMask = FALSE; + ContextIocp->State.Initialized = TRUE; + ContextIocp->CompletionCallback = NULL; + ContextIocp->CompletionContext = NULL; + + ContextIocp->StartedEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!ContextIocp->StartedEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + ContextIocp->ShutdownEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!ContextIocp->ShutdownEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_RUNDOWN PerfectHashContextIocpRundown; + +_Use_decl_annotations_ +VOID +PerfectHashContextIocpRundown( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Rundowns an IOCP-backed perfect hash table context. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + to rundown. + +Return Value: + + None. + +--*/ +{ + ULONG Index; + PRTL Rtl; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return; + } + + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + + if (ContextIocp->State.Running) { + PerfectHashContextIocpStop(ContextIocp); + } + + if (ContextIocp->Nodes) { + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; + ULONG ThreadIndex; + + if (Node->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, + Node->FileWorkBufferList); + } + + if (Allocator) { + if (Node->FileWorkBufferPools) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Node->FileWorkBufferPools + ); + } + + PerfectHashIocpRundownOversizePools( + Allocator, + &Node->FileWorkOversizePools, + &Node->FileWorkOversizePoolCount + ); + } + + Node->FileWorkBufferPoolCount = 0; + + RELEASE(Node->FileWorkBufferList); + + if (Node->WorkerThreads) { + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + if (Node->WorkerThreads[ThreadIndex]) { + CloseHandle(Node->WorkerThreads[ThreadIndex]); + } + } + if (Allocator) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Node->WorkerThreads + ); + } + } + + if (Node->IoCompletionPort) { + CloseHandle(Node->IoCompletionPort); + } + } + if (Allocator) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&ContextIocp->Nodes + ); + } + } + + if (ContextIocp->ShutdownEvent) { + CloseHandle(ContextIocp->ShutdownEvent); + ContextIocp->ShutdownEvent = NULL; + } + + if (ContextIocp->StartedEvent) { + CloseHandle(ContextIocp->StartedEvent); + ContextIocp->StartedEvent = NULL; + } + + RELEASE(ContextIocp->BaseOutputDirectory); + RELEASE(ContextIocp->Allocator); + RELEASE(ContextIocp->Rtl); +} + +PERFECT_HASH_CONTEXT_IOCP_START PerfectHashContextIocpStart; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpStart( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Starts the IOCP runtime for a PERFECT_HASH_CONTEXT_IOCP instance. This + enumerates NUMA nodes, creates one IOCP per node, and spawns worker threads + with appropriate affinity. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + for which the runtime is to be started. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#else + HRESULT Result; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + if (ContextIocp->State.Running) { + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + return S_FALSE; + } + + Result = PerfectHashContextIocpEnumerateNumaNodes(ContextIocp); + if (FAILED(Result)) { + goto Error; + } + + Result = PerfectHashContextIocpCreateIoCompletionPorts(ContextIocp); + if (FAILED(Result)) { + goto Error; + } + + Result = PerfectHashContextIocpCreateWorkerThreads(ContextIocp); + if (FAILED(Result)) { + goto Error; + } + + ContextIocp->State.Running = TRUE; + if (ContextIocp->StartedEvent) { + SetEvent(ContextIocp->StartedEvent); + } + + Result = S_OK; + goto End; + +Error: + + PerfectHashContextIocpStop(ContextIocp); + +End: + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return Result; +#endif +} + +PERFECT_HASH_CONTEXT_IOCP_STOP PerfectHashContextIocpStop; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpStop( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Stops the IOCP runtime for a PERFECT_HASH_CONTEXT_IOCP instance by posting + shutdown completions to each worker thread and releasing resources. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + for which the runtime is to be stopped. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#else + ULONG NodeIndex; + ULONG ThreadIndex; + PRTL Rtl; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ContextIocp->Nodes) { + return S_FALSE; + } + + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + + ContextIocp->State.Stopping = TRUE; + if (ContextIocp->ShutdownEvent) { + SetEvent(ContextIocp->ShutdownEvent); + } + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + PostQueuedCompletionStatus(Node->IoCompletionPort, + 0, + PERFECT_HASH_IOCP_SHUTDOWN_KEY, + NULL); + } + } + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + if (Node->WorkerThreads[ThreadIndex]) { + WaitForSingleObject(Node->WorkerThreads[ThreadIndex], + INFINITE); + } + } + } + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + + if (Node->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, + Node->FileWorkBufferList); + } + + if (Allocator) { + if (Node->FileWorkBufferPools) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Node->FileWorkBufferPools + ); + } + + PerfectHashIocpRundownOversizePools( + Allocator, + &Node->FileWorkOversizePools, + &Node->FileWorkOversizePoolCount + ); + } + + Node->FileWorkBufferPoolCount = 0; + + RELEASE(Node->FileWorkBufferList); + + if (Node->WorkerThreads) { + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + if (Node->WorkerThreads[ThreadIndex]) { + CloseHandle(Node->WorkerThreads[ThreadIndex]); + } + } + if (ContextIocp->Allocator) { + ContextIocp->Allocator->Vtbl->FreePointer( + ContextIocp->Allocator, + (PVOID *)&Node->WorkerThreads + ); + } + } + + if (Node->IoCompletionPort) { + CloseHandle(Node->IoCompletionPort); + Node->IoCompletionPort = NULL; + } + } + + if (ContextIocp->Allocator) { + ContextIocp->Allocator->Vtbl->FreePointer( + ContextIocp->Allocator, + (PVOID *)&ContextIocp->Nodes + ); + } + + ContextIocp->Nodes = NULL; + ContextIocp->NodeCount = 0; + ContextIocp->IoCompletionPortCount = 0; + ContextIocp->TotalWorkerThreadCount = 0; + + ContextIocp->State.Stopped = TRUE; + ContextIocp->State.Running = FALSE; + + return S_OK; +#endif +} + +static +VOID +PerfectHashIocpApplyNodeAffinity( + PPERFECT_HASH_IOCP_NODE Node + ) +{ +#ifdef PH_WINDOWS + GROUP_AFFINITY Previous; + + if (!SetThreadGroupAffinity(GetCurrentThread(), + &Node->GroupAffinity, + &Previous)) { + if (Node->GroupAffinity.Group == 0) { + SetThreadAffinityMask(GetCurrentThread(), + Node->GroupAffinity.Mask); + } + } +#else + UNREFERENCED_PARAMETER(Node); +#endif +} + +#ifdef PH_WINDOWS +static +DWORD +WINAPI +PerfectHashIocpWorkerThreadProc( + PVOID Parameter + ) +{ + BOOL Success; + ULONG_PTR Key; + DWORD NumberOfBytes; + LPOVERLAPPED Overlapped; + PPERFECT_HASH_IOCP_NODE Node = (PPERFECT_HASH_IOCP_NODE)Parameter; + + PerfectHashIocpApplyNodeAffinity(Node); + + while (TRUE) { + Success = GetQueuedCompletionStatus(Node->IoCompletionPort, + &NumberOfBytes, + &Key, + &Overlapped, + INFINITE); + + if (Key == PERFECT_HASH_IOCP_SHUTDOWN_KEY) { + break; + } + + if (Overlapped) { + PPERFECT_HASH_IOCP_WORK WorkItem = + (PPERFECT_HASH_IOCP_WORK)Overlapped; + + if (WorkItem->Signature == PH_IOCP_WORK_SIGNATURE && + WorkItem->CompletionCallback) { + WorkItem->CompletionCallback(Node->ContextIocp, + Key, + Overlapped, + NumberOfBytes, + Success); + continue; + } + } + + if (Node->ContextIocp->CompletionCallback && Overlapped) { + Node->ContextIocp->CompletionCallback(Node->ContextIocp, + Key, + Overlapped, + NumberOfBytes, + Success); + } + } + + return 0; +} +#else +static +VOID +PerfectHashIocpWorkerThreadProc( + PVOID Parameter + ) +{ + UNREFERENCED_PARAMETER(Parameter); +} +#endif + +static +HRESULT +PerfectHashContextIocpEnumerateNumaNodes( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +{ +#ifdef PH_WINDOWS + BOOL Success; + HRESULT Result; + USHORT NodeId; + ULONG HighestNode; + USHORT HighestNodeShort; + ULONG SelectedCount; + ULONG TotalProcessors; + ULONGLONG SelectedMask; + PRTL Rtl; + PALLOCATOR Allocator; + PPERFECT_HASH_IOCP_NODE Nodes; + + if (ContextIocp->Nodes) { + return S_OK; + } + + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + if (!GetNumaHighestNodeNumber(&HighestNode)) { + SYS_ERROR(GetNumaHighestNodeNumber); + return PH_E_SYSTEM_CALL_FAILED; + } + + if (HighestNode >= 64) { + return E_NOTIMPL; + } + + HighestNodeShort = (USHORT)HighestNode; + + SelectedMask = ContextIocp->Flags.UseNumaNodeMask ? + ContextIocp->NumaNodeMask : + PERFECT_HASH_NUMA_NODE_MASK_ALL; + + SelectedCount = 0; + TotalProcessors = 0; + + for (NodeId = 0; NodeId <= HighestNodeShort; NodeId++) { + GROUP_AFFINITY Affinity = { 0 }; + ULONG ProcessorCount; + + Success = GetNumaNodeProcessorMaskEx(NodeId, &Affinity); + if (!Success) { + continue; + } + + if (SelectedMask != PERFECT_HASH_NUMA_NODE_MASK_ALL && + !(SelectedMask & (1ULL << NodeId))) { + continue; + } + + ProcessorCount = (ULONG)( + Rtl->PopulationCountPointer((ULONG_PTR)Affinity.Mask) + ); + + if (ProcessorCount == 0) { + continue; + } + + SelectedCount++; + TotalProcessors += ProcessorCount; + } + + if (SelectedCount == 0 || TotalProcessors == 0) { + return E_INVALIDARG; + } + + Nodes = (PPERFECT_HASH_IOCP_NODE)( + Allocator->Vtbl->Calloc(Allocator, SelectedCount, sizeof(*Nodes)) + ); + if (!Nodes) { + return E_OUTOFMEMORY; + } + + ContextIocp->Nodes = Nodes; + ContextIocp->NodeCount = SelectedCount; + ContextIocp->IoCompletionPortCount = SelectedCount; + + SelectedCount = 0; + + for (NodeId = 0; NodeId <= HighestNodeShort; NodeId++) { + GROUP_AFFINITY Affinity = { 0 }; + ULONG ProcessorCount; + PPERFECT_HASH_IOCP_NODE Node; + + Success = GetNumaNodeProcessorMaskEx(NodeId, &Affinity); + if (!Success) { + continue; + } + + if (SelectedMask != PERFECT_HASH_NUMA_NODE_MASK_ALL && + !(SelectedMask & (1ULL << NodeId))) { + continue; + } + + ProcessorCount = (ULONG)( + Rtl->PopulationCountPointer((ULONG_PTR)Affinity.Mask) + ); + + if (ProcessorCount == 0) { + continue; + } + + Node = &Nodes[SelectedCount++]; + Node->ContextIocp = ContextIocp; + Node->NodeId = NodeId; + Node->ProcessorCount = ProcessorCount; + Node->GroupAffinity = Affinity; + InitializeSRWLock(&Node->FileWorkBufferPoolLock); + InitializeListHead(&Node->FileWorkOversizePools); + Node->FileWorkBufferPoolCount = 0; + Node->FileWorkOversizePoolCount = 0; + + Result = ContextIocp->Vtbl->CreateInstance( + ContextIocp, + NULL, + &IID_PERFECT_HASH_GUARDED_LIST, + &Node->FileWorkBufferList + ); + + if (FAILED(Result)) { + return Result; + } + } + + // + // Configure per-node IOCP concurrency and worker thread counts. + // + + { + ULONG Index; + ULONG TotalWorkerThreads = 0; + ULONG DefaultIocpConcurrency = ContextIocp->IocpConcurrency; + ULONG DefaultMaxThreads = ContextIocp->MaxWorkerThreads; + + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &Nodes[Index]; + ULONG IocpConcurrency; + ULONG MaxThreads; + + IocpConcurrency = DefaultIocpConcurrency; + if (IocpConcurrency == 0 || + IocpConcurrency > Node->ProcessorCount) { + IocpConcurrency = Node->ProcessorCount; + } + + if (IocpConcurrency == 0) { + IocpConcurrency = 1; + } + + MaxThreads = DefaultMaxThreads; + if (MaxThreads == 0) { + if (DefaultIocpConcurrency != 0) { + MaxThreads = IocpConcurrency * 2; + } else { + MaxThreads = Node->ProcessorCount; + } + } + + if (MaxThreads == 0) { + MaxThreads = 1; + } + + Node->IocpConcurrency = IocpConcurrency; + Node->WorkerThreadCount = MaxThreads; + TotalWorkerThreads += MaxThreads; + } + + ContextIocp->TotalWorkerThreadCount = TotalWorkerThreads; + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#endif +} + +static +HRESULT +PerfectHashContextIocpCreateIoCompletionPorts( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +{ +#ifdef PH_WINDOWS + ULONG Index; + + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; + + if (Node->IoCompletionPort) { + continue; + } + + Node->IoCompletionPort = CreateIoCompletionPort( + INVALID_HANDLE_VALUE, + NULL, + 0, + Node->IocpConcurrency + ); + + if (!Node->IoCompletionPort) { + SYS_ERROR(CreateIoCompletionPort); + return PH_E_SYSTEM_CALL_FAILED; + } + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#endif +} + +static +HRESULT +PerfectHashContextIocpCreateWorkerThreads( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +{ +#ifdef PH_WINDOWS + ULONG Index; + PALLOCATOR Allocator; + + Allocator = ContextIocp->Allocator; + + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; + ULONG ThreadIndex; + + if (Node->WorkerThreadCount == 0) { + continue; + } + + if (Node->WorkerThreads) { + continue; + } + + Node->WorkerThreads = (PHANDLE)( + Allocator->Vtbl->Calloc(Allocator, + Node->WorkerThreadCount, + sizeof(HANDLE)) + ); + if (!Node->WorkerThreads) { + return E_OUTOFMEMORY; + } + + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + HANDLE ThreadHandle; + + ThreadHandle = CreateThread(NULL, + 0, + PerfectHashIocpWorkerThreadProc, + Node, + 0, + NULL); + if (!ThreadHandle) { + SYS_ERROR(CreateThread); + return PH_E_SYSTEM_CALL_FAILED; + } + + Node->WorkerThreads[ThreadIndex] = ThreadHandle; + } + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#endif +} + +PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY +PerfectHashContextIocpSetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetMaximumConcurrency( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG MaximumConcurrency + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (MaximumConcurrency == 0) { + return E_INVALIDARG; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + ContextIocp->IocpConcurrency = MaximumConcurrency; + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY +PerfectHashContextIocpGetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetMaximumConcurrency( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PULONG MaximumConcurrency + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + *MaximumConcurrency = ContextIocp->IocpConcurrency; + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS + PerfectHashContextIocpSetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetMaximumThreads( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG MaximumThreads + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (MaximumThreads == 0) { + return E_INVALIDARG; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + ContextIocp->MaxWorkerThreads = MaximumThreads; + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS + PerfectHashContextIocpGetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetMaximumThreads( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PULONG MaximumThreads + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumThreads)) { + return E_POINTER; + } + + *MaximumThreads = ContextIocp->MaxWorkerThreads; + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK + PerfectHashContextIocpSetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetNumaNodeMask( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + ContextIocp->NumaNodeMask = NumaNodeMask; + ContextIocp->Flags.UseNumaNodeMask = TRUE; + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK + PerfectHashContextIocpGetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetNumaNodeMask( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(NumaNodeMask)) { + return E_POINTER; + } + + *NumaNodeMask = ContextIocp->NumaNodeMask; + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpSetBaseOutputDirectory; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetBaseOutputDirectory( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PCUNICODE_STRING BaseOutputDirectory + ) +{ + HRESULT Result = S_OK; + PPERFECT_HASH_PATH Path = NULL; + PPERFECT_HASH_PATH_PARTS Parts = NULL; + PPERFECT_HASH_DIRECTORY Directory; + PERFECT_HASH_DIRECTORY_CREATE_FLAGS DirectoryCreateFlags = { 0 }; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectory)) { + return E_POINTER; + } + + if (!IsValidMinimumDirectoryNullTerminatedUnicodeString( + BaseOutputDirectory)) { + return E_INVALIDARG; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + if (ContextIocp->BaseOutputDirectory) { + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + return PH_E_CONTEXT_BASE_OUTPUT_DIRECTORY_ALREADY_SET; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_PATH, + &Path); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashPathCreateInstance, Result); + goto Error; + } + + Result = Path->Vtbl->Copy(Path, BaseOutputDirectory, &Parts, NULL); + if (FAILED(Result)) { + PH_ERROR(PerfectHashPathCopy, Result); + goto Error; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_DIRECTORY, + &ContextIocp->BaseOutputDirectory); + + if (FAILED(Result)) { + PH_ERROR(CreateInstancePerfectHashDirectory, Result); + goto Error; + } + + Directory = ContextIocp->BaseOutputDirectory; + + Result = Directory->Vtbl->Create(Directory, + Path, + &DirectoryCreateFlags); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashDirectoryCreate, Result); + goto Error; + } + + goto End; + +Error: + + if (Result == S_OK) { + Result = PH_E_CONTEXT_SET_BASE_OUTPUT_DIRECTORY_FAILED; + } + +End: + + RELEASE(Path); + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + return Result; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpGetBaseOutputDirectory; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetBaseOutputDirectory( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PPERFECT_HASH_DIRECTORY *BaseOutputDirectoryPointer + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectoryPointer)) { + return E_POINTER; + } + + *BaseOutputDirectoryPointer = ContextIocp->BaseOutputDirectory; + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT + PerfectHashContextIocpCreateTableContext; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpCreateTableContext( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PPERFECT_HASH_CONTEXT *ContextPointer + ) +/*++ + +Routine Description: + + Creates a PERFECT_HASH_CONTEXT instance suitable for IOCP-native + workflows. The TLS flag CreateContextWithoutThreadpool is set for the + duration of the call to suppress threadpool initialization. + +Arguments: + + ContextIocp - Supplies a pointer to the IOCP context. + + ContextPointer - Receives the newly created PERFECT_HASH_CONTEXT instance. + +Return Value: + + S_OK on success, otherwise an error code. + +--*/ +{ + HRESULT Result; + BOOLEAN LocalTlsActive = FALSE; + PPERFECT_HASH_TLS_CONTEXT ActiveTls; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext = { 0 }; + PERFECT_HASH_TLS_CONTEXT_FLAGS SavedFlags = { 0 }; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ContextPointer)) { + return E_POINTER; + } + + *ContextPointer = NULL; + + ActiveTls = PerfectHashTlsGetContext(); + if (ActiveTls) { + SavedFlags = ActiveTls->Flags; + ActiveTls->Flags.CreateContextWithoutThreadpool = TRUE; + } else { + ActiveTls = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + ActiveTls->Flags.CreateContextWithoutThreadpool = TRUE; + LocalTlsActive = TRUE; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_CONTEXT, + ContextPointer); + + if (SUCCEEDED(Result) && *ContextPointer) { + SetContextUseOverlappedIo(*ContextPointer); + } + + if (LocalTlsActive) { + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + } else if (ActiveTls) { + ActiveTls->Flags = SavedFlags; + } + + return Result; +} + +PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE PerfectHashContextIocpBulkCreate; +PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW PerfectHashContextIocpBulkCreateArgvW; +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractBulkCreateArgsFromArgvW; +PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE PerfectHashContextIocpTableCreate; +PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW + PerfectHashContextIocpTableCreateArgvW; +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractTableCreateArgsFromArgvW; +PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA + PerfectHashContextIocpTableCreateArgvA; +PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA + PerfectHashContextIocpBulkCreateArgvA; + +static +HRESULT +PerfectHashContextIocpInvokeLegacyContextArgvW( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW, + _In_ BOOLEAN IsBulkCreate + ) +{ + HRESULT Result; + PPERFECT_HASH_CONTEXT Context; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_CONTEXT, + &Context); + if (FAILED(Result)) { + return Result; + } + + { + ULONG Concurrency; + + Concurrency = ContextIocp->MaxWorkerThreads; + if (Concurrency == 0) { + Concurrency = ContextIocp->IocpConcurrency; + } + + if (Concurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency( + Context, + Concurrency + ); + if (FAILED(Result)) { + RELEASE(Context); + return Result; + } + } + } + + if (IsBulkCreate) { + Result = Context->Vtbl->BulkCreateArgvW(Context, + NumberOfArguments, + ArgvW, + CommandLineW); + } else { + Result = Context->Vtbl->TableCreateArgvW(Context, + NumberOfArguments, + ArgvW, + CommandLineW); + } + + RELEASE(Context); + return Result; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpBulkCreate( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PCUNICODE_STRING KeysDirectory, + PCUNICODE_STRING BaseOutputDirectory, + PERFECT_HASH_ALGORITHM_ID AlgorithmId, + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(KeysDirectory); + UNREFERENCED_PARAMETER(BaseOutputDirectory); + UNREFERENCED_PARAMETER(AlgorithmId); + UNREFERENCED_PARAMETER(HashFunctionId); + UNREFERENCED_PARAMETER(MaskFunctionId); + UNREFERENCED_PARAMETER(ContextBulkCreateFlags); + UNREFERENCED_PARAMETER(KeysLoadFlags); + UNREFERENCED_PARAMETER(TableCreateFlags); + UNREFERENCED_PARAMETER(TableCompileFlags); + UNREFERENCED_PARAMETER(TableCreateParameters); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpBulkCreateArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW + ) +{ + return PerfectHashContextIocpInvokeLegacyContextArgvW( + ContextIocp, + NumberOfArguments, + ArgvW, + CommandLineW, + TRUE + ); +} + + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpTableCreate( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PCUNICODE_STRING KeysPath, + PCUNICODE_STRING BaseOutputDirectory, + PERFECT_HASH_ALGORITHM_ID AlgorithmId, + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(KeysPath); + UNREFERENCED_PARAMETER(BaseOutputDirectory); + UNREFERENCED_PARAMETER(AlgorithmId); + UNREFERENCED_PARAMETER(HashFunctionId); + UNREFERENCED_PARAMETER(MaskFunctionId); + UNREFERENCED_PARAMETER(ContextTableCreateFlags); + UNREFERENCED_PARAMETER(KeysLoadFlags); + UNREFERENCED_PARAMETER(TableCreateFlags); + UNREFERENCED_PARAMETER(TableCompileFlags); + UNREFERENCED_PARAMETER(TableCreateParameters); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpTableCreateArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW + ) +{ + return PerfectHashContextIocpInvokeLegacyContextArgvW( + ContextIocp, + NumberOfArguments, + ArgvW, + CommandLineW, + FALSE + ); +} + + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpTableCreateArgvA( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPSTR *ArgvA + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpBulkCreateArgvA( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPSTR *ArgvA + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + + return E_NOTIMPL; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashContextIocp.h b/src/PerfectHash/PerfectHashContextIocp.h new file mode 100644 index 00000000..fb783a1e --- /dev/null +++ b/src/PerfectHash/PerfectHashContextIocp.h @@ -0,0 +1,271 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + PerfectHashContextIocp.h + +Abstract: + + This is the private header file for the PERFECT_HASH_CONTEXT_IOCP + component of the perfect hash library. It defines the structure, and + function pointer typedefs for private non-vtbl members. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef union _PERFECT_HASH_CONTEXT_IOCP_STATE { + struct { + + ULONG Initialized:1; + ULONG Running:1; + ULONG Stopping:1; + ULONG Stopped:1; + + ULONG Unused:28; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CONTEXT_IOCP_STATE; +C_ASSERT(sizeof(PERFECT_HASH_CONTEXT_IOCP_STATE) == sizeof(ULONG)); +typedef PERFECT_HASH_CONTEXT_IOCP_STATE *PPERFECT_HASH_CONTEXT_IOCP_STATE; + +typedef union _PERFECT_HASH_CONTEXT_IOCP_FLAGS { + struct { + + // + // When set, indicates an explicit NUMA node mask is in effect. + // + + ULONG UseNumaNodeMask:1; + + ULONG Unused:31; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CONTEXT_IOCP_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_CONTEXT_IOCP_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_CONTEXT_IOCP_FLAGS *PPERFECT_HASH_CONTEXT_IOCP_FLAGS; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_IOCP_COMPLETION_CALLBACK)( + _In_ struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ); +typedef PERFECT_HASH_IOCP_COMPLETION_CALLBACK + *PPERFECT_HASH_IOCP_COMPLETION_CALLBACK; + +// +// IOCP work item header used to route completions to a per-item callback. +// + +#define PERFECT_HASH_IOCP_SHUTDOWN_KEY ((ULONG_PTR)1) + +#define PH_IOCP_WORK_SIGNATURE 0x50434F49u // 'IOCP' + +#define PH_IOCP_WORK_FLAG_PIPE 0x00000001 +#define PH_IOCP_WORK_FLAG_BULK 0x00000002 +#define PH_IOCP_WORK_FLAG_ASYNC 0x00000004 +#define PH_IOCP_WORK_FLAG_FILE_WORK 0x00000008 +#define PH_IOCP_WORK_FLAG_FILE_IO 0x00000010 + +typedef struct _PERFECT_HASH_IOCP_WORK { + OVERLAPPED Overlapped; + ULONG Signature; + ULONG Flags; + PPERFECT_HASH_IOCP_COMPLETION_CALLBACK CompletionCallback; + PVOID CompletionContext; +} PERFECT_HASH_IOCP_WORK; +typedef PERFECT_HASH_IOCP_WORK *PPERFECT_HASH_IOCP_WORK; + +C_ASSERT(FIELD_OFFSET(PERFECT_HASH_IOCP_WORK, Overlapped) == 0); + +typedef struct _PERFECT_HASH_IOCP_NODE { + struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp; + ULONG NodeId; + ULONG ProcessorCount; + ULONG IocpConcurrency; + ULONG Padding1; +#ifdef PH_WINDOWS + GROUP_AFFINITY GroupAffinity; +#else + ULONGLONG ProcessorMask; +#endif + HANDLE IoCompletionPort; + HANDLE *WorkerThreads; + ULONG WorkerThreadCount; + ULONG Padding2; + + // + // IOCP file work buffer pools (per-node). + // + + SRWLOCK FileWorkBufferPoolLock; + PPERFECT_HASH_IOCP_BUFFER_POOL FileWorkBufferPools; + ULONG FileWorkBufferPoolCount; + ULONG FileWorkOversizePoolCount; + LIST_ENTRY FileWorkOversizePools; + PGUARDED_LIST FileWorkBufferList; +} PERFECT_HASH_IOCP_NODE; +typedef PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT_IOCP { + + COMMON_COMPONENT_HEADER(PERFECT_HASH_CONTEXT_IOCP); + + // + // Configuration. + // + + ULONG IocpConcurrency; + ULONG MaxWorkerThreads; + ULONG NumaNodeCount; + ULONG Padding1; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; + ULONG Padding2; + ULONG Padding3; + + // + // Pointer to the base output directory, if set. + // + + PPERFECT_HASH_DIRECTORY BaseOutputDirectory; + + // + // Threading resources (per NUMA node IOCPs, worker threads, etc.). + // + + PPERFECT_HASH_IOCP_NODE Nodes; + ULONG NodeCount; + ULONG TotalWorkerThreadCount; + ULONG IoCompletionPortCount; + ULONG Padding4; + + // + // Completion dispatch callback (owned by higher-level components). + // + + PPERFECT_HASH_IOCP_COMPLETION_CALLBACK CompletionCallback; + PVOID CompletionContext; + + // + // Shutdown and coordination. + // + + HANDLE ShutdownEvent; + HANDLE StartedEvent; + + // + // Backing vtbl. + // + + PERFECT_HASH_CONTEXT_IOCP_VTBL Interface; + +} PERFECT_HASH_CONTEXT_IOCP; +typedef PERFECT_HASH_CONTEXT_IOCP *PPERFECT_HASH_CONTEXT_IOCP; + +#define TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp) \ + TryAcquireSRWLockExclusive(&ContextIocp->Lock) + +#define ReleasePerfectHashContextIocpLockExclusive(ContextIocp) \ + ReleaseSRWLockExclusive(&ContextIocp->Lock) + +// +// Function pointer typedefs for non-vtbl members (if applicable). +// + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_INITIALIZE)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_INITIALIZE + *PPERFECT_HASH_CONTEXT_IOCP_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_CONTEXT_IOCP_RUNDOWN)( + _In_ _Post_ptr_invalid_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_RUNDOWN + *PPERFECT_HASH_CONTEXT_IOCP_RUNDOWN; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_START)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_START + *PPERFECT_HASH_CONTEXT_IOCP_START; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_STOP)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_STOP + *PPERFECT_HASH_CONTEXT_IOCP_STOP; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Outptr_ PPERFECT_HASH_CONTEXT *Context + ); +typedef PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT + *PPERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT; + +extern PERFECT_HASH_CONTEXT_IOCP_INITIALIZE PerfectHashContextIocpInitialize; +extern PERFECT_HASH_CONTEXT_IOCP_RUNDOWN PerfectHashContextIocpRundown; +extern PERFECT_HASH_CONTEXT_IOCP_START PerfectHashContextIocpStart; +extern PERFECT_HASH_CONTEXT_IOCP_STOP PerfectHashContextIocpStop; +extern PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT + PerfectHashContextIocpCreateTableContext; + +extern PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY + PerfectHashContextIocpSetMaximumConcurrency; +extern PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY + PerfectHashContextIocpGetMaximumConcurrency; +extern PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS + PerfectHashContextIocpSetMaximumThreads; +extern PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS + PerfectHashContextIocpGetMaximumThreads; +extern PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK + PerfectHashContextIocpSetNumaNodeMask; +extern PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK + PerfectHashContextIocpGetNumaNodeMask; +extern PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpSetBaseOutputDirectory; +extern PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpGetBaseOutputDirectory; +extern PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE + PerfectHashContextIocpBulkCreate; +extern PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW + PerfectHashContextIocpBulkCreateArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractBulkCreateArgsFromArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE + PerfectHashContextIocpTableCreate; +extern PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW + PerfectHashContextIocpTableCreateArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractTableCreateArgsFromArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA + PerfectHashContextIocpTableCreateArgvA; +extern PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA + PerfectHashContextIocpBulkCreateArgvA; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashContextIocpArgs.c b/src/PerfectHash/PerfectHashContextIocpArgs.c new file mode 100644 index 00000000..1897277e --- /dev/null +++ b/src/PerfectHash/PerfectHashContextIocpArgs.c @@ -0,0 +1,859 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + PerfectHashContextIocpArgs.c + +Abstract: + + This module implements IOCP-native argument extraction helpers for + bulk-create and table-create operations. These routines mirror the + legacy context parsing logic but operate directly on the IOCP context's + RTL/allocator, avoiding the legacy context dependency. + +--*/ + +#include "stdafx.h" + +#define GET_LENGTH(Name) \ + ((USHORT)wcslen((Name)->Buffer) * (USHORT)sizeof(WCHAR)) +#define GET_MAX_LENGTH(Name) ((Name)->Length + sizeof(WCHAR)) + +#define VALIDATE_ID(Name, Upper) \ + if (FAILED(Rtl->RtlUnicodeStringToInteger(String, \ + 10, \ + (PULONG)Name##Id))) { \ + return PH_E_INVALID_##Upper##_ID; \ + } else if (*Name##Id == 0) { \ + Result = PerfectHashLookupIdForName(Rtl, \ + PerfectHash##Name##EnumId, \ + String, \ + (PULONG)Name##Id); \ + if (FAILED(Result)) { \ + return PH_E_INVALID_##Upper##_ID; \ + } \ + } \ + if (!IsValidPerfectHash##Name##Id(*Name##Id)) { \ + return PH_E_INVALID_##Upper##_ID; \ + } + +#define EXTRACT_ID(Name, Upper) \ + CurrentArg++; \ + String->Buffer = *ArgW++; \ + String->Length = GET_LENGTH(String); \ + String->MaximumLength = GET_MAX_LENGTH(String); \ + VALIDATE_ID(Name, Upper) + +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractBulkCreateArgsFromArgvW; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpExtractBulkCreateArgsFromArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW, + PUNICODE_STRING KeysDirectory, + PUNICODE_STRING BaseOutputDirectory, + PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PULONG MaximumConcurrency, + PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +/*++ + +Routine Description: + + Extracts arguments for the bulk-create functionality from an argument + vector array, typically obtained from a commandline invocation. + +Arguments: + + ContextIocp - Supplies a pointer to the PERFECT_HASH_CONTEXT_IOCP instance + for which the arguments are to be extracted. + + NumberOfArguments - Supplies the number of elements in the ArgvW array. + + ArgvW - Supplies a pointer to an array of wide C string arguments. + + CommandLineW - Supplies a pointer to the original command line used to + construct the ArgvW array above. This is only used for inclusion in + things like CSV output; it is not used programmatically (and is not + checked for correctness against ArgvW). + + KeysDirectory - Supplies a pointer to a UNICODE_STRING structure that + will be filled out with the keys directory. + + BaseOutputDirectory - Supplies a pointer to a UNICODE_STRING structure that + will be filled out with the output directory. + + AlgorithmId - Supplies the address of a variable that will receive the + algorithm ID. + + HashFunctionId - Supplies the address of a variable that will receive the + hash function ID. + + MaskFunctionId - Supplies the address of a variable that will receive the + mask function ID. + + MaximumConcurrency - Supplies the address of a variable that will receive + the maximum concurrency. + + ContextBulkCreateFlags - Supplies the address of a variable that will + receive the bulk-create flags. + + KeysLoadFlags - Supplies the address of a variable that will receive the + keys load flags. + + TableCreateFlags - Supplies the address of a variable that will receive + the table create flags. + + TableCompileFlags - Supplies the address of a variable that will receive + the table compile flags. + + TableCreateParameters - Supplies the address of a variable that will + receive a pointer to a table create parameters structure. + +Return Value: + + S_OK - Arguments extracted successfully. + + E_POINTER - One or more mandatory parameters were NULL pointers. + + PH_E_CONTEXT_BULK_CREATE_INVALID_NUM_ARGS - Invalid number of arguments. + + PH_E_INVALID_ALGORITHM_ID - Invalid algorithm ID. + + PH_E_INVALID_HASH_FUNCTION_ID - Invalid hash function ID. + + PH_E_INVALID_MASK_FUNCTION_ID - Invalid mask function ID. + + PH_E_INVALID_MAXIMUM_CONCURRENCY - Invalid maximum concurrency. + +--*/ +{ + PRTL Rtl; + LPWSTR *ArgW; + LPWSTR Arg; + HRESULT Result = S_OK; + HRESULT CleanupResult; + ULONG CurrentArg = 1; + PALLOCATOR Allocator; + UNICODE_STRING Temp; + PUNICODE_STRING String; + BOOLEAN InvalidPrefix; + BOOLEAN ValidNumberOfArguments; + + String = &Temp; + + // + // Validate arguments. + // + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ArgvW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(CommandLineW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysDirectory)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectory)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(AlgorithmId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(HashFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaskFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ContextBulkCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysLoadFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCompileFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateParameters)) { + return E_POINTER; + } + + ValidNumberOfArguments = (NumberOfArguments >= 7); + + if (!ValidNumberOfArguments) { + return PH_E_CONTEXT_BULK_CREATE_INVALID_NUM_ARGS; + } + + // + // Argument validation complete, continue. + // + + ArgW = &ArgvW[1]; + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + // + // The first six arguments (keys directory, base output directory, algo ID, + // hash function ID, mask function ID and maximum concurrency) are special + // in that they're mandatory and expected to appear sequentially, prior to + // any additional arguments (i.e. table create parameters) appearing. + // + + // + // Extract keys directory. + // + + CurrentArg++; + KeysDirectory->Buffer = *ArgW++; + KeysDirectory->Length = GET_LENGTH(KeysDirectory); + KeysDirectory->MaximumLength = GET_MAX_LENGTH(KeysDirectory); + + // + // Extract base output directory. + // + + CurrentArg++; + BaseOutputDirectory->Buffer = *ArgW++; + BaseOutputDirectory->Length = GET_LENGTH(BaseOutputDirectory); + BaseOutputDirectory->MaximumLength = GET_MAX_LENGTH(BaseOutputDirectory); + + // + // Extract algo, hash function and mask function IDs. + // + + EXTRACT_ID(Algorithm, ALGORITHM); + EXTRACT_ID(HashFunction, HASH_FUNCTION); + EXTRACT_ID(MaskFunction, MASK_FUNCTION); + + // + // Extract maximum concurrency. + // + + CurrentArg++; + String->Buffer = *ArgW++; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + if (FAILED(Rtl->RtlUnicodeStringToInteger(String, + 10, + MaximumConcurrency))) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + // + // Zero all flags (except for table create flags, as these may have + // default values) and table create parameters. + // + + KeysLoadFlags->AsULong = 0; + TableCompileFlags->AsULong = 0; + ContextBulkCreateFlags->AsULong = 0; + + for (; CurrentArg < NumberOfArguments; CurrentArg++, ArgW++) { + + String->Buffer = Arg = *ArgW; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + // + // If the argument doesn't start with two dashes, report it. + // + + InvalidPrefix = ( + (String->Length <= (sizeof(L'-') + sizeof(L'-'))) || + (!(*Arg++ == L'-' && *Arg++ == L'-')) + ); + + if (InvalidPrefix) { + goto InvalidArg; + } + + // + // Advance the buffer past the two dashes and update lengths + // accordingly. + // + + String->Buffer += 2; + String->Length -= (sizeof(WCHAR) * 2); + String->MaximumLength -= (sizeof(WCHAR) * 2); + + // + // Try each argument extraction routine for this argument; if it + // indicates an error, report it and break out of the loop. If it + // indicates it successfully extracted the argument (Result == S_OK), + // continue onto the next argument. Otherwise, verify it indicates + // that no argument was extracted (S_FALSE), then try the next routine. + // + +#define TRY_EXTRACT_ARG(Name) \ + Result = TryExtractArg##Name##Flags(Rtl, Allocator, String, Name##Flags); \ + if (FAILED(Result)) { \ + PH_ERROR(ExtractBulkCreateArgs_TryExtractArg##Name##Flags, Result); \ + break; \ + } else if (Result == S_OK) { \ + continue; \ + } else { \ + ASSERT(Result == S_FALSE); \ + } + + TRY_EXTRACT_ARG(ContextBulkCreate); + TRY_EXTRACT_ARG(KeysLoad); + TRY_EXTRACT_ARG(TableCreate); + TRY_EXTRACT_ARG(TableCompile); + + // + // If we get here, none of the previous extraction routines claimed the + // argument, so, provide the table create parameters extraction routine + // an opportunity to run. + // + + Result = TryExtractArgTableCreateParameters(Rtl, + String, + TableCreateParameters); + + if (FAILED(Result)) { + PH_ERROR(ExtractBulkCreateArgs_TryExtractTableCreateParams, Result); + break; + } + + if (Result == S_OK) { + continue; + } + + if (Result == PH_E_COMMANDLINE_ARG_MISSING_VALUE) { + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (FAILED(Result)) { + PH_ERROR(ExtractBulkCreateArgs_TryExtractTableCreateParams, Result); + break; + } + + ASSERT(Result == S_FALSE); + +InvalidArg: + + // + // If we get here, we don't recognize the argument. + // + + Result = PH_E_INVALID_COMMANDLINE_ARG; + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractBulkCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + + if (SUCCEEDED(Result) && + *MaximumConcurrency == 0 && + ContextIocp->IocpConcurrency > 0) { + *MaximumConcurrency = ContextIocp->IocpConcurrency; + } + + if (SUCCEEDED(Result) && + ContextIocp->IocpConcurrency > 0 && + *MaximumConcurrency > ContextIocp->IocpConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + // + // If we failed, clean up the table create parameters. If that fails, + // report the error, then replace our return value error code with that + // error code. + // + + if (FAILED(Result)) { + CleanupResult = CleanupTableCreateParameters(TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + Result = CleanupResult; + } + } + + return Result; +} + +#undef TRY_EXTRACT_ARG + +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractTableCreateArgsFromArgvW; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpExtractTableCreateArgsFromArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW, + PUNICODE_STRING KeysPath, + PUNICODE_STRING BaseOutputDirectory, + PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PULONG MaximumConcurrency, + PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +/*++ + +Routine Description: + + Extracts arguments for the table create functionality from an argument + vector array, typically obtained from a commandline invocation. + +Arguments: + + ContextIocp - Supplies a pointer to the PERFECT_HASH_CONTEXT_IOCP instance + for which the arguments are to be extracted. + + NumberOfArguments - Supplies the number of elements in the ArgvW array. + + ArgvW - Supplies a pointer to an array of wide C string arguments. + + CommandLineW - Supplies a pointer to the original command line used to + construct the ArgvW array above. This is only used for inclusion in + things like CSV output; it is not used programmatically (and is not + checked for correctness against ArgvW). + + KeysPath - Supplies a pointer to a UNICODE_STRING structure that will be + filled out with the keys path. + + BaseOutputDirectory - Supplies a pointer to a UNICODE_STRING structure that + will be filled out with the output directory. + + AlgorithmId - Supplies the address of a variable that will receive the + algorithm ID. + + HashFunctionId - Supplies the address of a variable that will receive the + hash function ID. + + MaskFunctionId - Supplies the address of a variable that will receive the + mask function ID. + + MaximumConcurrency - Supplies the address of a variable that will receive + the maximum concurrency. + + ContextTableCreateFlags - Supplies the address of a variable that will + receive the context table-create flags. + + KeysLoadFlags - Supplies the address of a variable that will receive the + keys load flags. + + TableCreateFlags - Supplies the address of a variable that will receive + the table create flags. + + TableCompileFlags - Supplies the address of a variable that will receive + the table compile flags. + + TableCreateParameters - Supplies the address of a table create params + structure that will receive any extracted params. + +Return Value: + + S_OK - Arguments extracted successfully. + + E_POINTER - One or more mandatory parameters were NULL pointers. + + PH_E_CONTEXT_TABLE_CREATE_INVALID_NUM_ARGS - Invalid number of arguments. + + PH_E_INVALID_ALGORITHM_ID - Invalid algorithm ID. + + PH_E_INVALID_HASH_FUNCTION_ID - Invalid hash function ID. + + PH_E_INVALID_MASK_FUNCTION_ID - Invalid mask function ID. + + PH_E_INVALID_MAXIMUM_CONCURRENCY - Invalid maximum concurrency. + +--*/ +{ + PRTL Rtl; + LPWSTR *ArgW; + LPWSTR Arg; + HRESULT Result = S_OK; + HRESULT CleanupResult; + HRESULT DebuggerResult; + ULONG CurrentArg = 1; + PALLOCATOR Allocator; + UNICODE_STRING Temp; + PUNICODE_STRING String; + BOOLEAN InvalidPrefix; + DEBUGGER_CONTEXT_FLAGS Flags; + BOOLEAN ValidNumberOfArguments; + PDEBUGGER_CONTEXT DebuggerContext; + + String = &Temp; + + // + // Validate arguments. + // + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ArgvW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(CommandLineW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysPath)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectory)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(AlgorithmId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(HashFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaskFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ContextTableCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysLoadFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCompileFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateParameters)) { + return E_POINTER; + } + + ValidNumberOfArguments = (NumberOfArguments >= 7); + + if (!ValidNumberOfArguments) { + return PH_E_CONTEXT_TABLE_CREATE_INVALID_NUM_ARGS; + } + + // + // Argument validation complete, continue. + // + + ArgW = &ArgvW[1]; + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + // + // Extract keys path. + // + + CurrentArg++; + KeysPath->Buffer = *ArgW++; + KeysPath->Length = GET_LENGTH(KeysPath); + KeysPath->MaximumLength = GET_MAX_LENGTH(KeysPath); + + // + // Extract base output directory. + // + + CurrentArg++; + BaseOutputDirectory->Buffer = *ArgW++; + BaseOutputDirectory->Length = GET_LENGTH(BaseOutputDirectory); + BaseOutputDirectory->MaximumLength = GET_MAX_LENGTH(BaseOutputDirectory); + + // + // Extract algorithm ID, hash function and mask function. + // + + EXTRACT_ID(Algorithm, ALGORITHM); + EXTRACT_ID(HashFunction, HASH_FUNCTION); + EXTRACT_ID(MaskFunction, MASK_FUNCTION); + + // + // Extract maximum concurrency. + // + + CurrentArg++; + String->Buffer = *ArgW++; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + if (FAILED(Rtl->RtlUnicodeStringToInteger(String, + 10, + MaximumConcurrency))) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + // + // Zero all flags (except for table create flags, as these may have + // default values) and table create parameters. + // + + KeysLoadFlags->AsULong = 0; + TableCompileFlags->AsULong = 0; + + for (; CurrentArg < NumberOfArguments; CurrentArg++, ArgW++) { + + String->Buffer = Arg = *ArgW; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + // + // If the argument doesn't start with two dashes, report it. + // + + InvalidPrefix = ( + (String->Length <= (sizeof(L'-') + sizeof(L'-'))) || + (!(*Arg++ == L'-' && *Arg++ == L'-')) + ); + + if (InvalidPrefix) { + goto InvalidArg; + } + + // + // Advance the buffer past the two dashes and update lengths + // accordingly. + // + + String->Buffer += 2; + String->Length -= (sizeof(WCHAR) * 2); + String->MaximumLength -= (sizeof(WCHAR) * 2); + + // + // Try each argument extraction routine for this argument; if it + // indicates an error, report it and break out of the loop. If it + // indicates it successfully extracted the argument (Result == S_OK), + // continue onto the next argument. Otherwise, verify it indicates + // that no argument was extracted (S_FALSE), then try the next routine. + // + +#define TRY_EXTRACT_ARG(Name) \ + Result = TryExtractArg##Name##Flags(Rtl, Allocator, String, Name##Flags); \ + if (FAILED(Result)) { \ + PH_ERROR(ExtractTableCreateArgs_TryExtractArg##Name##Flags, Result); \ + break; \ + } else if (Result == S_OK) { \ + continue; \ + } else { \ + ASSERT(Result == S_FALSE); \ + } + + TRY_EXTRACT_ARG(ContextTableCreate); + TRY_EXTRACT_ARG(KeysLoad); + TRY_EXTRACT_ARG(TableCreate); + TRY_EXTRACT_ARG(TableCompile); + + // + // If we get here, none of the previous extraction routines claimed the + // argument, so, provide the table create parameters extraction routine + // an opportunity to run. + // + + Result = TryExtractArgTableCreateParameters(Rtl, + String, + TableCreateParameters); + + if (Result == S_OK) { + continue; + } + + if (Result == PH_E_COMMANDLINE_ARG_MISSING_VALUE) { + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (FAILED(Result)) { + PH_ERROR(ExtractBulkCreateArgs_TryExtractTableCreateParams, Result); + break; + } + + ASSERT(Result == S_FALSE); + +InvalidArg: + + // + // If we get here, we don't recognize the argument. + // + + Result = PH_E_INVALID_COMMANDLINE_ARG; + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractTableCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + + if (SUCCEEDED(Result) && + *MaximumConcurrency == 0 && + ContextIocp->IocpConcurrency > 0) { + *MaximumConcurrency = ContextIocp->IocpConcurrency; + } + + if (SUCCEEDED(Result) && + ContextIocp->IocpConcurrency > 0 && + *MaximumConcurrency > ContextIocp->IocpConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + if (SUCCEEDED(Result)) { + + // + // Initialize the debugger flags from the table create flags, initialize + // the debugger context, then, maybe wait for a debugger attach. This + // is a no-op on Windows, or if no debugger has been requested. + // + + Flags.AsULong = 0; + Flags.WaitForGdb = (TableCreateFlags->WaitForGdb != FALSE); + Flags.WaitForCudaGdb = (TableCreateFlags->WaitForCudaGdb != FALSE); + Flags.UseGdbForHostDebugging = ( + TableCreateFlags->UseGdbForHostDebugging != FALSE + ); + + DebuggerContext = &Rtl->DebuggerContext; + DebuggerResult = InitializeDebuggerContext(DebuggerContext, &Flags); + + if (FAILED(DebuggerResult)) { + + PH_ERROR(InitializeDebuggerContext, DebuggerResult); + + // + // *Now* we can propagate the debugger result back as the primary + // result, which ensures the cleanup code below runs. + // + + Result = DebuggerResult; + + } else { + + // + // Debugger context was successfully initialized, so, maybe wait for + // a debugger to attach (depending on what flags were supplied). + // + + Result = MaybeWaitForDebuggerAttach(DebuggerContext); + if (FAILED(Result)) { + PH_ERROR(MaybeWaitForDebuggerAttach, Result); + } + + } + } + + // + // If we failed, clean up the table create parameters. If that fails, + // report the error, then replace our return value error code with that + // error code. + // + + if (FAILED(Result)) { + CleanupResult = CleanupTableCreateParameters(TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + Result = CleanupResult; + } + } + + return Result; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashContextTableCreate.c b/src/PerfectHash/PerfectHashContextTableCreate.c index 7333bd38..628c7420 100644 --- a/src/PerfectHash/PerfectHashContextTableCreate.c +++ b/src/PerfectHash/PerfectHashContextTableCreate.c @@ -44,6 +44,64 @@ HRESULT typedef PREPARE_TABLE_CREATE_CSV_FILE *PPREPARE_TABLE_CREATE_CSV_FILE; extern PREPARE_TABLE_CREATE_CSV_FILE PrepareTableCreateCsvFile; +static +VOID +PerfectHashLogTableCreateFailure( + _In_opt_ PCUNICODE_STRING KeysPath, + _In_ ULONG Stage, + _In_ HRESULT Result + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[1024]; + ULONG PathChars = 0; + + if (GetEnvironmentVariableW(L"PH_LOG_TABLE_CREATE_FAILURES", NULL, 0) == 0) { + return; + } + + if (ARGUMENT_PRESENT(KeysPath) && KeysPath->Buffer) { + PathChars = KeysPath->Length / sizeof(WCHAR); + } + + LogHandle = CreateFileW(L"PerfectHashTableCreateFailures.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Result=0x%08lX Path=%.*S\r\n", + Stage, + Result, + (int)PathChars, + (PathChars ? KeysPath->Buffer : L"")); + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(KeysPath); + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(Result); +#endif +} + // // Method implementations. // @@ -169,6 +227,7 @@ Return Value: ASSIGNED_MEMORY_COVERAGE EmptyCoverage; PASSIGNED_MEMORY_COVERAGE Coverage; BOOLEAN UnknownTableCreateResult = FALSE; + ULONG Stage = 0; // // Validate arguments. @@ -185,6 +244,10 @@ Return Value: VALIDATE_FLAGS(KeysLoad, KEYS_LOAD, ULong); VALIDATE_FLAGS(TableCompile, TABLE_COMPILE, ULong); + if (UseOverlappedIo(Context)) { + KeysLoadFlags.UseOverlappedIo = TRUE; + } + // // IsValidTableCreateFlags() returns a more specific error code than the // other validation routines invoked above (which would be converted into @@ -214,16 +277,19 @@ Return Value: return E_INVALIDARG; } else { + Stage = 1; Result = Context->Vtbl->SetBaseOutputDirectory(Context, BaseOutputDirectory); if (FAILED(Result)) { PH_ERROR(PerfectHashContextSetBaseOutputDirectory, Result); + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } Result = Context->Vtbl->GetBaseOutputDirectory(Context, &BaseOutputDir); if (FAILED(Result)) { PH_ERROR(PerfectHashContextGetBaseOutputDirectory, Result); + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } RELEASE(BaseOutputDir); @@ -250,16 +316,19 @@ Return Value: ZeroStruct(EmptyCoverage); MonitorLowMemory = (ContextTableCreateFlags.MonitorLowMemory != FALSE); + Stage = 2; Result = PerfectHashContextInitializeLowMemoryMonitor( Context, MonitorLowMemory ); if (FAILED(Result)) { PH_ERROR(PerfectHashContextInitializeLowMemoryMonitor, Result); + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } #ifdef PH_WINDOWS + Stage = 3; Result = PerfectHashContextTryPrepareCallbackTableValuesFile( Context, TableCreateFlags @@ -281,6 +350,7 @@ Return Value: NumberOfPages = TABLE_CREATE_CSV_ROW_BUFFER_NUMBER_OF_PAGES; } + Stage = 4; Result = Rtl->Vtbl->CreateBuffer(Rtl, &ProcessHandle, NumberOfPages, @@ -290,6 +360,7 @@ Return Value: if (FAILED(Result)) { Result = E_OUTOFMEMORY; + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } @@ -302,20 +373,25 @@ Return Value: // Get a reference to the stdout handle. // - if (!Context->OutputHandle) { - Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + Stage = 5; + if (!Silent) { if (!Context->OutputHandle) { - SYS_ERROR(GetStdHandle); - goto Error; + Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + if (!Context->OutputHandle || + Context->OutputHandle == INVALID_HANDLE_VALUE) { + Context->OutputHandle = NULL; + Silent = TRUE; + } } - } - OutputHandle = Context->OutputHandle; + OutputHandle = Context->OutputHandle; + } // // Prepare the .csv file if applicable. // + Stage = 6; if (TableCreateFlags.DisableCsvOutputFile == FALSE) { NumberOfRows = 1; Result = PrepareTableCreateCsvFile(Context, @@ -350,6 +426,7 @@ Return Value: // We pass this value to Keys->Vtbl->Load(). // + Stage = 7; Result = PerfectHashContextInitializeKeySize(&KeysLoadFlags, TableCreateParameters, &KeySizeInBytes); @@ -362,6 +439,7 @@ Return Value: // Create a keys instance. // + Stage = 8; Result = Context->Vtbl->CreateInstance(Context, NULL, &IID_PERFECT_HASH_KEYS, @@ -376,6 +454,7 @@ Return Value: // Load the keys. // + Stage = 9; Result = Keys->Vtbl->Load(Keys, &KeysLoadFlags, KeysPath, @@ -386,6 +465,7 @@ Return Value: goto Error; } + Stage = 10; Result = Keys->Vtbl->GetFlags(Keys, sizeof(KeysFlags), &KeysFlags); @@ -395,6 +475,7 @@ Return Value: goto Error; } + Stage = 11; Result = Keys->Vtbl->GetAddress(Keys, &KeysBaseAddress, &NumberOfKeys); @@ -429,6 +510,7 @@ Return Value: ASSERT(Table == NULL); + Stage = 12; Result = Context->Vtbl->CreateInstance(Context, NULL, &IID_PERFECT_HASH_TABLE, @@ -439,6 +521,7 @@ Return Value: goto Error; } + Stage = 13; Result = Table->Vtbl->Create(Table, Context, AlgorithmId, @@ -477,6 +560,7 @@ Return Value: if (!ContextTableCreateFlags.SkipTestAfterCreate) { + Stage = 14; Result = Table->Vtbl->Test(Table, Keys, FALSE); if (FAILED(Result)) { @@ -491,6 +575,7 @@ Return Value: if (ContextTableCreateFlags.Compile) { + Stage = 15; Result = Table->Vtbl->Compile(Table, &TableCompileFlags, CpuArchId); @@ -506,6 +591,7 @@ Return Value: // Write the .csv row if applicable. // + Stage = 16; if (TableCreateFlags.DisableCsvOutputFile != FALSE) { goto End; } @@ -550,6 +636,8 @@ Return Value: Result = E_UNEXPECTED; } + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); + // // Intentional follow-on to End. // @@ -1286,6 +1374,27 @@ Return Value: break; } + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractTableCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + if (SUCCEEDED(Result)) { // diff --git a/src/PerfectHash/PerfectHashDirectory.c b/src/PerfectHash/PerfectHashDirectory.c index fef5faab..495ed9b7 100644 --- a/src/PerfectHash/PerfectHashDirectory.c +++ b/src/PerfectHash/PerfectHashDirectory.c @@ -1725,10 +1725,10 @@ Return Value: goto Error; } - if (IsDirectoryClosed(Directory)) { - Result = PH_E_DIRECTORY_CLOSED; - goto Error; - } + // + // Allow removals even if the directory has been closed. This is required + // to break directory/file reference cycles during rundown. + // // // Argument validation complete. Add the file to the files list and diff --git a/src/PerfectHash/PerfectHashErrors.dbg b/src/PerfectHash/PerfectHashErrors.dbg index 16c3078d..f6a1a93a 100644 --- a/src/PerfectHash/PerfectHashErrors.dbg +++ b/src/PerfectHash/PerfectHashErrors.dbg @@ -22,6 +22,7 @@ struct { (HRESULT) PH_S_FUNCTION_HOOK_CALLBACK_DLL_INITIALIZED, (char*) "PH_S_FUNCTION_HOOK_CALLBACK_DLL_INITIALIZED", (HRESULT) PH_S_TABLE_CREATE_PARAMETER_NOT_FOUND, (char*) "PH_S_TABLE_CREATE_PARAMETER_NOT_FOUND", (HRESULT) PH_S_CU_KERNEL_RUNTIME_TARGET_REACHED, (char*) "PH_S_CU_KERNEL_RUNTIME_TARGET_REACHED", + (HRESULT) PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED, (char*) "PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED", (HRESULT) PH_I_CREATE_TABLE_ROUTINE_RECEIVED_SHUTDOWN_EVENT, (char*) "PH_I_CREATE_TABLE_ROUTINE_RECEIVED_SHUTDOWN_EVENT", (HRESULT) PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION, (char*) "PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION", (HRESULT) PH_I_MAXIMUM_NUMBER_OF_TABLE_RESIZE_EVENTS_REACHED, (char*) "PH_I_MAXIMUM_NUMBER_OF_TABLE_RESIZE_EVENTS_REACHED", @@ -177,6 +178,7 @@ struct { (HRESULT) PH_E_ERROR_DURING_PREPARE_BUILD_SOLUTION_BATCH_FILE, (char*) "PH_E_ERROR_DURING_PREPARE_BUILD_SOLUTION_BATCH_FILE", (HRESULT) PH_E_ERROR_DURING_PREPARE_C_HEADER_COMPILED_PERFECT_HASH_FILE, (char*) "PH_E_ERROR_DURING_PREPARE_C_HEADER_COMPILED_PERFECT_HASH_FILE", (HRESULT) PH_E_ERROR_DURING_PREPARE_VCPROPS_COMPILED_PERFECT_HASH_FILE, (char*) "PH_E_ERROR_DURING_PREPARE_VCPROPS_COMPILED_PERFECT_HASH_FILE", + (HRESULT) PH_E_SERVER_BULK_CREATE_FAILED, (char*) "PH_E_SERVER_BULK_CREATE_FAILED", (HRESULT) PH_E_ERROR_DURING_SAVE_VCPROJECT_DLL_FILE, (char*) "PH_E_ERROR_DURING_SAVE_VCPROJECT_DLL_FILE", (HRESULT) PH_E_ERROR_DURING_SAVE_C_SOURCE_SUPPORT_FILE, (char*) "PH_E_ERROR_DURING_SAVE_C_SOURCE_SUPPORT_FILE", (HRESULT) PH_E_ERROR_DURING_SAVE_C_SOURCE_TEST_FILE, (char*) "PH_E_ERROR_DURING_SAVE_C_SOURCE_TEST_FILE", diff --git a/src/PerfectHash/PerfectHashErrors.mc b/src/PerfectHash/PerfectHashErrors.mc index 56b8c19a..8a0d7ddd 100644 --- a/src/PerfectHash/PerfectHashErrors.mc +++ b/src/PerfectHash/PerfectHashErrors.mc @@ -136,6 +136,14 @@ Language=English CUDA kernel runtime target reached. . +MessageId=0x00f +Severity=Success +Facility=ITF +SymbolicName=PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED +Language=English +Bulk create request completed successfully. +. + ; ;//////////////////////////////////////////////////////////////////////////////// ;// PH_SEVERITY_INFORMATIONAL @@ -2338,8 +2346,16 @@ Language=English Error preparing CompiledPerfectHash.props file. . +MessageId=0x290 +Severity=Fail +Facility=ITF +SymbolicName=PH_E_SERVER_BULK_CREATE_FAILED +Language=English +Bulk create request failed. +. + ;// -;// Spare IDs: 0x290, 0x291. +;// Spare ID: 0x291. ;// MessageId=0x292 diff --git a/src/PerfectHash/PerfectHashErrors_English.bin b/src/PerfectHash/PerfectHashErrors_English.bin index 96762a9f..a319a2e7 100644 Binary files a/src/PerfectHash/PerfectHashErrors_English.bin and b/src/PerfectHash/PerfectHashErrors_English.bin differ diff --git a/src/PerfectHash/PerfectHashFile.c b/src/PerfectHash/PerfectHashFile.c index 120bcd56..ee4e8d0e 100644 --- a/src/PerfectHash/PerfectHashFile.c +++ b/src/PerfectHash/PerfectHashFile.c @@ -15,6 +15,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" // // Helper inline method for updating the file's FILE_INFO structure. @@ -41,6 +42,457 @@ PerfectHashFileUpdateFileInfo( return S_OK; } +static +VOID +PerfectHashFileGetIocpBufferPoolState( + _In_ PPERFECT_HASH_CONTEXT Context, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL **PoolsPointer, + _Out_ PULONG *PoolCountPointer, + _Outptr_ PGUARDED_LIST *BufferListPointer, + _Outptr_ PLIST_ENTRY *OversizeListPointer, + _Outptr_ PULONG *OversizeCountPointer, + _Outptr_ PSRWLOCK *PoolLockPointer, + _Out_ PUSHORT NumaNodePointer + ) +{ + PPERFECT_HASH_IOCP_NODE Node; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer) || + !ARGUMENT_PRESENT(BufferListPointer) || + !ARGUMENT_PRESENT(OversizeListPointer) || + !ARGUMENT_PRESENT(OversizeCountPointer) || + !ARGUMENT_PRESENT(PoolLockPointer) || + !ARGUMENT_PRESENT(NumaNodePointer)) { + return; + } + + Node = Context->IocpNode; + if (Node) { + *PoolsPointer = &Node->FileWorkBufferPools; + *PoolCountPointer = &Node->FileWorkBufferPoolCount; + *BufferListPointer = Node->FileWorkBufferList; + *OversizeListPointer = &Node->FileWorkOversizePools; + *OversizeCountPointer = &Node->FileWorkOversizePoolCount; + *PoolLockPointer = &Node->FileWorkBufferPoolLock; + *NumaNodePointer = (USHORT)Node->NodeId; + } else { + *PoolsPointer = &Context->FileWorkBufferPools; + *PoolCountPointer = &Context->FileWorkBufferPoolCount; + *BufferListPointer = Context->FileWorkBufferList; + *OversizeListPointer = &Context->FileWorkOversizePools; + *OversizeCountPointer = &Context->FileWorkOversizePoolCount; + *PoolLockPointer = &Context->Lock; + *NumaNodePointer = 0; + } +} + +static +HRESULT +PerfectHashFileEnsureIocpBufferPools( + _In_ PPERFECT_HASH_CONTEXT Context, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolsPointer, + _Out_ PULONG PoolCountPointer + ) +{ + HRESULT Result; + ULONG Flags; + ULONG PoolIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PerfectHashFileGetIocpBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!PoolArrayPointer || !PoolCountField || !BufferList) { + return E_UNEXPECTED; + } + + Pools = *PoolArrayPointer; + if (!Pools) { + AcquireSRWLockExclusive(PoolLock); + Pools = *PoolArrayPointer; + if (!Pools) { + PoolCount = PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT; + Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + PoolCount, + sizeof(*Pools) + ); + if (!Pools) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; + } + + Flags = UseIocpBufferGuardPages(Context) ? + PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES : + 0; + for (PoolIndex = 0; PoolIndex < PoolCount; PoolIndex++) { + PayloadSize = + PerfectHashIocpBufferGetPayloadSizeFromClassIndex( + (LONG)PoolIndex + ); + + Result = PerfectHashIocpBufferPoolInitialize( + Rtl, + &Pools[PoolIndex], + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList + ); + + if (FAILED(Result)) { + ReleaseSRWLockExclusive(PoolLock); + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Pools + ); + return Result; + } + } + + *PoolArrayPointer = Pools; + *PoolCountField = PoolCount; + } + ReleaseSRWLockExclusive(PoolLock); + } + + *PoolsPointer = Pools; + *PoolCountPointer = *PoolCountField; + return S_OK; +} + +static +HRESULT +PerfectHashFileGetOversizeIocpBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG PayloadSize, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + ULONG Flags; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PLIST_ENTRY Entry; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PerfectHashFileGetIocpBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!OversizeList || !OversizeCount || !PoolLock) { + return E_UNEXPECTED; + } + + AcquireSRWLockExclusive(PoolLock); + + Entry = OversizeList->Flink; + while (Entry != OversizeList) { + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + if (Pool->PayloadSize == PayloadSize) { + ReleaseSRWLockExclusive(PoolLock); + *PoolPointer = Pool; + return S_OK; + } + + Entry = Entry->Flink; + } + + Pool = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*Pool) + ); + if (!Pool) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; + } + + Flags = PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE; + if (UseIocpBufferGuardPages(Context)) { + Flags |= PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES; + } + + Result = PerfectHashIocpBufferPoolInitialize(Rtl, + Pool, + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList); + + if (FAILED(Result)) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + ReleaseSRWLockExclusive(PoolLock); + return Result; + } + + InsertTailList(OversizeList, &Pool->ListEntry); + (*OversizeCount)++; + ReleaseSRWLockExclusive(PoolLock); + + *PoolPointer = Pool; + return S_OK; +} + +static +HRESULT +PerfectHashFileGetIocpBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + LONG ClassIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Rtl = Context->Rtl; + if (!Rtl) { + return E_UNEXPECTED; + } + + ClassIndex = PerfectHashIocpBufferGetClassIndex(Rtl, + RequiredPayloadBytes); + if (ClassIndex >= 0) { + Result = PerfectHashFileEnsureIocpBufferPools(Context, + &Pools, + &PoolCount); + if (FAILED(Result)) { + return Result; + } + + if ((ULONG)ClassIndex >= PoolCount) { + return E_INVALIDARG; + } + + *PoolPointer = &Pools[ClassIndex]; + return S_OK; + } + + PayloadSize = Rtl->RoundUpPowerOfTwo64(RequiredPayloadBytes); + if (PayloadSize == 0) { + return E_INVALIDARG; + } + + return PerfectHashFileGetOversizeIocpBufferPool(Context, + PayloadSize, + PoolPointer); +} + +static +HRESULT +PerfectHashFileAcquireIocpBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + HRESULT Result; + PRTL Rtl; + PALLOCATOR Allocator; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolPointer) || + !ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; + } + + Rtl = Context->Rtl; + Allocator = Context->Allocator; + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + Result = PerfectHashFileGetIocpBufferPool(Context, + RequiredPayloadBytes, + &Pool); + if (FAILED(Result)) { + return Result; + } + + Result = PerfectHashIocpBufferPoolAcquire(Rtl, + Allocator, + Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + *PoolPointer = Pool; + *BufferPointer = Buffer; + return S_OK; +} + +static +HRESULT +PerfectHashFileReadOverlapped( + _In_ PPERFECT_HASH_FILE File, + _In_ ULONGLONG FileSize + ) +{ + HRESULT Result; + BOOL Success; + DWORD BytesRead; + ULONG LastError; + ULONGLONG Offset; + ULONGLONG Remaining; + PBYTE Dest; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(File)) { + return E_POINTER; + } + + if (FileSize == 0) { + return PH_E_FILE_EMPTY; + } + + TlsContext = PerfectHashTlsGetContext(); + if (!TlsContext || !TlsContext->Context) { + return PH_E_NO_TLS_CONTEXT_SET; + } + + Context = TlsContext->Context; + Result = PerfectHashFileAcquireIocpBuffer(Context, + FileSize, + &Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + File->IocpBufferPool = Pool; + File->IocpBuffer = Buffer; + File->BaseAddress = PerfectHashIocpBufferPayload(Buffer); + + Offset = 0; + Remaining = FileSize; + Dest = (PBYTE)File->BaseAddress; + + while (Remaining) { + DWORD Chunk; + OVERLAPPED Overlapped; + + Chunk = (Remaining > MAXDWORD) ? MAXDWORD : (DWORD)Remaining; + ZeroStructInline(Overlapped); + Overlapped.Offset = (DWORD)Offset; + Overlapped.OffsetHigh = (DWORD)(Offset >> 32); + + BytesRead = 0; + Success = ReadFile(File->FileHandle, + Dest, + Chunk, + &BytesRead, + &Overlapped); + if (!Success) { + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + Success = GetOverlappedResult(File->FileHandle, + &Overlapped, + &BytesRead, + TRUE); + if (!Success) { + SYS_ERROR(GetOverlappedResult); + Result = PH_E_SYSTEM_CALL_FAILED; + break; + } + } else { + SYS_ERROR(ReadFile); + Result = PH_E_SYSTEM_CALL_FAILED; + break; + } + } + + if (BytesRead != Chunk) { + Result = PH_E_INVARIANT_CHECK_FAILED; + break; + } + + Remaining -= BytesRead; + Offset += BytesRead; + Dest += BytesRead; + } + + if (FAILED(Result)) { + PerfectHashIocpBufferPoolRelease(Pool, Buffer); + File->IocpBufferPool = NULL; + File->IocpBuffer = NULL; + File->BaseAddress = NULL; + } + + return Result; +} + PERFECT_HASH_FILE_INITIALIZE PerfectHashFileInitialize; @@ -356,6 +808,8 @@ Return Value: } File->State.IsReadOnly = TRUE; + File->FileCreateFlags.AsULong = 0; + File->FileCreateFlags.SkipMapping = FileLoadFlags.SkipMapping; // // Add a reference to the source path. @@ -420,15 +874,34 @@ Return Value: EndOfFilePointer->QuadPart = File->FileInfo.EndOfFile.QuadPart; } - // - // Map the file into memory. - // + if (FileLoadFlags.SkipMapping) { - Result = File->Vtbl->Map(File); + // + // Load the file data into an overlapped buffer. + // - if (FAILED(Result)) { - PH_ERROR(PerfectHashFileMap, Result); - goto Error; + Result = PerfectHashFileReadOverlapped( + File, + (ULONGLONG)File->FileInfo.EndOfFile.QuadPart + ); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileReadOverlapped, Result); + goto Error; + } + + } else { + + // + // Map the file into memory. + // + + Result = File->Vtbl->Map(File); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileMap, Result); + goto Error; + } } // @@ -629,6 +1102,7 @@ Return Value: } File->State.IsReadOnly = FALSE; + File->FileCreateFlags.AsULong = FileCreateFlags.AsULong; // // Add a reference to the source path. @@ -700,7 +1174,19 @@ Return Value: EndOfFile.QuadPart = File->FileInfo.EndOfFile.QuadPart; - if (!FileCreateFlags.EndOfFileIsExtensionSizeIfFileExists) { + if (EndOfFile.QuadPart == 0) { + + // + // Existing file is empty; treat it like a new file and extend + // it to the requested size. This avoids invalid end-of-file + // failures when mapping zero-length streams. + // + + ASSERT(EndOfFilePointer->QuadPart > 0); + + EndOfFile.QuadPart = EndOfFilePointer->QuadPart; + + } else if (!FileCreateFlags.EndOfFileIsExtensionSizeIfFileExists) { DoTruncate = FALSE; @@ -760,11 +1246,12 @@ Return Value: // Map the file into memory. // - Result = File->Vtbl->Map(File); - - if (FAILED(Result)) { - PH_ERROR(PerfectHashFileMap, Result); - goto Error; + if (!FileCreateFlags.SkipMapping) { + Result = File->Vtbl->Map(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileMap, Result); + goto Error; + } } // @@ -915,6 +1402,14 @@ Return Value: } } + if (File->IocpBufferPool && File->IocpBuffer) { + PerfectHashIocpBufferPoolRelease(File->IocpBufferPool, + File->IocpBuffer); + File->IocpBufferPool = NULL; + File->IocpBuffer = NULL; + File->BaseAddress = NULL; + } + // // If the end of file is non-zero, truncate the file. Otherwise, if it's // zero, treat this as an indication that the file should be deleted. @@ -1319,6 +1814,11 @@ Return Value: return PH_E_FILE_NOT_OPEN; } + if (File->FileCreateFlags.SkipMapping) { + SetFileUnmapped(File); + return S_OK; + } + if (IsFileUnmapped(File)) { return PH_E_FILE_ALREADY_UNMAPPED; } @@ -1792,6 +2292,10 @@ Return Value: File->NumberOfBytesWritten.QuadPart = 0; + if (File->FileCreateFlags.SkipMapping) { + goto End; + } + Result = File->Vtbl->Map(File); if (FAILED(Result)) { PH_ERROR(PerfectHashFileExtend, Result); diff --git a/src/PerfectHash/PerfectHashFile.h b/src/PerfectHash/PerfectHashFile.h index cdc3e9f7..7ab8120f 100644 --- a/src/PerfectHash/PerfectHashFile.h +++ b/src/PerfectHash/PerfectHashFile.h @@ -21,6 +21,11 @@ Module Name: #include "stdafx.h" +typedef struct _PERFECT_HASH_IOCP_BUFFER_POOL PERFECT_HASH_IOCP_BUFFER_POOL; +typedef struct _PERFECT_HASH_IOCP_BUFFER PERFECT_HASH_IOCP_BUFFER; +typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; +typedef PERFECT_HASH_IOCP_BUFFER *PPERFECT_HASH_IOCP_BUFFER; + // // Private vtbl methods. // @@ -317,6 +322,13 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_FILE { PVOID MappedAddress; + // + // Optional IOCP buffer state used when SkipMapping is active. + // + + PPERFECT_HASH_IOCP_BUFFER_POOL IocpBufferPool; + PPERFECT_HASH_IOCP_BUFFER IocpBuffer; + // // File info for the current file. Updated whenever any routine mutates // end of file or allocation size. diff --git a/src/PerfectHash/PerfectHashIocpBufferPool.c b/src/PerfectHash/PerfectHashIocpBufferPool.c new file mode 100644 index 00000000..b5f9757b --- /dev/null +++ b/src/PerfectHash/PerfectHashIocpBufferPool.c @@ -0,0 +1,240 @@ +/*++ + +Copyright (c) 2024-2026 Trent Nelson + +Module Name: + + PerfectHashIocpBufferPool.c + +Abstract: + + This module implements IOCP buffer pool routines used by the IOCP backend + for overlapped file I/O. Buffers are allocated on demand from size-class + pools and returned to SLIST free lists for reuse. + +--*/ + +#include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" + +PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE PerfectHashIocpBufferPoolInitialize; +PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE PerfectHashIocpBufferPoolAcquire; +PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE PerfectHashIocpBufferPoolRelease; +PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN PerfectHashIocpBufferPoolRundown; + +static +_Must_inspect_result_ +HRESULT +PerfectHashIocpBufferPoolAllocate( + _In_ PRTL Rtl, + _In_opt_ PALLOCATOR Allocator, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + ULONG Pages; + HRESULT Result; + HANDLE ProcessHandle; + PVOID BaseAddress; + ULONGLONG AllocationSize; + ULONGLONG UsableBufferSize; + ULONGLONG PayloadSize; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + UNREFERENCED_PARAMETER(Allocator); + + if (!ARGUMENT_PRESENT(Rtl) || + !ARGUMENT_PRESENT(Pool) || + !ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; + } + + *BufferPointer = NULL; + BaseAddress = NULL; + UsableBufferSize = 0; + PayloadSize = Pool->PayloadSize; + + ProcessHandle = Pool->ProcessHandle; + if (!ProcessHandle) { + ProcessHandle = GetCurrentProcess(); + } + + AllocationSize = Pool->AllocationSize; + if (AllocationSize == 0) { + return E_INVALIDARG; + } + + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES) { + Pages = (ULONG)BYTES_TO_PAGES(AllocationSize); + if (Pages == 0) { + Pages = 1; + } + + Result = Rtl->Vtbl->CreateBuffer(Rtl, + &ProcessHandle, + Pages, + NULL, + &UsableBufferSize, + &BaseAddress); + if (FAILED(Result)) { + return Result; + } + + AllocationSize = ( + ((ULONGLONG)Pages + 1) * (ULONGLONG)PAGE_SIZE + ); + if (UsableBufferSize <= PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE) { + Rtl->Vtbl->DestroyBuffer(Rtl, + ProcessHandle, + &BaseAddress, + AllocationSize); + return E_FAIL; + } + + PayloadSize = UsableBufferSize - + (ULONGLONG)PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + + } else { + + BaseAddress = VirtualAllocEx(ProcessHandle, + NULL, + AllocationSize, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE); + if (!BaseAddress) { + SYS_ERROR(VirtualAllocEx); + return E_OUTOFMEMORY; + } + } + + Buffer = (PPERFECT_HASH_IOCP_BUFFER)BaseAddress; + Rtl->RtlZeroMemory(Buffer, PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE); + Buffer->SizeOfStruct = sizeof(*Buffer); + Buffer->PayloadOffset = PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + Buffer->PayloadSize = PayloadSize; + Buffer->AllocationSize = AllocationSize; + Buffer->BytesWritten = 0; + Buffer->OwnerPool = Pool; + Buffer->Flags = 0; + + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES) { + Buffer->Flags |= PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES; + } + + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE) { + Buffer->Flags |= PERFECT_HASH_IOCP_BUFFER_FLAG_OVERSIZE; + } + + if (Pool->BufferList) { + Pool->BufferList->Vtbl->InsertTail(Pool->BufferList, + &Buffer->ListEntry); + } + + *BufferPointer = Buffer; + return S_OK; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashIocpBufferPoolInitialize( + PRTL Rtl, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + ULONGLONG PayloadSize, + USHORT NumaNode, + ULONG Flags, + HANDLE ProcessHandle, + PGUARDED_LIST BufferList + ) +{ + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(Pool)) { + return E_POINTER; + } + + if (!PayloadSize) { + return E_INVALIDARG; + } + + Rtl->RtlZeroMemory(Pool, sizeof(*Pool)); + InitializeSListHead(&Pool->FreeList); + InitializeListHead(&Pool->ListEntry); + + Pool->SizeOfStruct = sizeof(*Pool); + Pool->PayloadSize = PayloadSize; + Pool->AllocationSize = PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE + PayloadSize; + Pool->Flags = Flags | PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_INITIALIZED; + Pool->NumaNode = NumaNode; + Pool->ProcessHandle = ProcessHandle; + Pool->BufferList = BufferList; + + return S_OK; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashIocpBufferPoolAcquire( + PRTL Rtl, + PALLOCATOR Allocator, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + PSLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; + } + + *BufferPointer = NULL; + + Entry = InterlockedPopEntrySList(&Pool->FreeList); + if (Entry) { + Buffer = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER, + FreeListEntry); + *BufferPointer = Buffer; + return S_OK; + } + + return PerfectHashIocpBufferPoolAllocate(Rtl, + Allocator, + Pool, + BufferPointer); +} + +_Use_decl_annotations_ +VOID +PerfectHashIocpBufferPoolRelease( + PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + Buffer->BytesWritten = 0; + InterlockedPushEntrySList(&Pool->FreeList, &Buffer->FreeListEntry); +} + +_Use_decl_annotations_ +VOID +PerfectHashIocpBufferPoolRundown( + PRTL Rtl, + PALLOCATOR Allocator, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ) +{ + UNREFERENCED_PARAMETER(Rtl); + UNREFERENCED_PARAMETER(Allocator); + + if (!ARGUMENT_PRESENT(Pool)) { + return; + } + + InitializeSListHead(&Pool->FreeList); + InitializeListHead(&Pool->ListEntry); + ZeroMemory(Pool, sizeof(*Pool)); +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashIocpBufferPool.h b/src/PerfectHash/PerfectHashIocpBufferPool.h new file mode 100644 index 00000000..6a17f87d --- /dev/null +++ b/src/PerfectHash/PerfectHashIocpBufferPool.h @@ -0,0 +1,226 @@ +/*++ + +Copyright (c) 2024-2026 Trent Nelson + +Module Name: + + PerfectHashIocpBufferPool.h + +Abstract: + + This is the private header file for IOCP buffer pool support. It defines + buffer header and pool structures used by the IOCP backend for overlapped + file I/O, along with helper routines for pool management. + +--*/ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifndef ALIGN_UP +#define ALIGN_UP(Address, Alignment) ( \ + (((ULONG_PTR)(Address)) + (Alignment) - 1) & \ + ~((ULONG_PTR)(Alignment) - 1) \ +) +#endif + +typedef struct _RTL RTL; +typedef RTL *PRTL; + +// +// IOCP buffer size-class configuration (payload sizes in bytes). +// + +#define PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT 12 // 4KB +#define PERFECT_HASH_IOCP_BUFFER_MAX_SHIFT 24 // 16MB + +#define PERFECT_HASH_IOCP_BUFFER_MIN_SIZE \ + (1ULL << PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT) + +#define PERFECT_HASH_IOCP_BUFFER_MAX_SIZE \ + (1ULL << PERFECT_HASH_IOCP_BUFFER_MAX_SHIFT) + +#define PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT ( \ + PERFECT_HASH_IOCP_BUFFER_MAX_SHIFT - \ + PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT + \ + 1 \ +) + +// +// Buffer flags. +// + +#define PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES 0x00000001 +#define PERFECT_HASH_IOCP_BUFFER_FLAG_OVERSIZE 0x00000002 + +// +// Pool flags. +// + +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_INITIALIZED 0x00000001 +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES 0x00000002 +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE 0x00000004 + +// +// IOCP buffer header used by IOCP file work. Payload begins at PayloadOffset. +// + +typedef struct DECLSPEC_ALIGN(16) _PERFECT_HASH_IOCP_BUFFER { + SLIST_ENTRY FreeListEntry; + LIST_ENTRY ListEntry; + struct _PERFECT_HASH_IOCP_BUFFER_POOL *OwnerPool; + ULONGLONG BytesWritten; + ULONGLONG PayloadSize; + ULONGLONG AllocationSize; + ULONG SizeOfStruct; + ULONG PayloadOffset; + ULONG Flags; + ULONG Padding1; +} PERFECT_HASH_IOCP_BUFFER; +typedef PERFECT_HASH_IOCP_BUFFER *PPERFECT_HASH_IOCP_BUFFER; + +#define PERFECT_HASH_IOCP_BUFFER_HEADER_ALIGNMENT MEMORY_ALLOCATION_ALIGNMENT + +#define PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE ( \ + (ULONG)ALIGN_UP(sizeof(PERFECT_HASH_IOCP_BUFFER), \ + PERFECT_HASH_IOCP_BUFFER_HEADER_ALIGNMENT) \ +) + +FORCEINLINE +PVOID +PerfectHashIocpBufferPayload( + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + return (PVOID)((PCHAR)Buffer + Buffer->PayloadOffset); +} + +FORCEINLINE +LONG +PerfectHashIocpBufferGetClassIndex( + _In_ PRTL Rtl, + _In_ ULONGLONG PayloadBytes + ) +{ + ULONGLONG Rounded; + ULONG TrailingZeros; + + if (!ARGUMENT_PRESENT(Rtl)) { + return -1; + } + + if (PayloadBytes <= PERFECT_HASH_IOCP_BUFFER_MIN_SIZE) { + return 0; + } + + Rounded = Rtl->RoundUpPowerOfTwo64(PayloadBytes); + if (Rounded == 0) { + return -1; + } + + if (Rounded > PERFECT_HASH_IOCP_BUFFER_MAX_SIZE) { + return -1; + } + + TrailingZeros = (ULONG)Rtl->TrailingZeros64(Rounded); + return (LONG)TrailingZeros - PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT; +} + +FORCEINLINE +ULONGLONG +PerfectHashIocpBufferGetPayloadSizeFromClassIndex( + _In_ LONG ClassIndex + ) +{ + ULONGLONG Shift; + + if (ClassIndex < 0) { + return 0; + } + + Shift = (ULONGLONG)ClassIndex + PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT; + return (1ULL << Shift); +} + +// +// IOCP buffer pool. +// + +typedef struct DECLSPEC_ALIGN(16) _PERFECT_HASH_IOCP_BUFFER_POOL { + LIST_ENTRY ListEntry; + SLIST_HEADER FreeList; + PGUARDED_LIST BufferList; + HANDLE ProcessHandle; + ULONGLONG PayloadSize; + ULONGLONG AllocationSize; + ULONG SizeOfStruct; + ULONG Flags; + ULONG Padding1; + USHORT NumaNode; + USHORT Padding2; +} PERFECT_HASH_IOCP_BUFFER_POOL; +typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE)( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ ULONGLONG PayloadSize, + _In_ USHORT NumaNode, + _In_ ULONG Flags, + _In_opt_ HANDLE ProcessHandle, + _In_opt_ PGUARDED_LIST BufferList + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE + *PPERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE)( + _In_ PRTL Rtl, + _In_opt_ PALLOCATOR Allocator, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE + *PPERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE; + +typedef +VOID +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE)( + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE + *PPERFECT_HASH_IOCP_BUFFER_POOL_RELEASE; + +typedef +VOID +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN)( + _In_ PRTL Rtl, + _In_opt_ PALLOCATOR Allocator, + _Inout_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN + *PPERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN; + +extern PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE + PerfectHashIocpBufferPoolInitialize; +extern PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE + PerfectHashIocpBufferPoolAcquire; +extern PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE + PerfectHashIocpBufferPoolRelease; +extern PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN + PerfectHashIocpBufferPoolRundown; + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/src/PerfectHash/PerfectHashKeysLoad.c b/src/PerfectHash/PerfectHashKeysLoad.c index df9b9e4d..5aca7bfd 100644 --- a/src/PerfectHash/PerfectHashKeysLoad.c +++ b/src/PerfectHash/PerfectHashKeysLoad.c @@ -231,6 +231,10 @@ Return Value: KeysLoadFlags.TryLargePagesForKeysData ); + if (KeysLoadFlags.UseOverlappedIo) { + FileLoadFlags.SkipMapping = TRUE; + } + Result = File->Vtbl->Load(File, Path, &EndOfFile, &FileLoadFlags); if (FAILED(Result)) { diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c new file mode 100644 index 00000000..e27e4872 --- /dev/null +++ b/src/PerfectHash/PerfectHashServer.c @@ -0,0 +1,3507 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + PerfectHashServer.c + +Abstract: + + This module implements the PERFECT_HASH_SERVER component. The initial + implementation provides configuration plumbing and scaffolding for the + IOCP runtime and request processing. + +--*/ + +#include "stdafx.h" +#include "PerfectHashTls.h" +#include +#include + +#ifdef PH_WINDOWS + +static const UNICODE_STRING PerfectHashServerDefaultPipeName = + RTL_CONSTANT_STRING(L"\\\\.\\pipe\\PerfectHashServer"); + +typedef enum _PERFECT_HASH_SERVER_PIPE_STATE { + PerfectHashServerPipeStateInvalid = 0, + PerfectHashServerPipeStateAccepting, + PerfectHashServerPipeStateReadingHeader, + PerfectHashServerPipeStateReadingPayload, + PerfectHashServerPipeStateWritingResponseHeader, + PerfectHashServerPipeStateWritingResponsePayload +} PERFECT_HASH_SERVER_PIPE_STATE; + +typedef struct _PERFECT_HASH_SERVER_PIPE { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_IOCP_NODE Node; + HANDLE Pipe; + PERFECT_HASH_SERVER_PIPE_STATE State; + ULONG Flags; + ULONG BytesTransferred; + ULONG PayloadLength; + BOOLEAN ShutdownAfterSend; + UCHAR Padding1[7]; + PERFECT_HASH_SERVER_REQUEST_HEADER RequestHeader; + PERFECT_HASH_SERVER_RESPONSE_HEADER ResponseHeader; + PVOID PayloadBuffer; + ULONG PayloadBufferSize; + ULONG Padding2; +} PERFECT_HASH_SERVER_PIPE; +typedef PERFECT_HASH_SERVER_PIPE *PPERFECT_HASH_SERVER_PIPE; + +typedef struct _PERFECT_HASH_SERVER_BULK_REQUEST { + ULONG SizeOfStruct; + volatile LONG OutstandingWorkItems; + volatile LONG FailedWorkItems; + volatile LONG PendingNodes; + volatile LONG CompletionSignaled; + ULONG TotalWorkItems; + ULONG PerFileMaximumConcurrency; + ULONG Padding1; + PPERFECT_HASH_SERVER Server; + LPWSTR *ArgvW; + PWSTR CommandLineBuffer; + PERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags; + PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags; + ULONG Padding2; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags; + ULONG Padding3; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters; + PERFECT_HASH_ALGORITHM_ID AlgorithmId; + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId; + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId; + ULONG Padding4; + UNICODE_STRING BaseOutputDirectory; + PWSTR BaseOutputDirectoryBuffer; + HANDLE CompletionEvent; + HANDLE ResultMappingHandle; + PPERFECT_HASH_SERVER_BULK_RESULT ResultMapping; + volatile LONG FirstFailure; + volatile LONG DispatchComplete; + PPERFECT_HASH_IOCP_NODE Nodes; + ULONG NodeCount; + ULONG NextNodeIndex; + PLONG NodeOutstandingCounts; +} PERFECT_HASH_SERVER_BULK_REQUEST; +typedef PERFECT_HASH_SERVER_BULK_REQUEST *PPERFECT_HASH_SERVER_BULK_REQUEST; + +typedef struct _PERFECT_HASH_SERVER_BULK_WORK_ITEM { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_SERVER_BULK_REQUEST Request; + PPERFECT_HASH_IOCP_NODE Node; + ULONG NodeIndex; + ULONG Padding1; + UNICODE_STRING KeysPath; + PWSTR KeysPathBuffer; + ULONG LastError; + ULONG Padding2; +} PERFECT_HASH_SERVER_BULK_WORK_ITEM; +typedef PERFECT_HASH_SERVER_BULK_WORK_ITEM *PPERFECT_HASH_SERVER_BULK_WORK_ITEM; + +static +HRESULT +PerfectHashServerCreatePipes( + _In_ PPERFECT_HASH_SERVER Server + ); + +static +VOID +PerfectHashServerDestroyPipes( + _In_ PPERFECT_HASH_SERVER Server + ); + +static +HRESULT +PerfectHashServerIssueConnect( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueReadHeader( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueReadPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueWriteResponseHeader( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueWriteResponsePayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +VOID +PerfectHashServerResetPipe( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ BOOLEAN Reconnect + ); + +static +HRESULT +PerfectHashServerDispatchRequest( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerDispatchBulkCreateDirectoryRequest( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLine, + _Out_ PBOOLEAN ArgvWOwned + ); + +static +HRESULT +PerfectHashServerDispatchTableCreateRequest( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLine + ); + +static +HRESULT +PerfectHashServerPrepareErrorPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ HRESULT Error + ); + +static +HRESULT +PerfectHashServerPreparePingPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerPrepareBulkCreateDirectoryPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ HANDLE EventHandle, + _In_ HANDLE ResultHandle + ); + +static +HRESULT +PerfectHashServerEnqueueBulkRequest( + _In_ PPERFECT_HASH_SERVER_BULK_REQUEST Request, + _In_ PUNICODE_STRING KeysDirectory + ); + +static +VOID +PerfectHashServerCompleteBulkRequest( + _In_ PPERFECT_HASH_SERVER_BULK_REQUEST Request + ); + +static +HRESULT +PerfectHashServerBulkCreateWorkItemCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ); + +static +VOID +PerfectHashServerLogBulkCreateException( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_ struct _EXCEPTION_POINTERS *ExceptionPointers + ); + +static +VOID +PerfectHashServerLogBulkCreateCounts( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_opt_ PPERFECT_HASH_SERVER_BULK_REQUEST Request, + _In_ LONG Outstanding + ); + +static +LONG +PerfectHashServerBulkCreateExceptionFilter( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_ struct _EXCEPTION_POINTERS *ExceptionPointers + ); + +static +HRESULT +PerfectHashServerIocpCompletionCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ); + +#endif + +PERFECT_HASH_SERVER_INITIALIZE PerfectHashServerInitialize; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerInitialize( + PPERFECT_HASH_SERVER Server + ) +/*++ + +Routine Description: + + Initializes a PERFECT_HASH_SERVER instance. This routine creates the + Rtl, Allocator, and IOCP context components and initializes default + configuration values. + +Arguments: + + Server - Supplies a pointer to a PERFECT_HASH_SERVER structure for which + initialization is to be performed. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + HRESULT Result = E_UNEXPECTED; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_RTL, + &Server->Rtl); + if (FAILED(Result)) { + return Result; + } + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_ALLOCATOR, + &Server->Allocator); + if (FAILED(Result)) { + return Result; + } + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_CONTEXT_IOCP, + &Server->ContextIocp); + if (FAILED(Result)) { + return Result; + } + + ContextIocp = Server->ContextIocp; + + Server->IocpConcurrency = ContextIocp->IocpConcurrency; + Server->MaxWorkerThreads = ContextIocp->MaxWorkerThreads; + Server->NumaNodeMask = ContextIocp->NumaNodeMask; + Server->NumaNodeCount = ContextIocp->NumaNodeCount; + + Server->StartedEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!Server->StartedEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + Server->ShutdownEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!Server->ShutdownEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + Server->Endpoint = PerfectHashServerDefaultPipeName; + Server->Flags.LocalOnly = TRUE; + Server->Flags.EndpointAllocated = FALSE; + Server->Flags.Verbose = FALSE; + Server->Flags.NoFileIo = FALSE; + Server->Flags.IocpBufferGuardPages = FALSE; + Server->State.Initialized = TRUE; + + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_RUNDOWN PerfectHashServerRundown; + +_Use_decl_annotations_ +VOID +PerfectHashServerRundown( + PPERFECT_HASH_SERVER Server + ) +/*++ + +Routine Description: + + Releases resources associated with a PERFECT_HASH_SERVER instance. + +Arguments: + + Server - Supplies a pointer to a PERFECT_HASH_SERVER structure for which + rundown is to be performed. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(Server)) { + return; + } + + Server->State.Stopping = TRUE; + + if (Server->ContextIocp) { +#ifdef PH_WINDOWS + PerfectHashServerDestroyPipes(Server); +#endif + PerfectHashContextIocpStop(Server->ContextIocp); + } + +#ifdef PH_WINDOWS + if (Server->ShutdownEvent) { + CloseHandle(Server->ShutdownEvent); + Server->ShutdownEvent = NULL; + } + if (Server->StartedEvent) { + CloseHandle(Server->StartedEvent); + Server->StartedEvent = NULL; + } +#endif + + if (Server->Flags.EndpointAllocated && Server->Endpoint.Buffer) { + if (Server->Allocator) { + Server->Allocator->Vtbl->FreePointer( + Server->Allocator, + (PVOID *)&Server->Endpoint.Buffer + ); + } + Server->Endpoint.Length = 0; + Server->Endpoint.MaximumLength = 0; + Server->Flags.EndpointAllocated = FALSE; + } + + RELEASE(Server->ContextIocp); + RELEASE(Server->Allocator); + RELEASE(Server->Rtl); +} + +PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY + PerfectHashServerSetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetMaximumConcurrency( + PPERFECT_HASH_SERVER Server, + ULONG MaximumConcurrency + ) +{ + HRESULT Result = E_UNEXPECTED; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (MaximumConcurrency == 0) { + return E_INVALIDARG; + } + + if (Server->ContextIocp) { + Result = Server->ContextIocp->Vtbl->SetMaximumConcurrency( + Server->ContextIocp, + MaximumConcurrency + ); + if (FAILED(Result)) { + return Result; + } + } + + Server->IocpConcurrency = MaximumConcurrency; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY + PerfectHashServerGetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetMaximumConcurrency( + PPERFECT_HASH_SERVER Server, + PULONG MaximumConcurrency + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + *MaximumConcurrency = Server->IocpConcurrency; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS + PerfectHashServerSetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetMaximumThreads( + PPERFECT_HASH_SERVER Server, + ULONG MaximumThreads + ) +{ + HRESULT Result = E_UNEXPECTED; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (MaximumThreads == 0) { + return E_INVALIDARG; + } + + if (Server->ContextIocp) { + Result = Server->ContextIocp->Vtbl->SetMaximumThreads( + Server->ContextIocp, + MaximumThreads + ); + if (FAILED(Result)) { + return Result; + } + } + + Server->MaxWorkerThreads = MaximumThreads; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS + PerfectHashServerGetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetMaximumThreads( + PPERFECT_HASH_SERVER Server, + PULONG MaximumThreads + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumThreads)) { + return E_POINTER; + } + + *MaximumThreads = Server->MaxWorkerThreads; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK PerfectHashServerSetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetNumaNodeMask( + PPERFECT_HASH_SERVER Server, + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->ContextIocp) { + Result = Server->ContextIocp->Vtbl->SetNumaNodeMask( + Server->ContextIocp, + NumaNodeMask + ); + if (FAILED(Result)) { + return Result; + } + } + + Server->NumaNodeMask = NumaNodeMask; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK PerfectHashServerGetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetNumaNodeMask( + PPERFECT_HASH_SERVER Server, + PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(NumaNodeMask)) { + return E_POINTER; + } + + *NumaNodeMask = Server->NumaNodeMask; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_ENDPOINT PerfectHashServerSetEndpoint; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetEndpoint( + PPERFECT_HASH_SERVER Server, + PCUNICODE_STRING Endpoint + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + PWSTR Buffer; + ULONG BytesToAllocate; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Endpoint)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + if (!IsValidNullTerminatedUnicodeString(Endpoint)) { + return E_INVALIDARG; + } + + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + BytesToAllocate = Endpoint->Length + sizeof(WCHAR); + if (BytesToAllocate < Endpoint->Length) { + return E_INVALIDARG; + } + if (BytesToAllocate > (ULONG)USHRT_MAX) { + return E_INVALIDARG; + } + + Buffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, 1, BytesToAllocate); + if (!Buffer) { + return E_OUTOFMEMORY; + } + + CopyMemory(Buffer, Endpoint->Buffer, Endpoint->Length); + + if (Server->Flags.EndpointAllocated && Server->Endpoint.Buffer) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Server->Endpoint.Buffer); + } + + Server->Endpoint.Buffer = Buffer; + Server->Endpoint.Length = Endpoint->Length; + Server->Endpoint.MaximumLength = (USHORT)BytesToAllocate; + Server->Flags.EndpointAllocated = TRUE; + + return S_OK; +} + +PERFECT_HASH_SERVER_GET_ENDPOINT PerfectHashServerGetEndpoint; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetEndpoint( + PPERFECT_HASH_SERVER Server, + PUNICODE_STRING Endpoint + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Endpoint)) { + return E_POINTER; + } + + *Endpoint = Server->Endpoint; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_LOCAL_ONLY PerfectHashServerSetLocalOnly; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetLocalOnly( + PPERFECT_HASH_SERVER Server, + BOOLEAN LocalOnly + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.LocalOnly = LocalOnly ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_LOCAL_ONLY PerfectHashServerGetLocalOnly; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetLocalOnly( + PPERFECT_HASH_SERVER Server, + PBOOLEAN LocalOnly + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(LocalOnly)) { + return E_POINTER; + } + + *LocalOnly = (Server->Flags.LocalOnly != 0); + return S_OK; +} + +PERFECT_HASH_SERVER_SET_VERBOSE PerfectHashServerSetVerbose; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetVerbose( + PPERFECT_HASH_SERVER Server, + BOOLEAN Verbose + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.Verbose = Verbose ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_VERBOSE PerfectHashServerGetVerbose; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetVerbose( + PPERFECT_HASH_SERVER Server, + PBOOLEAN Verbose + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Verbose)) { + return E_POINTER; + } + + *Verbose = (Server->Flags.Verbose != 0); + return S_OK; +} + +PERFECT_HASH_SERVER_SET_NO_FILE_IO PerfectHashServerSetNoFileIo; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetNoFileIo( + PPERFECT_HASH_SERVER Server, + BOOLEAN NoFileIo + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.NoFileIo = NoFileIo ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_NO_FILE_IO PerfectHashServerGetNoFileIo; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetNoFileIo( + PPERFECT_HASH_SERVER Server, + PBOOLEAN NoFileIo + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(NoFileIo)) { + return E_POINTER; + } + + *NoFileIo = (Server->Flags.NoFileIo != 0); + return S_OK; +} + +PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerSetIocpBufferGuardPages; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetIocpBufferGuardPages( + PPERFECT_HASH_SERVER Server, + BOOLEAN GuardPages + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.IocpBufferGuardPages = GuardPages ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerGetIocpBufferGuardPages; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetIocpBufferGuardPages( + PPERFECT_HASH_SERVER Server, + PBOOLEAN GuardPages + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(GuardPages)) { + return E_POINTER; + } + + *GuardPages = (Server->Flags.IocpBufferGuardPages != 0); + return S_OK; +} + +PERFECT_HASH_SERVER_START PerfectHashServerStart; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerStart( + PPERFECT_HASH_SERVER Server + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->ContextIocp) { + Result = PerfectHashContextIocpStart(Server->ContextIocp); + if (FAILED(Result)) { + return Result; + } + } + + Result = PerfectHashServerCreatePipes(Server); + if (FAILED(Result)) { + PerfectHashContextIocpStop(Server->ContextIocp); + return Result; + } + + Server->State.Running = TRUE; + if (Server->StartedEvent) { + SetEvent(Server->StartedEvent); + } + + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_STOP PerfectHashServerStop; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerStop( + PPERFECT_HASH_SERVER Server + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + Server->State.Stopping = TRUE; + + if (Server->ContextIocp) { + PerfectHashServerDestroyPipes(Server); + Result = PerfectHashContextIocpStop(Server->ContextIocp); + if (FAILED(Result)) { + return Result; + } + } + + if (Server->ShutdownEvent) { + SetEvent(Server->ShutdownEvent); + } + + Server->State.Stopped = TRUE; + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_WAIT PerfectHashServerWait; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerWait( + PPERFECT_HASH_SERVER Server + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->ShutdownEvent) { + WaitForSingleObject(Server->ShutdownEvent, INFINITE); + } + + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_SUBMIT_REQUEST PerfectHashServerSubmitRequest; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSubmitRequest( + PPERFECT_HASH_SERVER Server, + PPERFECT_HASH_SERVER_REQUEST Request + ) +{ + UNREFERENCED_PARAMETER(Server); + UNREFERENCED_PARAMETER(Request); + + return E_NOTIMPL; +} + +#ifdef PH_WINDOWS + +static +HRESULT +PerfectHashServerCreatePipes( + PPERFECT_HASH_SERVER Server + ) +{ + ULONG NodeIndex; + ULONG PipeIndex; + ULONG PipeCount; + HRESULT Result; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->Pipes) { + return S_OK; + } + + ContextIocp = Server->ContextIocp; + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!ContextIocp || !Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PipeCount = ContextIocp->TotalWorkerThreadCount; + if (PipeCount == 0) { + PipeCount = 1; + } + + Server->Pipes = (PPERFECT_HASH_SERVER_PIPE)( + Allocator->Vtbl->Calloc(Allocator, + PipeCount, + sizeof(*Server->Pipes)) + ); + if (!Server->Pipes) { + return E_OUTOFMEMORY; + } + + Server->PipeCount = PipeCount; + PipeIndex = 0; + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + ULONG InstanceIndex; + + for (InstanceIndex = 0; + InstanceIndex < Node->WorkerThreadCount; + InstanceIndex++) { + DWORD OpenMode; + DWORD PipeMode; + HANDLE PipeHandle; + HANDLE PortHandle; + PPERFECT_HASH_SERVER_PIPE Pipe; + + if (PipeIndex >= Server->PipeCount) { + break; + } + + Pipe = &Server->Pipes[PipeIndex++]; + Pipe->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + Pipe->Iocp.Flags = PH_IOCP_WORK_FLAG_PIPE; + Pipe->Iocp.CompletionCallback = + PerfectHashServerIocpCompletionCallback; + Pipe->Iocp.CompletionContext = Pipe; + Pipe->Server = Server; + Pipe->Node = Node; + + OpenMode = PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED; + PipeMode = PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT; + +#ifdef PIPE_REJECT_REMOTE_CLIENTS + if (Server->Flags.LocalOnly) { + PipeMode |= PIPE_REJECT_REMOTE_CLIENTS; + } +#endif + + PipeHandle = CreateNamedPipeW(Server->Endpoint.Buffer, + OpenMode, + PipeMode, + PIPE_UNLIMITED_INSTANCES, + 0, + 0, + 0, + NULL); + if (!PipeHandle || PipeHandle == INVALID_HANDLE_VALUE) { + SYS_ERROR(CreateNamedPipeW); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Pipe->Pipe = PipeHandle; + + PortHandle = CreateIoCompletionPort(PipeHandle, + Node->IoCompletionPort, + (ULONG_PTR)Pipe, + 0); + if (!PortHandle) { + SYS_ERROR(CreateIoCompletionPort); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Result = PerfectHashServerIssueConnect(Pipe); + if (FAILED(Result)) { + goto Error; + } + } + } + + return S_OK; + +Error: + + PerfectHashServerDestroyPipes(Server); + return Result; +} + +static +VOID +PerfectHashServerDestroyPipes( + PPERFECT_HASH_SERVER Server + ) +{ + ULONG Index; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(Server)) { + return; + } + + if (!Server->Pipes) { + return; + } + + Allocator = Server->Allocator; + + for (Index = 0; Index < Server->PipeCount; Index++) { + PPERFECT_HASH_SERVER_PIPE Pipe = &Server->Pipes[Index]; + + if (Pipe->Pipe) { + CancelIoEx(Pipe->Pipe, NULL); + CloseHandle(Pipe->Pipe); + Pipe->Pipe = NULL; + } + + if (Pipe->PayloadBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + Pipe->PayloadBufferSize = 0; + } + } + + if (Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Server->Pipes); + } + + Server->Pipes = NULL; + Server->PipeCount = 0; +} + +static +HRESULT +PerfectHashServerIssueConnect( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateAccepting; + Pipe->BytesTransferred = 0; + Pipe->PayloadLength = 0; + Pipe->ShutdownAfterSend = FALSE; + + Success = ConnectNamedPipe(Pipe->Pipe, &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + if (LastError == ERROR_PIPE_CONNECTED) { + if (!PostQueuedCompletionStatus(Pipe->Node->IoCompletionPort, + 0, + (ULONG_PTR)Pipe, + &Pipe->Iocp.Overlapped)) { + SYS_ERROR(PostQueuedCompletionStatus); + return PH_E_SYSTEM_CALL_FAILED; + } + return S_OK; + } + + SYS_ERROR(ConnectNamedPipe); + return PH_E_SYSTEM_CALL_FAILED; +} + +static +HRESULT +PerfectHashServerIssueReadHeader( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = sizeof(PERFECT_HASH_SERVER_REQUEST_HEADER) - + Pipe->BytesTransferred; + Buffer = (PUCHAR)&Pipe->RequestHeader + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateReadingHeader; + + Success = ReadFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +HRESULT +PerfectHashServerIssueReadPayload( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = Pipe->PayloadLength - Pipe->BytesTransferred; + Buffer = (PUCHAR)Pipe->PayloadBuffer + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateReadingPayload; + + Success = ReadFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +HRESULT +PerfectHashServerIssueWriteResponseHeader( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER) - + Pipe->BytesTransferred; + Buffer = (PUCHAR)&Pipe->ResponseHeader + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateWritingResponseHeader; + + Success = WriteFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +HRESULT +PerfectHashServerIssueWriteResponsePayload( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = Pipe->ResponseHeader.PayloadLength - Pipe->BytesTransferred; + Buffer = (PUCHAR)Pipe->PayloadBuffer + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateWritingResponsePayload; + + Success = WriteFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +VOID +PerfectHashServerResetPipe( + PPERFECT_HASH_SERVER_PIPE Pipe, + BOOLEAN Reconnect + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + PPERFECT_HASH_SERVER Server; + + if (!ARGUMENT_PRESENT(Pipe)) { + return; + } + + Server = Pipe->Server; + Allocator = Server ? Server->Allocator : NULL; + Rtl = Server ? Server->Rtl : NULL; + + Pipe->BytesTransferred = 0; + Pipe->PayloadLength = 0; + Pipe->ShutdownAfterSend = FALSE; + + if (Pipe->PayloadBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + Pipe->PayloadBufferSize = 0; + } + + if (Rtl) { + ZeroMemory(&Pipe->RequestHeader, sizeof(Pipe->RequestHeader)); + ZeroMemory(&Pipe->ResponseHeader, sizeof(Pipe->ResponseHeader)); + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + } + + if (Pipe->Pipe) { + DisconnectNamedPipe(Pipe->Pipe); + } + + if (Reconnect && Server && + !Server->State.Stopping && + !Server->State.Stopped) { + PerfectHashServerIssueConnect(Pipe); + } +} + +static +HRESULT +PerfectHashServerPrepareErrorPayload( + PPERFECT_HASH_SERVER_PIPE Pipe, + HRESULT Error + ) +{ + BOOL FreeMessage; + DWORD Flags; + DWORD Count; + ULONG PayloadLength; + ULONG BytesToAllocate; + PRTL Rtl; + PALLOCATOR Allocator; + PWSTR Buffer; + PWSTR Message; + WCHAR Fallback[64]; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + FreeMessage = FALSE; + Message = NULL; + Fallback[0] = L'\0'; + + Flags = ( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_HMODULE | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS + ); + + Count = FormatMessageW(Flags, + PerfectHashModule, + (DWORD)Error, + 0, + (LPWSTR)&Message, + 0, + NULL); + + if (Count == 0 || !Message) { + swprintf_s(Fallback, + ARRAYSIZE(Fallback), + L"Error 0x%08lX", + (ULONG)Error); + Message = Fallback; + Count = (DWORD)wcslen(Message); + } else { + FreeMessage = TRUE; + } + + while (Count > 0) { + WCHAR Ch = Message[Count - 1]; + if (Ch == L'\r' || Ch == L'\n' || Ch == L' ' || Ch == L'\t') { + Message[Count - 1] = L'\0'; + Count--; + continue; + } + break; + } + + PayloadLength = (Count + 1) * sizeof(WCHAR); + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + ULONG MaxChars; + + MaxChars = (PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE / sizeof(WCHAR)); + if (MaxChars == 0) { + PayloadLength = 0; + } else { + MaxChars -= 1; + PayloadLength = (MaxChars + 1) * sizeof(WCHAR); + Message[MaxChars] = L'\0'; + Count = MaxChars; + } + } + + if (PayloadLength == 0) { + if (FreeMessage) { + LocalFree(Message); + } + return E_UNEXPECTED; + } + + BytesToAllocate = PayloadLength; + + if (!Pipe->PayloadBuffer || Pipe->PayloadBufferSize < BytesToAllocate) { + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + BytesToAllocate); + if (!Pipe->PayloadBuffer) { + if (FreeMessage) { + LocalFree(Message); + } + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = BytesToAllocate; + } else { + ZeroMemory(Pipe->PayloadBuffer, BytesToAllocate); + } + + Buffer = (PWSTR)Pipe->PayloadBuffer; + CopyMemory(Buffer, Message, Count * sizeof(WCHAR)); + Buffer[Count] = L'\0'; + + Pipe->ResponseHeader.PayloadLength = PayloadLength; + Pipe->ResponseHeader.Flags |= PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE; + + if (FreeMessage) { + LocalFree(Message); + } + + return S_OK; +} + +static +HRESULT +PerfectHashServerPreparePingPayload( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + WCHAR Pong[] = L"PONG"; + ULONG PayloadLength; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PayloadLength = (ULONG)sizeof(Pong); + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + return E_INVALIDARG; + } + + if (!Pipe->PayloadBuffer || Pipe->PayloadBufferSize < PayloadLength) { + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + PayloadLength); + if (!Pipe->PayloadBuffer) { + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = PayloadLength; + } else { + ZeroMemory(Pipe->PayloadBuffer, PayloadLength); + } + + CopyMemory(Pipe->PayloadBuffer, Pong, PayloadLength); + + Pipe->ResponseHeader.PayloadLength = PayloadLength; + Pipe->ResponseHeader.Flags |= PERFECT_HASH_SERVER_RESPONSE_FLAG_PONG; + + return S_OK; +} + +static +HRESULT +PerfectHashServerPrepareBulkCreateDirectoryPayload( + PPERFECT_HASH_SERVER_PIPE Pipe, + HANDLE EventHandle, + HANDLE ResultHandle + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + WCHAR Payload[128]; + int Count; + ULONG PayloadLength; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + Count = swprintf_s(Payload, + ARRAYSIZE(Payload), + PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT, + (ULONGLONG)(ULONG_PTR)EventHandle, + (ULONGLONG)(ULONG_PTR)ResultHandle); + if (Count <= 0) { + return E_UNEXPECTED; + } + + PayloadLength = (ULONG)((Count + 1) * sizeof(WCHAR)); + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + return E_INVALIDARG; + } + + if (!Pipe->PayloadBuffer || Pipe->PayloadBufferSize < PayloadLength) { + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + PayloadLength); + if (!Pipe->PayloadBuffer) { + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = PayloadLength; + } else { + ZeroMemory(Pipe->PayloadBuffer, PayloadLength); + } + + CopyMemory(Pipe->PayloadBuffer, Payload, PayloadLength); + + Pipe->ResponseHeader.PayloadLength = PayloadLength; + Pipe->ResponseHeader.Flags |= + PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN; + + return S_OK; +} + +static +VOID +PerfectHashServerCompleteBulkRequest( + PPERFECT_HASH_SERVER_BULK_REQUEST Request + ) +{ + ULONG Failed; + ULONG Total; + ULONG Succeeded; + HRESULT Result; + HRESULT FirstFailure; + PPERFECT_HASH_SERVER_BULK_RESULT BulkResult; + PALLOCATOR Allocator; + LONG Outstanding; + + if (!ARGUMENT_PRESENT(Request)) { + return; + } + + if (InterlockedCompareExchange(&Request->CompletionSignaled, 1, 0) != 0) { + return; + } + + Outstanding = Request->OutstandingWorkItems; + PerfectHashServerLogBulkCreateCounts(3, NULL, Request, Outstanding); + + Failed = (ULONG)Request->FailedWorkItems; + Total = Request->TotalWorkItems; + Succeeded = (Total >= Failed) ? (Total - Failed) : 0; + FirstFailure = (HRESULT)Request->FirstFailure; + + if (Total == 0) { + Result = (FirstFailure != S_OK) ? + FirstFailure : + PH_E_NO_KEYS_FOUND_IN_DIRECTORY; + } else if (Failed == 0) { + Result = PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED; + } else { + Result = PH_E_SERVER_BULK_CREATE_FAILED; + } + + BulkResult = Request->ResultMapping; + if (BulkResult) { + BulkResult->SizeOfStruct = sizeof(*BulkResult); + BulkResult->Version = PERFECT_HASH_SERVER_BULK_RESULT_VERSION; + BulkResult->Result = Result; + BulkResult->Flags = 0; + BulkResult->TotalFiles = Total; + BulkResult->SucceededFiles = Succeeded; + BulkResult->FailedFiles = Failed; + BulkResult->FirstFailure = FirstFailure; + } + + if (Request->CompletionEvent) { + SetEvent(Request->CompletionEvent); + } + + if (Request->ResultMapping) { + UnmapViewOfFile(Request->ResultMapping); + } + + if (Request->ResultMappingHandle) { + CloseHandle(Request->ResultMappingHandle); + } + + if (Request->CompletionEvent) { + CloseHandle(Request->CompletionEvent); + } + + if (Request->TableCreateParameters.SizeOfStruct != 0) { + HRESULT CleanupResult; + + CleanupResult = CleanupTableCreateParameters( + &Request->TableCreateParameters + ); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + } + } + + if (Request->ArgvW) { + LocalFree(Request->ArgvW); + Request->ArgvW = NULL; + } + + Allocator = Request->Server ? Request->Server->Allocator : NULL; + if (Allocator) { + if (Request->NodeOutstandingCounts) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->NodeOutstandingCounts + ); + } + + if (Request->BaseOutputDirectoryBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->BaseOutputDirectoryBuffer + ); + } + + if (Request->CommandLineBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->CommandLineBuffer + ); + } + + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Request); + } +} + +static +VOID +PerfectHashServerLogBulkCreateException( + ULONG Stage, + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + struct _EXCEPTION_POINTERS *ExceptionPointers + ) +{ +#ifdef PH_WINDOWS + HANDLE LogHandle; + DWORD BytesWritten; + ULONG NodeIndex = 0; + ULONG_PTR Address = 0; + ULONG_PTR Offset = 0; + DWORD Code = 0; + DWORD ThreadId; + CHAR Buffer[256]; + HMODULE Module = NULL; + + if (!ARGUMENT_PRESENT(ExceptionPointers)) { + return; + } + + if (GetEnvironmentVariableW(L"PH_LOG_SERVER_CRASH", NULL, 0) == 0) { + return; + } + + if (WorkItem) { + NodeIndex = WorkItem->NodeIndex; + } + + LogHandle = CreateFileW(L"PerfectHashServerBulkCreateCrash.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + ThreadId = GetCurrentThreadId(); + Code = ExceptionPointers->ExceptionRecord->ExceptionCode; + Address = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress; + + if (GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)ExceptionPointers->ExceptionRecord->ExceptionAddress, + &Module)) { + Offset = Address - (ULONG_PTR)Module; + } + + _snprintf_s( + Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Code=0x%08lX Address=0x%p Module=0x%p Offset=0x%Ix " + "ThreadId=%lu Node=%lu\r\n", + Stage, + Code, + (PVOID)Address, + Module, + Offset, + ThreadId, + NodeIndex + ); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(WorkItem); + UNREFERENCED_PARAMETER(ExceptionPointers); +#endif +} + +static +VOID +PerfectHashServerLogBulkCreateCounts( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_opt_ PPERFECT_HASH_SERVER_BULK_REQUEST Request, + _In_ LONG Outstanding + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[512]; + ULONG PathChars = 0; + LONG DispatchComplete = 0; + LONG Failed = 0; + LONG PendingNodes = 0; + ULONG Total = 0; + ULONG NodeIndex = 0; + + if (GetEnvironmentVariableW(L"PH_LOG_BULK_CREATE_COUNTS", NULL, 0) == 0) { + return; + } + + if (Request) { + DispatchComplete = Request->DispatchComplete; + Failed = Request->FailedWorkItems; + PendingNodes = Request->PendingNodes; + Total = Request->TotalWorkItems; + } + + if (WorkItem) { + NodeIndex = WorkItem->NodeIndex; + if (WorkItem->KeysPath.Buffer) { + PathChars = WorkItem->KeysPath.Length / sizeof(WCHAR); + } + } + + LogHandle = CreateFileW(L"PerfectHashServerBulkCreateCounts.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s( + Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Outstanding=%ld Dispatch=%ld Total=%lu Failed=%ld " + "PendingNodes=%ld Node=%lu Path=%.*S\r\n", + Stage, + Outstanding, + DispatchComplete, + Total, + Failed, + PendingNodes, + NodeIndex, + (int)PathChars, + WorkItem ? WorkItem->KeysPath.Buffer : L"" + ); + + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(WorkItem); + UNREFERENCED_PARAMETER(Request); + UNREFERENCED_PARAMETER(Outstanding); +#endif +} + +static +VOID +PerfectHashServerLogBulkCreateFailure( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_ HRESULT Result + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[1024]; + ULONG PathChars = 0; + + if (GetEnvironmentVariableW(L"PH_LOG_BULK_CREATE_FAILURES", NULL, 0) == 0) { + return; + } + + if (!WorkItem || !WorkItem->KeysPath.Buffer) { + return; + } + + PathChars = WorkItem->KeysPath.Length / sizeof(WCHAR); + + LogHandle = CreateFileW(L"PerfectHashServerBulkCreateFailures.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Result=0x%08lX LastError=%lu (0x%08lX) " + "Path=%.*S\r\n", + Stage, + Result, + WorkItem->LastError, + WorkItem->LastError, + (int)PathChars, + WorkItem->KeysPath.Buffer); + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(WorkItem); + UNREFERENCED_PARAMETER(Result); +#endif +} + +static +LONG +PerfectHashServerBulkCreateExceptionFilter( + ULONG Stage, + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + struct _EXCEPTION_POINTERS *ExceptionPointers + ) +{ + PerfectHashServerLogBulkCreateException(Stage, + WorkItem, + ExceptionPointers); + return EXCEPTION_EXECUTE_HANDLER; +} + +static +HRESULT +PerfectHashServerBulkCreateWorkItemCallback( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG_PTR CompletionKey, + LPOVERLAPPED Overlapped, + DWORD NumberOfBytesTransferred, + BOOL Success + ) +{ + HRESULT Result; + HRESULT Failure; + LONG Outstanding; + LONG NodeOutstanding; + PALLOCATOR Allocator = NULL; + PPERFECT_HASH_CONTEXT Context = NULL; + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem; + PPERFECT_HASH_SERVER_BULK_REQUEST Request; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext = { 0 }; + ULONG Stage = 0; + BOOLEAN ExceptionRaised = FALSE; + + UNREFERENCED_PARAMETER(NumberOfBytesTransferred); + + WorkItem = CONTAINING_RECORD(Overlapped, + PERFECT_HASH_SERVER_BULK_WORK_ITEM, + Iocp.Overlapped); + Request = WorkItem->Request; + Allocator = (Request && Request->Server) ? Request->Server->Allocator : NULL; + WorkItem->LastError = ERROR_SUCCESS; + + if (!Success) { + WorkItem->LastError = GetLastError(); + Failure = HRESULT_FROM_WIN32(WorkItem->LastError); + } else { + Failure = S_OK; + } + + __try { + Stage = 1; + + if ((ULONG_PTR)WorkItem != CompletionKey) { + Failure = E_UNEXPECTED; + } + + if (FAILED(Failure)) { + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + Stage = 2; + Result = PerfectHashContextIocpCreateTableContext(ContextIocp, + &Context); + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + if (Request && Request->Server && + Request->Server->Flags.IocpBufferGuardPages) { + SetContextUseIocpBufferGuardPages(Context); + } + + if (Request->CommandLineBuffer) { + Context->CommandLineW = Request->CommandLineBuffer; + } + + if (WorkItem->Node && WorkItem->Node->IoCompletionPort) { + Context->IocpNode = WorkItem->Node; + Context->IocpNodeIndex = WorkItem->NodeIndex; + Context->FileWorkIoCompletionPort = WorkItem->Node->IoCompletionPort; + } + + SetContextSkipContextFileWork(Context); + + if (Request->PerFileMaximumConcurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency( + Context, + Request->PerFileMaximumConcurrency + ); + if (FAILED(Result)) { + if (Result == PH_E_SET_MAXIMUM_CONCURRENCY_FAILED && + GetLastError() == ERROR_ACCESS_DENIED) { + // + // Ignore access-denied threadpool sizing failures; proceed + // with the default concurrency configured by initialization. + // + Result = S_OK; + } + + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + } + } + + Stage = 3; + Result = PerfectHashContextInitializeFunctionHookCallbackDll( + Context, + &Request->TableCreateFlags, + &Request->TableCreateParameters + ); + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + Stage = 4; + Result = PerfectHashContextInitializeRng( + Context, + &Request->TableCreateFlags, + &Request->TableCreateParameters + ); + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + ContextTableCreateFlags = Request->ContextTableCreateFlags; + + Stage = 5; + TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + TlsContext->Context = Context; + + Stage = 6; + Result = Context->Vtbl->TableCreate( + Context, + &WorkItem->KeysPath, + &Request->BaseOutputDirectory, + Request->AlgorithmId, + Request->HashFunctionId, + Request->MaskFunctionId, + &ContextTableCreateFlags, + &Request->KeysLoadFlags, + &Request->TableCreateFlags, + &Request->TableCompileFlags, + &Request->TableCreateParameters + ); + + Stage = 7; + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + + if (FAILED(Result)) { + WorkItem->LastError = GetLastError(); + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + } + + } __except (PerfectHashServerBulkCreateExceptionFilter( + Stage, + WorkItem, + GetExceptionInformation())) { + ExceptionRaised = TRUE; + } + + if (ExceptionRaised) { + Failure = E_UNEXPECTED; + if (Stage >= 6) { + __try { + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + } __except (EXCEPTION_EXECUTE_HANDLER) { + NOTHING; + } + } + if (Request) { + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + } + } + +Complete: + + if (FAILED(Failure)) { + PerfectHashServerLogBulkCreateFailure(Stage, WorkItem, Failure); + } + + if (Context) { + Context->Vtbl->Release(Context); + } + + if (WorkItem->KeysPathBuffer && Request->Server && + Request->Server->Allocator) { + Request->Server->Allocator->Vtbl->FreePointer( + Request->Server->Allocator, + (PVOID *)&WorkItem->KeysPathBuffer + ); + } + + if (Request->NodeOutstandingCounts) { + NodeOutstanding = InterlockedDecrement( + &Request->NodeOutstandingCounts[WorkItem->NodeIndex] + ); + if (NodeOutstanding == 0) { + InterlockedDecrement(&Request->PendingNodes); + } + } + + Outstanding = InterlockedDecrement(&Request->OutstandingWorkItems); + PerfectHashServerLogBulkCreateCounts(1, WorkItem, Request, Outstanding); + + if (Outstanding == 0 && Request->DispatchComplete != 0) { + PerfectHashServerCompleteBulkRequest(Request); + Request = NULL; + } + + if (Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + } + + return S_OK; +} + +static +HRESULT +PerfectHashServerEnqueueBulkRequest( + PPERFECT_HASH_SERVER_BULK_REQUEST Request, + PUNICODE_STRING KeysDirectory + ) +{ + BOOL Success; + HRESULT Result = S_OK; + DWORD LastError; + ULONG NodeIndex; + ULONG BytesToAllocate; + ULONG WildcardBytes; + ULONG KeysPathBytes; + LONG Outstanding; + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + PWSTR WildcardBuffer = NULL; + PWSTR KeysPathBuffer = NULL; + BOOLEAN DispatchInitialized = FALSE; + PWSTR Dest; + UNICODE_STRING WildcardPath; + WIN32_FIND_DATAW FindData; + HANDLE FindHandle = NULL; + + if (!ARGUMENT_PRESENT(Request)) { + return E_POINTER; + } + + Request->OutstandingWorkItems = 1; + Request->TotalWorkItems = 0; + Request->DispatchComplete = 0; + DispatchInitialized = TRUE; + + if (!ARGUMENT_PRESENT(KeysDirectory)) { + Result = E_POINTER; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + if (!IsValidMinimumDirectoryUnicodeString(KeysDirectory)) { + Result = E_INVALIDARG; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + Allocator = Request->Server ? Request->Server->Allocator : NULL; + if (!Allocator) { + Result = E_UNEXPECTED; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + Rtl = Request->Server ? Request->Server->Rtl : NULL; + if (!Rtl) { + Result = E_UNEXPECTED; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + BytesToAllocate = KeysDirectory->Length + + sizeof(WCHAR) + + KeysWildcardSuffix.Length + + sizeof(WCHAR); + if (BytesToAllocate < KeysDirectory->Length) { + Result = E_INVALIDARG; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + WildcardBuffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, + 1, + BytesToAllocate); + if (!WildcardBuffer) { + Result = E_OUTOFMEMORY; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + CopyMemory(WildcardBuffer, + KeysDirectory->Buffer, + KeysDirectory->Length); + + Dest = (PWSTR)RtlOffsetToPointer(WildcardBuffer, + KeysDirectory->Length); + *Dest++ = PATHSEP; + CopyMemory(Dest, KeysWildcardSuffix.Buffer, KeysWildcardSuffix.Length); + + WildcardBytes = BytesToAllocate - sizeof(WCHAR); + WildcardPath.Buffer = WildcardBuffer; + WildcardPath.Length = (USHORT)WildcardBytes; + WildcardPath.MaximumLength = (USHORT)BytesToAllocate; + + FindHandle = FindFirstFileW(WildcardPath.Buffer, &FindData); + if (!IsValidHandle(FindHandle)) { + LastError = GetLastError(); + if (LastError == ERROR_FILE_NOT_FOUND) { + Result = PH_E_NO_KEYS_FOUND_IN_DIRECTORY; + } else { + Result = HRESULT_FROM_WIN32(LastError); + } + if (Result != PH_E_NO_KEYS_FOUND_IN_DIRECTORY) { + InterlockedIncrement(&Request->FailedWorkItems); + } + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + NodeIndex = Request->NextNodeIndex; + + do { + ULONG FileNameLength; + PPERFECT_HASH_IOCP_NODE Node; + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem; + LONG NodeOutstanding; + + if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + + FileNameLength = (ULONG)wcslen(FindData.cFileName) * sizeof(WCHAR); + + KeysPathBytes = KeysDirectory->Length + + sizeof(WCHAR) + + FileNameLength + + sizeof(WCHAR); + if (KeysPathBytes < KeysDirectory->Length || + KeysPathBytes > (ULONG)USHRT_MAX) { + Result = E_INVALIDARG; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + break; + } + + WorkItem = (PPERFECT_HASH_SERVER_BULK_WORK_ITEM)( + Allocator->Vtbl->Calloc(Allocator, 1, sizeof(*WorkItem)) + ); + if (!WorkItem) { + Result = E_OUTOFMEMORY; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + break; + } + + KeysPathBuffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, + 1, + KeysPathBytes); + if (!KeysPathBuffer) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + Result = E_OUTOFMEMORY; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + break; + } + + CopyMemory(KeysPathBuffer, + KeysDirectory->Buffer, + KeysDirectory->Length); + + Dest = (PWSTR)RtlOffsetToPointer(KeysPathBuffer, + KeysDirectory->Length); + *Dest++ = PATHSEP; + CopyMemory(Dest, FindData.cFileName, FileNameLength); + + WorkItem->Request = Request; + WorkItem->NodeIndex = NodeIndex; + WorkItem->Node = &Request->Nodes[NodeIndex]; + WorkItem->KeysPathBuffer = KeysPathBuffer; + WorkItem->KeysPath.Buffer = KeysPathBuffer; + WorkItem->KeysPath.Length = (USHORT)(KeysPathBytes - sizeof(WCHAR)); + WorkItem->KeysPath.MaximumLength = (USHORT)KeysPathBytes; + WorkItem->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + WorkItem->Iocp.Flags = PH_IOCP_WORK_FLAG_BULK; + WorkItem->Iocp.CompletionCallback = + PerfectHashServerBulkCreateWorkItemCallback; + WorkItem->Iocp.CompletionContext = WorkItem; + + Node = WorkItem->Node; + InterlockedIncrement(&Request->OutstandingWorkItems); + Request->TotalWorkItems++; + NodeOutstanding = InterlockedIncrement( + &Request->NodeOutstandingCounts[NodeIndex] + ); + if (NodeOutstanding == 1) { + InterlockedIncrement(&Request->PendingNodes); + } + + Success = PostQueuedCompletionStatus(Node->IoCompletionPort, + 0, + (ULONG_PTR)WorkItem, + &WorkItem->Iocp.Overlapped); + if (!Success) { + Result = HRESULT_FROM_WIN32(GetLastError()); + + InterlockedDecrement(&Request->OutstandingWorkItems); + Request->TotalWorkItems--; + NodeOutstanding = InterlockedDecrement( + &Request->NodeOutstandingCounts[NodeIndex] + ); + if (NodeOutstanding == 0) { + InterlockedDecrement(&Request->PendingNodes); + } + + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + + if (WorkItem->KeysPathBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&WorkItem->KeysPathBuffer); + } + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + } + + NodeIndex++; + if (NodeIndex >= Request->NodeCount) { + NodeIndex = 0; + } + + } while (FindNextFileW(FindHandle, &FindData)); + + if (Result == S_OK) { + LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) { + Result = HRESULT_FROM_WIN32(LastError); + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + } + } + + Request->NextNodeIndex = NodeIndex; + + if (Request->TotalWorkItems == 0 && Result == S_OK) { + Result = PH_E_NO_KEYS_FOUND_IN_DIRECTORY; + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + } + +End: + + if (FindHandle && FindHandle != INVALID_HANDLE_VALUE) { + FindClose(FindHandle); + } + + if (DispatchInitialized) { + Request->DispatchComplete = 1; + Outstanding = InterlockedDecrement(&Request->OutstandingWorkItems); + PerfectHashServerLogBulkCreateCounts(2, NULL, Request, Outstanding); + if (Outstanding == 0) { + PerfectHashServerCompleteBulkRequest(Request); + } + } + + if (WildcardBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WildcardBuffer); + } + + return Result; +} + +static +HRESULT +PerfectHashServerDispatchBulkCreateDirectoryRequest( + PPERFECT_HASH_SERVER_PIPE Pipe, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLine, + PBOOLEAN ArgvWOwned + ) +{ + BOOL Success; + HRESULT Result; + HRESULT CleanupResult; + ULONG BytesToAllocate; + ULONG CommandLineChars; + ULONG CommandLineBytes; + DWORD ClientProcessId = 0; + HANDLE ClientProcess = NULL; + HANDLE EventHandle = NULL; + HANDLE ResultHandle = NULL; + HANDLE ClientEventHandle = NULL; + HANDLE ClientResultHandle = NULL; + ULONG AdjustedArguments; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + PPERFECT_HASH_SERVER_BULK_REQUEST Request = NULL; + PPERFECT_HASH_SERVER_BULK_RESULT BulkResult = NULL; + UNICODE_STRING KeysDirectory = { 0 }; + UNICODE_STRING BaseOutputDirectory = { 0 }; + PERFECT_HASH_ALGORITHM_ID AlgorithmId = 0; + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId = 0; + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId = 0; + ULONG MaximumConcurrency = 0; + PERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags = { 0 }; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags = { 0 }; + PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags = { 0 }; + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters = { 0 }; + LPWSTR *AdjustedArgvW = NULL; + BOOLEAN ParsedArgs = FALSE; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Server = Pipe->Server; + if (!Server) { + return E_UNEXPECTED; + } + + ContextIocp = Server->ContextIocp; + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!ContextIocp || !Allocator || !Rtl) { + return E_UNEXPECTED; + } + + if (!ARGUMENT_PRESENT(ArgvWOwned)) { + return E_POINTER; + } + + *ArgvWOwned = FALSE; + + Result = LoadDefaultTableCreateFlags(&TableCreateFlags); + if (FAILED(Result)) { + goto Error; + } + + TableCreateParameters.SizeOfStruct = sizeof(TableCreateParameters); + TableCreateParameters.Allocator = Allocator; + + AdjustedArguments = NumberOfArguments + 1; + AdjustedArgvW = Allocator->Vtbl->Calloc(Allocator, + AdjustedArguments, + sizeof(*AdjustedArgvW)); + if (!AdjustedArgvW) { + Result = E_OUTOFMEMORY; + goto Error; + } + + AdjustedArgvW[0] = L"PerfectHashBulkCreate"; + CopyMemory(&AdjustedArgvW[1], + ArgvW, + NumberOfArguments * sizeof(*AdjustedArgvW)); + + Result = ContextIocp->Vtbl->ExtractBulkCreateArgsFromArgvW( + ContextIocp, + AdjustedArguments, + AdjustedArgvW, + CommandLine, + &KeysDirectory, + &BaseOutputDirectory, + &AlgorithmId, + &HashFunctionId, + &MaskFunctionId, + &MaximumConcurrency, + &ContextBulkCreateFlags, + &KeysLoadFlags, + &TableCreateFlags, + &TableCompileFlags, + &TableCreateParameters + ); + if (FAILED(Result)) { + goto Error; + } + ParsedArgs = TRUE; + + ContextTableCreateFlags.AsULong = 0; + ContextTableCreateFlags.SkipTestAfterCreate = + ContextBulkCreateFlags.SkipTestAfterCreate; + ContextTableCreateFlags.Compile = ContextBulkCreateFlags.Compile; + ContextTableCreateFlags.MonitorLowMemory = + ContextBulkCreateFlags.MonitorLowMemory; + + // + // Suppress per-file CSV output for async bulk operations. + // + + TableCreateFlags.DisableCsvOutputFile = TRUE; + if (Server->Flags.NoFileIo) { + TableCreateFlags.NoFileIo = TRUE; + } + if (!Server->Flags.Verbose) { + TableCreateFlags.Silent = TRUE; + TableCreateFlags.Quiet = FALSE; + } + + Request = (PPERFECT_HASH_SERVER_BULK_REQUEST)( + Allocator->Vtbl->Calloc(Allocator, 1, sizeof(*Request)) + ); + if (!Request) { + Result = E_OUTOFMEMORY; + goto Error; + } + + Request->SizeOfStruct = sizeof(*Request); + Request->Server = Server; + Request->ContextBulkCreateFlags = ContextBulkCreateFlags; + Request->ContextTableCreateFlags = ContextTableCreateFlags; + Request->KeysLoadFlags = KeysLoadFlags; + Request->TableCreateFlags = TableCreateFlags; + Request->TableCompileFlags = TableCompileFlags; + Request->TableCreateParameters = TableCreateParameters; + Request->AlgorithmId = AlgorithmId; + Request->HashFunctionId = HashFunctionId; + Request->MaskFunctionId = MaskFunctionId; + Request->PerFileMaximumConcurrency = MaximumConcurrency; + Request->FirstFailure = (LONG)S_OK; + Request->ArgvW = ArgvW; + *ArgvWOwned = TRUE; + + CommandLineChars = (ULONG)wcslen(CommandLine); + if (CommandLineChars > ((ULONG_MAX / sizeof(WCHAR)) - 1)) { + Result = E_INVALIDARG; + goto Error; + } + + CommandLineBytes = (CommandLineChars + 1) * sizeof(WCHAR); + Request->CommandLineBuffer = (PWSTR)( + Allocator->Vtbl->Calloc(Allocator, 1, CommandLineBytes) + ); + if (!Request->CommandLineBuffer) { + Result = E_OUTOFMEMORY; + goto Error; + } + + CopyMemory(Request->CommandLineBuffer, + CommandLine, + CommandLineBytes); + + Request->Nodes = ContextIocp->Nodes; + Request->NodeCount = ContextIocp->NodeCount; + + if (!Request->Nodes || Request->NodeCount == 0) { + Result = E_UNEXPECTED; + goto Error; + } + + Request->NodeOutstandingCounts = (PLONG)( + Allocator->Vtbl->Calloc(Allocator, + Request->NodeCount, + sizeof(*Request->NodeOutstandingCounts)) + ); + if (!Request->NodeOutstandingCounts) { + Result = E_OUTOFMEMORY; + goto Error; + } + + BytesToAllocate = BaseOutputDirectory.Length + sizeof(WCHAR); + if (BytesToAllocate < BaseOutputDirectory.Length || + BytesToAllocate > (ULONG)USHRT_MAX) { + Result = E_INVALIDARG; + goto Error; + } + + Request->BaseOutputDirectoryBuffer = (PWSTR)( + Allocator->Vtbl->Calloc(Allocator, 1, BytesToAllocate) + ); + if (!Request->BaseOutputDirectoryBuffer) { + Result = E_OUTOFMEMORY; + goto Error; + } + + CopyMemory(Request->BaseOutputDirectoryBuffer, + BaseOutputDirectory.Buffer, + BaseOutputDirectory.Length); + + Request->BaseOutputDirectory.Buffer = Request->BaseOutputDirectoryBuffer; + Request->BaseOutputDirectory.Length = BaseOutputDirectory.Length; + Request->BaseOutputDirectory.MaximumLength = (USHORT)BytesToAllocate; + + EventHandle = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!EventHandle) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + Request->CompletionEvent = EventHandle; + + ResultHandle = CreateFileMappingW(INVALID_HANDLE_VALUE, + NULL, + PAGE_READWRITE, + 0, + sizeof(PERFECT_HASH_SERVER_BULK_RESULT), + NULL); + if (!ResultHandle) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + Request->ResultMappingHandle = ResultHandle; + + BulkResult = (PPERFECT_HASH_SERVER_BULK_RESULT)( + MapViewOfFile(ResultHandle, FILE_MAP_WRITE, 0, 0, 0) + ); + if (!BulkResult) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + ZeroMemory(BulkResult, sizeof(*BulkResult)); + Request->ResultMapping = BulkResult; + + Success = GetNamedPipeClientProcessId(Pipe->Pipe, &ClientProcessId); + if (!Success) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + ClientProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, ClientProcessId); + if (!ClientProcess) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Success = DuplicateHandle(GetCurrentProcess(), + EventHandle, + ClientProcess, + &ClientEventHandle, + 0, + FALSE, + DUPLICATE_SAME_ACCESS); + if (!Success) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Success = DuplicateHandle(GetCurrentProcess(), + ResultHandle, + ClientProcess, + &ClientResultHandle, + 0, + FALSE, + DUPLICATE_SAME_ACCESS); + if (!Success) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + CloseHandle(ClientProcess); + ClientProcess = NULL; + + Result = PerfectHashServerPrepareBulkCreateDirectoryPayload( + Pipe, + ClientEventHandle, + ClientResultHandle + ); + if (FAILED(Result)) { + goto Error; + } + + Result = PerfectHashServerEnqueueBulkRequest(Request, &KeysDirectory); + if (FAILED(Result)) { + // + // Enqueue failures are recorded and signaled by the enqueue routine. + // + } + + if (AdjustedArgvW) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); + } + + return S_OK; + +Error: + + if (AdjustedArgvW && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); + } + + if (ClientProcess) { + CloseHandle(ClientProcess); + } + + if (BulkResult) { + UnmapViewOfFile(BulkResult); + } + + if (ResultHandle) { + CloseHandle(ResultHandle); + } + + if (EventHandle) { + CloseHandle(EventHandle); + } + + if (Request) { + if (Request->ArgvW) { + LocalFree(Request->ArgvW); + Request->ArgvW = NULL; + } + + if (Request->NodeOutstandingCounts) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->NodeOutstandingCounts + ); + } + + if (Request->BaseOutputDirectoryBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->BaseOutputDirectoryBuffer + ); + } + + if (Request->CommandLineBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->CommandLineBuffer + ); + } + + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Request); + } + + if (ParsedArgs) { + CleanupResult = CleanupTableCreateParameters(&TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + Result = CleanupResult; + } + } + + return Result; +} + +static +HRESULT +PerfectHashServerDispatchTableCreateRequest( + PPERFECT_HASH_SERVER_PIPE Pipe, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLine + ) +{ + HRESULT Result; + HRESULT CleanupResult; + ULONG CommandLineChars; + ULONG CommandLineBytes; + ULONG AdjustedArguments; + ULONG MaximumConcurrency = 0; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_CONTEXT Context = NULL; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + PPERFECT_HASH_IOCP_NODE Node = NULL; + LPWSTR *AdjustedArgvW = NULL; + LPWSTR *ArgvToUse = NULL; + PWSTR CommandLineBuffer = NULL; + UNICODE_STRING KeysPath = { 0 }; + UNICODE_STRING BaseOutputDirectory = { 0 }; + PERFECT_HASH_ALGORITHM_ID AlgorithmId = 0; + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId = 0; + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId = 0; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags = { 0 }; + PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags = { 0 }; + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters = { 0 }; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext = { 0 }; + BOOLEAN ParsedArgs = FALSE; + BOOLEAN TlsContextSet = FALSE; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Server = Pipe->Server; + if (!Server) { + return E_UNEXPECTED; + } + + ContextIocp = Server->ContextIocp; + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!ContextIocp || !Allocator || !Rtl) { + return E_UNEXPECTED; + } + + if (!ARGUMENT_PRESENT(ArgvW) || NumberOfArguments == 0) { + return E_INVALIDARG; + } + + Result = LoadDefaultTableCreateFlags(&TableCreateFlags); + if (FAILED(Result)) { + return Result; + } + + TableCreateParameters.SizeOfStruct = sizeof(TableCreateParameters); + TableCreateParameters.Allocator = Allocator; + + ArgvToUse = ArgvW; + AdjustedArguments = NumberOfArguments; + + if (ArgvW[0][0] == L'-' || ArgvW[0][0] == L'/') { + AdjustedArgvW = Allocator->Vtbl->Calloc(Allocator, + NumberOfArguments + 1, + sizeof(*AdjustedArgvW)); + if (!AdjustedArgvW) { + Result = E_OUTOFMEMORY; + goto Error; + } + + AdjustedArgvW[0] = L"PerfectHashTableCreate"; + CopyMemory(&AdjustedArgvW[1], + ArgvW, + NumberOfArguments * sizeof(*AdjustedArgvW)); + ArgvToUse = AdjustedArgvW; + AdjustedArguments = NumberOfArguments + 1; + } + + Result = ContextIocp->Vtbl->ExtractTableCreateArgsFromArgvW( + ContextIocp, + AdjustedArguments, + ArgvToUse, + CommandLine, + &KeysPath, + &BaseOutputDirectory, + &AlgorithmId, + &HashFunctionId, + &MaskFunctionId, + &MaximumConcurrency, + &ContextTableCreateFlags, + &KeysLoadFlags, + &TableCreateFlags, + &TableCompileFlags, + &TableCreateParameters + ); + if (FAILED(Result)) { + goto Error; + } + ParsedArgs = TRUE; + + if (Server->Flags.NoFileIo) { + TableCreateFlags.NoFileIo = TRUE; + } + if (!Server->Flags.Verbose) { + TableCreateFlags.Silent = TRUE; + TableCreateFlags.Quiet = FALSE; + } + + Result = PerfectHashContextIocpCreateTableContext(ContextIocp, + &Context); + if (FAILED(Result)) { + goto Error; + } + + if (Server->Flags.IocpBufferGuardPages) { + SetContextUseIocpBufferGuardPages(Context); + } + + if (CommandLine) { + CommandLineChars = (ULONG)wcslen(CommandLine); + if (CommandLineChars > ((ULONG_MAX / sizeof(WCHAR)) - 1)) { + Result = E_INVALIDARG; + goto Error; + } + + CommandLineBytes = (CommandLineChars + 1) * sizeof(WCHAR); + CommandLineBuffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, + 1, + CommandLineBytes); + if (!CommandLineBuffer) { + Result = E_OUTOFMEMORY; + goto Error; + } + + CopyMemory(CommandLineBuffer, CommandLine, CommandLineBytes); + Context->CommandLineW = CommandLineBuffer; + } + + if (ContextIocp->NodeCount > 0 && ContextIocp->Nodes) { + Node = &ContextIocp->Nodes[0]; + if (Node->IoCompletionPort) { + Context->FileWorkIoCompletionPort = Node->IoCompletionPort; + } + } + + SetContextSkipContextFileWork(Context); + + Result = PerfectHashContextInitializeFunctionHookCallbackDll( + Context, + &TableCreateFlags, + &TableCreateParameters + ); + if (FAILED(Result)) { + goto Error; + } + + if (MaximumConcurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency(Context, + MaximumConcurrency); + if (FAILED(Result)) { + if (Result == PH_E_SET_MAXIMUM_CONCURRENCY_FAILED && + GetLastError() == ERROR_ACCESS_DENIED) { + Result = S_OK; + } + } + if (FAILED(Result)) { + goto Error; + } + } + + Result = PerfectHashContextInitializeRng(Context, + &TableCreateFlags, + &TableCreateParameters); + if (FAILED(Result)) { + goto Error; + } + + TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + TlsContext->Context = Context; + TlsContextSet = TRUE; + + Result = Context->Vtbl->TableCreate(Context, + &KeysPath, + &BaseOutputDirectory, + AlgorithmId, + HashFunctionId, + MaskFunctionId, + &ContextTableCreateFlags, + &KeysLoadFlags, + &TableCreateFlags, + &TableCompileFlags, + &TableCreateParameters); + +Error: + + if (TlsContextSet) { + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + } + + if (ParsedArgs) { + CleanupResult = CleanupTableCreateParameters(&TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + if (SUCCEEDED(Result)) { + Result = CleanupResult; + } + } + } + + if (Context) { + Context->Vtbl->Release(Context); + } + + if (CommandLineBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&CommandLineBuffer); + } + + if (AdjustedArgvW && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); + } + + return Result; +} + +static +HRESULT +PerfectHashServerDispatchRequest( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + HRESULT Result = E_UNEXPECTED; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + PALLOCATOR Allocator; + LPWSTR CommandLine; + LPWSTR *ArgvW; + ULONG NumberOfArguments; + PRTL Rtl; + BOOLEAN ArgvWOwned; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + ContextIocp = Pipe->Server ? Pipe->Server->ContextIocp : NULL; + RequestType = Pipe->RequestHeader.RequestType; + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + + ZeroMemory(&Pipe->ResponseHeader, sizeof(Pipe->ResponseHeader)); + Pipe->ResponseHeader.SizeOfStruct = + sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER); + Pipe->ResponseHeader.Version = PERFECT_HASH_SERVER_MESSAGE_VERSION; + Pipe->ResponseHeader.RequestId = Pipe->RequestHeader.RequestId; + Pipe->ResponseHeader.PayloadLength = 0; + + ArgvW = NULL; + NumberOfArguments = 0; + CommandLine = NULL; + ArgvWOwned = FALSE; + + switch (RequestType) { + case PerfectHashNullServerRequestType: + case PerfectHashInvalidServerRequestType: + Result = E_INVALIDARG; + break; + + case PerfectHashPingServerRequestType: + if (Pipe->PayloadLength != 0) { + Result = E_INVALIDARG; + break; + } + if (Pipe->Server && Pipe->Server->StartedEvent) { + WaitForSingleObject(Pipe->Server->StartedEvent, INFINITE); + } + Result = PerfectHashServerPreparePingPayload(Pipe); + break; + + case PerfectHashShutdownServerRequestType: + Result = S_OK; + Pipe->ShutdownAfterSend = TRUE; + break; + + case PerfectHashTableCreateServerRequestType: + case PerfectHashBulkCreateServerRequestType: + case PerfectHashBulkCreateDirectoryServerRequestType: + if (!ContextIocp || !Allocator) { + Result = E_UNEXPECTED; + break; + } + + if (Pipe->PayloadLength == 0 || + (Pipe->PayloadLength % sizeof(WCHAR)) != 0) { + Result = E_INVALIDARG; + break; + } + + if (!Pipe->PayloadBuffer) { + Result = E_UNEXPECTED; + break; + } + + CommandLine = (LPWSTR)Pipe->PayloadBuffer; + if (CommandLine[0] == L'\0') { + Result = E_INVALIDARG; + break; + } + + ArgvW = CommandLineToArgvW(CommandLine, + (PINT)&NumberOfArguments); + if (!ArgvW || NumberOfArguments == 0) { + Result = HRESULT_FROM_WIN32(GetLastError()); + if (Result == S_OK) { + Result = E_INVALIDARG; + } + break; + } + + if (RequestType == PerfectHashTableCreateServerRequestType) { + Result = PerfectHashServerDispatchTableCreateRequest( + Pipe, + NumberOfArguments, + ArgvW, + CommandLine + ); + } else if (RequestType == + PerfectHashBulkCreateDirectoryServerRequestType || + RequestType == PerfectHashBulkCreateServerRequestType) { + Result = PerfectHashServerDispatchBulkCreateDirectoryRequest( + Pipe, + NumberOfArguments, + ArgvW, + CommandLine, + &ArgvWOwned + ); + } + + break; + + default: + Result = E_INVALIDARG; + break; + } + + if (ArgvW && !ArgvWOwned) { + LocalFree(ArgvW); + } + + Pipe->ResponseHeader.Result = Result; + if (FAILED(Result)) { + HRESULT PayloadResult; + + PayloadResult = PerfectHashServerPrepareErrorPayload(Pipe, Result); + if (FAILED(PayloadResult)) { + Pipe->ResponseHeader.PayloadLength = 0; + Pipe->ResponseHeader.Flags = 0; + } + } + Pipe->BytesTransferred = 0; + + return PerfectHashServerIssueWriteResponseHeader(Pipe); +} + +static +HRESULT +PerfectHashServerIocpCompletionCallback( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG_PTR CompletionKey, + LPOVERLAPPED Overlapped, + DWORD NumberOfBytesTransferred, + BOOL Success + ) +{ + HRESULT Result; + DWORD LastError; + ULONG Expected; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_SERVER_PIPE Pipe; + + UNREFERENCED_PARAMETER(ContextIocp); + + if (!Overlapped) { + return E_POINTER; + } + + Pipe = CONTAINING_RECORD(Overlapped, + PERFECT_HASH_SERVER_PIPE, + Iocp.Overlapped); + Server = Pipe->Server; + + if ((ULONG_PTR)Pipe != CompletionKey) { + return E_UNEXPECTED; + } + + if (!Success) { + LastError = GetLastError(); + switch (LastError) { + case ERROR_BROKEN_PIPE: + case ERROR_PIPE_NOT_CONNECTED: + case ERROR_NO_DATA: + case ERROR_OPERATION_ABORTED: + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + default: + SYS_ERROR(GetQueuedCompletionStatus); + PerfectHashServerResetPipe(Pipe, TRUE); + return HRESULT_FROM_WIN32(LastError); + } + } + + switch (Pipe->State) { + case PerfectHashServerPipeStateInvalid: + PerfectHashServerResetPipe(Pipe, TRUE); + return E_UNEXPECTED; + + case PerfectHashServerPipeStateAccepting: + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueReadHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + + case PerfectHashServerPipeStateReadingHeader: + if (NumberOfBytesTransferred == 0) { + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + } + + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = sizeof(PERFECT_HASH_SERVER_REQUEST_HEADER); + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueReadHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->RequestHeader.SizeOfStruct != Expected || + Pipe->RequestHeader.Version != + PERFECT_HASH_SERVER_MESSAGE_VERSION || + !IsValidPerfectHashServerRequestType( + Pipe->RequestHeader.RequestType) || + Pipe->RequestHeader.PayloadLength > + PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + Pipe->ResponseHeader.SizeOfStruct = + sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER); + Pipe->ResponseHeader.Version = + PERFECT_HASH_SERVER_MESSAGE_VERSION; + Pipe->ResponseHeader.RequestId = + Pipe->RequestHeader.RequestId; + Pipe->ResponseHeader.Result = E_INVALIDARG; + Pipe->ResponseHeader.PayloadLength = 0; + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueWriteResponseHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + Pipe->PayloadLength = Pipe->RequestHeader.PayloadLength; + + if (Pipe->PayloadLength == 0) { + Result = PerfectHashServerDispatchRequest(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (!Pipe->PayloadBuffer || + Pipe->PayloadBufferSize < Pipe->PayloadLength) { + SIZE_T AllocationSize; + PALLOCATOR Allocator; + + Allocator = Server ? Server->Allocator : NULL; + if (!Allocator) { + PerfectHashServerResetPipe(Pipe, TRUE); + return E_UNEXPECTED; + } + + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + &Pipe->PayloadBuffer + ); + } + + AllocationSize = (SIZE_T)Pipe->PayloadLength + + sizeof(WCHAR); + if (AllocationSize < Pipe->PayloadLength) { + PerfectHashServerResetPipe(Pipe, TRUE); + return E_INVALIDARG; + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc( + Allocator, + 1, + AllocationSize + ); + if (!Pipe->PayloadBuffer) { + PerfectHashServerResetPipe(Pipe, TRUE); + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = (ULONG)AllocationSize; + } + + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueReadPayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + + case PerfectHashServerPipeStateReadingPayload: + if (NumberOfBytesTransferred == 0) { + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + } + + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = Pipe->PayloadLength; + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueReadPayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + Result = PerfectHashServerDispatchRequest(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + + case PerfectHashServerPipeStateWritingResponseHeader: + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER); + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueWriteResponseHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->ResponseHeader.PayloadLength > 0) { + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueWriteResponsePayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->ShutdownAfterSend && Server) { + if (Server->ShutdownEvent) { + SetEvent(Server->ShutdownEvent); + } + } + + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + + case PerfectHashServerPipeStateWritingResponsePayload: + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = Pipe->ResponseHeader.PayloadLength; + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueWriteResponsePayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->ShutdownAfterSend && Server) { + if (Server->ShutdownEvent) { + SetEvent(Server->ShutdownEvent); + } + } + + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + + default: + PerfectHashServerResetPipe(Pipe, TRUE); + return E_UNEXPECTED; + } +} + +#endif // PH_WINDOWS + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashServer.h b/src/PerfectHash/PerfectHashServer.h new file mode 100644 index 00000000..d5ec26ad --- /dev/null +++ b/src/PerfectHash/PerfectHashServer.h @@ -0,0 +1,152 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashServer.h + +Abstract: + + This is the private header file for the PERFECT_HASH_SERVER component of + the perfect hash library. It defines the structure, and function pointer + typedefs for private non-vtbl members. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef union _PERFECT_HASH_SERVER_STATE { + struct { + + ULONG Initialized:1; + ULONG Running:1; + ULONG Stopping:1; + ULONG Stopped:1; + + ULONG Unused:28; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_SERVER_STATE; +C_ASSERT(sizeof(PERFECT_HASH_SERVER_STATE) == sizeof(ULONG)); +typedef PERFECT_HASH_SERVER_STATE *PPERFECT_HASH_SERVER_STATE; + +typedef union _PERFECT_HASH_SERVER_FLAGS { + struct { + + // + // When set, indicates the server should only accept local clients. + // + + ULONG LocalOnly:1; + ULONG EndpointAllocated:1; + ULONG Verbose:1; + ULONG NoFileIo:1; + ULONG IocpBufferGuardPages:1; + + ULONG Unused:27; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_SERVER_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_SERVER_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_SERVER_FLAGS *PPERFECT_HASH_SERVER_FLAGS; + +struct _PERFECT_HASH_SERVER_PIPE; +typedef struct _PERFECT_HASH_SERVER_PIPE *PPERFECT_HASH_SERVER_PIPE; + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_SERVER { + + COMMON_COMPONENT_HEADER(PERFECT_HASH_SERVER); + + ULONG IocpConcurrency; + ULONG MaxWorkerThreads; + ULONG NumaNodeCount; + ULONG Padding1; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; + ULONG Padding2; + ULONG Padding3; + + // + // The underlying IOCP context used by the server. + // + + struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp; + + // + // Named pipe endpoint (if applicable). + // + + UNICODE_STRING Endpoint; + + // + // Pipe instances. + // + + PPERFECT_HASH_SERVER_PIPE Pipes; + ULONG PipeCount; + ULONG Padding4; + + HANDLE ShutdownEvent; + HANDLE StartedEvent; + + // + // Backing vtbl. + // + + PERFECT_HASH_SERVER_VTBL Interface; + +} PERFECT_HASH_SERVER; +typedef PERFECT_HASH_SERVER *PPERFECT_HASH_SERVER; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_SERVER_INITIALIZE)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_INITIALIZE *PPERFECT_HASH_SERVER_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_SERVER_RUNDOWN)( + _In_ _Post_ptr_invalid_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_RUNDOWN *PPERFECT_HASH_SERVER_RUNDOWN; + +extern PERFECT_HASH_SERVER_INITIALIZE PerfectHashServerInitialize; +extern PERFECT_HASH_SERVER_RUNDOWN PerfectHashServerRundown; + +extern PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY + PerfectHashServerSetMaximumConcurrency; +extern PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY + PerfectHashServerGetMaximumConcurrency; +extern PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS + PerfectHashServerSetMaximumThreads; +extern PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS + PerfectHashServerGetMaximumThreads; +extern PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK + PerfectHashServerSetNumaNodeMask; +extern PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK + PerfectHashServerGetNumaNodeMask; +extern PERFECT_HASH_SERVER_SET_ENDPOINT PerfectHashServerSetEndpoint; +extern PERFECT_HASH_SERVER_GET_ENDPOINT PerfectHashServerGetEndpoint; +extern PERFECT_HASH_SERVER_SET_LOCAL_ONLY PerfectHashServerSetLocalOnly; +extern PERFECT_HASH_SERVER_GET_LOCAL_ONLY PerfectHashServerGetLocalOnly; +extern PERFECT_HASH_SERVER_SET_VERBOSE PerfectHashServerSetVerbose; +extern PERFECT_HASH_SERVER_GET_VERBOSE PerfectHashServerGetVerbose; +extern PERFECT_HASH_SERVER_SET_NO_FILE_IO PerfectHashServerSetNoFileIo; +extern PERFECT_HASH_SERVER_GET_NO_FILE_IO PerfectHashServerGetNoFileIo; +extern PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerSetIocpBufferGuardPages; +extern PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerGetIocpBufferGuardPages; +extern PERFECT_HASH_SERVER_START PerfectHashServerStart; +extern PERFECT_HASH_SERVER_STOP PerfectHashServerStop; +extern PERFECT_HASH_SERVER_WAIT PerfectHashServerWait; +extern PERFECT_HASH_SERVER_SUBMIT_REQUEST PerfectHashServerSubmitRequest; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashTable.c b/src/PerfectHash/PerfectHashTable.c index d5d69f1a..4aa00222 100644 --- a/src/PerfectHash/PerfectHashTable.c +++ b/src/PerfectHash/PerfectHashTable.c @@ -152,6 +152,30 @@ Return Value: Allocator = Table->Allocator; + // + // Detach any files still linked to a parent directory. This breaks the + // directory/file reference cycle prior to releasing file objects. + // + +#define EXPAND_AS_DETACH_FILE( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Table->Name && Table->Name->ParentDirectory) { \ + RemoveResult = Table->Name->ParentDirectory->Vtbl->RemoveFile( \ + Table->Name->ParentDirectory, \ + Table->Name \ + ); \ + if (FAILED(RemoveResult)) { \ + PH_ERROR(PerfectHashDirectoryRemoveFile, RemoveResult); \ + } \ + } + + FILE_WORK_TABLE_ENTRY(EXPAND_AS_DETACH_FILE); + +#undef EXPAND_AS_DETACH_FILE + // // Free the memory used for the values array, if applicable. // diff --git a/src/PerfectHash/PerfectHashTableCreate.c b/src/PerfectHash/PerfectHashTableCreate.c index 4f41ce37..b598dee6 100644 --- a/src/PerfectHash/PerfectHashTableCreate.c +++ b/src/PerfectHash/PerfectHashTableCreate.c @@ -16,6 +16,36 @@ Module Name: --*/ #include "stdafx.h" +#ifdef PH_WINDOWS +#include "Chm01Async.h" +#endif + +// +// Async gating. +// + +#ifdef PH_WINDOWS +static +BOOLEAN +IsChm01AsyncEnabled( + VOID + ) +{ + static LONG Initialized = 0; + static BOOLEAN Enabled = FALSE; + WCHAR Buffer[32]; + DWORD Length; + + if (InterlockedCompareExchange(&Initialized, 1, 0) == 0) { + Length = GetEnvironmentVariableW(L"PERFECT_HASH_IOCP_ASYNC_CHM01", + Buffer, + ARRAYSIZE(Buffer)); + Enabled = (Length > 0); + } + + return Enabled; +} +#endif // // Cap the maximum key set size we're willing to process to 2 billion. @@ -393,7 +423,22 @@ Return Value: // Dispatch remaining creation work to the algorithm-specific routine. // +#ifdef PH_WINDOWS + if (AlgorithmId == PerfectHashChm01AlgorithmId && + Table->Context && + Table->Context->FileWorkIoCompletionPort && + (IsChm01AsyncEnabled() || + SkipThreadpoolInitialization(Table->Context))) { + Result = CreatePerfectHashTableImplChm01Async( + Table, + Table->Context->FileWorkIoCompletionPort + ); + } else { + Result = CreationRoutines[AlgorithmId](Table); + } +#else Result = CreationRoutines[AlgorithmId](Table); +#endif if (Table->OutputDirectory) { CloseResult = Table->OutputDirectory->Vtbl->Close(Table->OutputDirectory); @@ -566,6 +611,9 @@ Return Value: BOOLEAN SawResizeLimit = FALSE; BOOLEAN SawInitialResizes = FALSE; BOOLEAN SawResizeThreshold = FALSE; + BOOLEAN SawInitialPerFileConcurrency = FALSE; + BOOLEAN SawMaxPerFileConcurrency = FALSE; + BOOLEAN SawIncreaseConcurrencyAfter = FALSE; PERFECT_HASH_TABLE_BEST_COVERAGE_TYPE_ID CoverageType; PPERFECT_HASH_CONTEXT Context; PPERFECT_HASH_TABLE_CREATE_PARAMETER Param; @@ -788,6 +836,21 @@ Return Value: ); break; + case TableCreateParameterInitialPerFileConcurrencyId: + Context->InitialPerFileConcurrency = Param->AsULong; + SawInitialPerFileConcurrency = TRUE; + break; + + case TableCreateParameterMaxPerFileConcurrencyId: + Context->MaxPerFileConcurrency = Param->AsULong; + SawMaxPerFileConcurrency = TRUE; + break; + + case TableCreateParameterIncreaseConcurrencyAfterMillisecondsId: + Context->IncreaseConcurrencyAfterMilliseconds = Param->AsULong; + SawIncreaseConcurrencyAfter = TRUE; + break; + case TableCreateParameterBestCoverageTargetValueId: // @@ -826,6 +889,36 @@ Return Value: DEFAULT_MIN_NUMBER_OF_KEYS_FOR_FIND_BEST_GRAPH; } + if (!SawInitialPerFileConcurrency) { + Context->InitialPerFileConcurrency = 1; + } + + if (!SawMaxPerFileConcurrency) { + if (Context->MaximumConcurrency > 0) { + Context->MaxPerFileConcurrency = Context->MaximumConcurrency; + } else { + Context->MaxPerFileConcurrency = 1; + } + } + + if (!SawIncreaseConcurrencyAfter) { + Context->IncreaseConcurrencyAfterMilliseconds = 500; + } + + if (Context->InitialPerFileConcurrency == 0 || + Context->MaxPerFileConcurrency == 0 || + Context->InitialPerFileConcurrency > Context->MaxPerFileConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + goto Error; + } + + if (!SkipThreadpoolInitialization(Context) && + Context->MaximumConcurrency > 0 && + Context->MaxPerFileConcurrency > Context->MaximumConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + goto Error; + } + // // If the Silent flag has been supplied, also set Quiet. // diff --git a/src/PerfectHash/PerfectHashTls.h b/src/PerfectHash/PerfectHashTls.h index 491eca39..f428c1a4 100644 --- a/src/PerfectHash/PerfectHashTls.h +++ b/src/PerfectHash/PerfectHashTls.h @@ -45,11 +45,18 @@ typedef union _PERFECT_HASH_TLS_CONTEXT_FLAGS { ULONG GlobalComponentLockAcquired:1; + // + // When set, indicates new PERFECT_HASH_CONTEXT instances should skip + // threadpool initialization (IOCP-native contexts). + // + + ULONG CreateContextWithoutThreadpool:1; + // // Unused bits. (Consume these before the Unused2 bits.) // - ULONG Unused1:3; + ULONG Unused1:5; // // The following bits, when set, prevent the global component logic @@ -88,7 +95,7 @@ typedef union _PERFECT_HASH_TLS_CONTEXT_FLAGS { // Remaining unused bits. (Consume Unused1 before using these.) // - ULONG Unused2:24; + ULONG Unused2:21; }; LONG AsLong; diff --git a/src/PerfectHash/_dtoa.c b/src/PerfectHash/_dtoa.c index 74937ac3..18e84593 100644 --- a/src/PerfectHash/_dtoa.c +++ b/src/PerfectHash/_dtoa.c @@ -130,6 +130,36 @@ PALLOCATOR _dtoa_Allocator; +static +PALLOCATOR +PhGetDtoaAllocator( + VOID + ) +{ + PALLOCATOR Allocator; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + + Allocator = _dtoa_Allocator; + if (Allocator) { + return Allocator; + } + + TlsContext = PerfectHashTlsGetContext(); + if (!TlsContext) { + return NULL; + } + + if (TlsContext->Allocator) { + return TlsContext->Allocator; + } + + if (TlsContext->Context) { + return TlsContext->Context->Allocator; + } + + return NULL; +} + _Must_inspect_result_ _Ret_maybenull_ _Success_(return != 0) @@ -139,11 +169,14 @@ PhMalloc( _In_ SIZE_T Size ) { - return _dtoa_Allocator->Vtbl->Calloc( - _dtoa_Allocator, - 1, - Size - ); + PALLOCATOR Allocator; + + Allocator = PhGetDtoaAllocator(); + if (!Allocator) { + return NULL; + } + + return Allocator->Vtbl->Calloc(Allocator, 1, Size); } VOID @@ -151,7 +184,14 @@ PhFree( _Frees_ptr_opt_ PVOID Address ) { - _dtoa_Allocator->Vtbl->Free(_dtoa_Allocator, Address); + PALLOCATOR Allocator; + + Allocator = PhGetDtoaAllocator(); + if (!Allocator) { + return; + } + + Allocator->Vtbl->Free(Allocator, Address); } //#include "float.h" diff --git a/src/PerfectHash/stdafx.h b/src/PerfectHash/stdafx.h index 865334ba..242a3d28 100644 --- a/src/PerfectHash/stdafx.h +++ b/src/PerfectHash/stdafx.h @@ -166,6 +166,12 @@ Module Name: #include "Rng.h" #include "GraphCu.h" #include "PerfectHashContext.h" +#include "PerfectHashContextIocp.h" +#ifdef PH_WINDOWS +#include "PerfectHashAsync.h" +#endif +#include "PerfectHashServer.h" +#include "PerfectHashClient.h" #include "PerfectHashConstants.h" #include "PerfectHashErrorHandling.h" #include "GraphImpl.h" diff --git a/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c b/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c index 294a5b82..d59663a9 100644 --- a/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c +++ b/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c @@ -25,6 +25,7 @@ Module Name: typedef DWORD MINIDUMP_TYPE; #define MiniDumpWithDataSegs ((MINIDUMP_TYPE)0x00000001) +#define MiniDumpIgnoreInaccessibleMemory ((MINIDUMP_TYPE)0x00020000) #pragma warning(push) #pragma warning(disable: 4820) @@ -58,6 +59,79 @@ typedef BOOL (WINAPI *PMINIDUMP_WRITE_DUMP)( PMINIDUMP_CALLBACK_INFORMATION CallbackParam ); +static HMODULE BulkCreateDbgHelpModule = NULL; +static PMINIDUMP_WRITE_DUMP BulkCreateMiniDumpWriteDump = NULL; +static WCHAR BulkCreateCrashDir[MAX_PATH]; + +static +VOID +BuildCrashPath( + _In_opt_ PCWSTR CrashDir, + _In_ PCWSTR FileName, + _Out_writes_(BufferChars) PWSTR Buffer, + _In_ ULONG BufferChars + ) +{ + size_t Length; + + if (!CrashDir || !*CrashDir) { + wcscpy_s(Buffer, BufferChars, FileName); + return; + } + + Length = wcslen(CrashDir); + if (CrashDir[Length - 1] == L'\\' || CrashDir[Length - 1] == L'/') { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, FileName); + } else { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, L"\\"); + wcscat_s(Buffer, BufferChars, FileName); + } +} + +static +VOID +InitializeBulkCreateCrashDir( + VOID + ) +{ + DWORD Length; + + BulkCreateCrashDir[0] = L'\0'; + Length = GetEnvironmentVariableW(L"PH_BULK_CREATE_CRASH_DIR", + BulkCreateCrashDir, + ARRAYSIZE(BulkCreateCrashDir)); + if (Length == 0 || Length >= ARRAYSIZE(BulkCreateCrashDir)) { + BulkCreateCrashDir[0] = L'\0'; + } else { + BulkCreateCrashDir[Length] = L'\0'; + } +} + +static +VOID +PreloadBulkCreateDbgHelp( + VOID + ) +{ + if (BulkCreateMiniDumpWriteDump) { + return; + } + + BulkCreateDbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); + if (!BulkCreateDbgHelpModule) { + return; + } + +#pragma warning(push) +#pragma warning(disable: 4191) + BulkCreateMiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( + GetProcAddress(BulkCreateDbgHelpModule, "MiniDumpWriteDump") + ); +#pragma warning(pop) +} + static LONG WINAPI @@ -69,6 +143,7 @@ BulkCreateUnhandledExceptionFilter( HANDLE FileHandle = INVALID_HANDLE_VALUE; PMINIDUMP_WRITE_DUMP MiniDumpWriteDump; MINIDUMP_EXCEPTION_INFORMATION ExceptionInfo; + PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam = NULL; MINIDUMP_TYPE DumpType; BOOL DumpResult = FALSE; BOOL FallbackResult = FALSE; @@ -89,24 +164,32 @@ BulkCreateUnhandledExceptionFilter( CHAR ModulePathA[MAX_PATH]; DWORD ModulePathLength; INT ModulePathChars; + WCHAR DumpPath[MAX_PATH]; + WCHAR LogPath[MAX_PATH]; - DbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); - if (!DbgHelpModule) { - goto End; + BuildCrashPath(BulkCreateCrashDir, + L"PerfectHashBulkCreateCrash.dmp", + DumpPath, + ARRAYSIZE(DumpPath)); + + BuildCrashPath(BulkCreateCrashDir, + L"PerfectHashBulkCreateCrash.log", + LogPath, + ARRAYSIZE(LogPath)); + + if (!BulkCreateMiniDumpWriteDump) { + PreloadBulkCreateDbgHelp(); } -#pragma warning(push) -#pragma warning(disable: 4191) - MiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( - GetProcAddress(DbgHelpModule, "MiniDumpWriteDump") - ); -#pragma warning(pop) + DbgHelpModule = BulkCreateDbgHelpModule; + MiniDumpWriteDump = BulkCreateMiniDumpWriteDump; if (!MiniDumpWriteDump) { - goto End; + DumpError = GetLastError(); + goto LogOnly; } - FileHandle = CreateFileW(L"PerfectHashBulkCreateCrash.dmp", + FileHandle = CreateFileW(DumpPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -115,14 +198,18 @@ BulkCreateUnhandledExceptionFilter( NULL); if (FileHandle == INVALID_HANDLE_VALUE) { - goto End; + DumpError = GetLastError(); + goto LogOnly; } - ExceptionInfo.ThreadId = GetCurrentThreadId(); - ExceptionInfo.ExceptionPointers = ExceptionPointers; - ExceptionInfo.ClientPointers = FALSE; + if (ExceptionPointers) { + ExceptionInfo.ThreadId = GetCurrentThreadId(); + ExceptionInfo.ExceptionPointers = ExceptionPointers; + ExceptionInfo.ClientPointers = FALSE; + ExceptionParam = &ExceptionInfo; + } - DumpType = MiniDumpWithDataSegs; + DumpType = MiniDumpWithDataSegs | MiniDumpIgnoreInaccessibleMemory; ForceFallback = (GetEnvironmentVariableW( L"PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK", @@ -135,7 +222,7 @@ BulkCreateUnhandledExceptionFilter( GetCurrentProcessId(), FileHandle, DumpType, - &ExceptionInfo, + ExceptionParam, NULL, NULL); } else { @@ -147,10 +234,28 @@ BulkCreateUnhandledExceptionFilter( if (!ForceFallback) { DumpError = GetLastError(); } + + if (DumpError == ERROR_NOACCESS && ExceptionParam) { + DumpResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + NULL, + NULL, + NULL); + if (!DumpResult) { + DumpError = GetLastError(); + } + } + + if (DumpResult) { + goto LogOnly; + } + TriedFallback = TRUE; CloseHandle(FileHandle); - FileHandle = CreateFileW(L"PerfectHashBulkCreateCrash.dmp", + FileHandle = CreateFileW(DumpPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -159,12 +264,12 @@ BulkCreateUnhandledExceptionFilter( NULL); if (FileHandle != INVALID_HANDLE_VALUE) { - DumpType = 0; + DumpType = MiniDumpIgnoreInaccessibleMemory; FallbackResult = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), FileHandle, DumpType, - &ExceptionInfo, + ExceptionParam, NULL, NULL); if (!FallbackResult) { @@ -173,7 +278,9 @@ BulkCreateUnhandledExceptionFilter( } } - LogHandle = CreateFileW(L"PerfectHashBulkCreateCrash.log", +LogOnly: + + LogHandle = CreateFileW(LogPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -229,21 +336,53 @@ BulkCreateUnhandledExceptionFilter( NULL); } - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "ExceptionCode: 0x%08lX\r\n", - ExceptionPointers->ExceptionRecord->ExceptionCode); + if (ExceptionPointers) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); - WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "ExceptionAddress: 0x%p\r\n", - ExceptionPointers->ExceptionRecord->ExceptionAddress); + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); - WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } else { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } Frames = RtlCaptureStackBackTrace(0, ARRAYSIZE(BackTrace), @@ -332,9 +471,23 @@ InstallBulkCreateCrashHandler( return; } + InitializeBulkCreateCrashDir(); + PreloadBulkCreateDbgHelp(); + SetUnhandledExceptionFilter(BulkCreateUnhandledExceptionFilter); } +static +VOID +TriggerBulkCreateCrashTest( + _In_ ULONG ExceptionCode + ) +{ + UNREFERENCED_PARAMETER(ExceptionCode); + BulkCreateUnhandledExceptionFilter(NULL); + ExitProcess((ULONG)E_UNEXPECTED); +} + static VOID MaybeTriggerBulkCreateCrashTest( @@ -343,7 +496,6 @@ MaybeTriggerBulkCreateCrashTest( { DWORD Length; WCHAR Buffer[32]; - Length = GetEnvironmentVariableW(L"PH_BULK_CREATE_CRASH_TEST", Buffer, ARRAYSIZE(Buffer)); @@ -354,12 +506,13 @@ MaybeTriggerBulkCreateCrashTest( Buffer[Length] = L'\0'; + InitializeBulkCreateCrashDir(); + if (_wcsicmp(Buffer, L"AV") == 0 || _wcsicmp(Buffer, L"ACCESS_VIOLATION") == 0) { - volatile int *Ptr = (volatile int *)1; - *Ptr = 1; + TriggerBulkCreateCrashTest(STATUS_ACCESS_VIOLATION); } else { - RaiseException(0xE0000001, 0, 0, NULL); + TriggerBulkCreateCrashTest(0xE0000001); } } DECLSPEC_NORETURN diff --git a/src/PerfectHashClientExe/CMakeLists.txt b/src/PerfectHashClientExe/CMakeLists.txt new file mode 100644 index 00000000..9f5f2b20 --- /dev/null +++ b/src/PerfectHashClientExe/CMakeLists.txt @@ -0,0 +1,65 @@ +set(target PerfectHashClientExe) + +################################################################################ +# Source groups +################################################################################ +set(Header_Files + "stdafx.h" + "targetver.h" +) +source_group("Header Files" FILES ${Header_Files}) + +set(Source_Files + "PerfectHashClientExe.c" + #"stdafx.c" +) +source_group("Source Files" FILES ${Source_Files}) + +set(ALL_FILES + ${Header_Files} + ${Source_Files} +) + +set(Resource_Files + "PerfectHashClientExe.rc" +) +source_group("Resource Files" FILES ${Resource_Files}) + +if(WIN32) + list( + APPEND ALL_FILES + ${Resource_Files} + ) +endif() + +add_executable(${target} ${ALL_FILES}) + +################################################################################ +# Target +################################################################################ + +perfecthash_apply_common_settings(${target}) + +################################################################################ +# Output executable name +################################################################################ +set_target_properties(${target} PROPERTIES + OUTPUT_NAME "PerfectHashClient" +) +################################################################################ +# Compile definitions +################################################################################ + +################################################################################ +# Compile and link options +################################################################################ +################################################################################ +# Dependencies +################################################################################ +add_dependencies(${target} + PerfectHash +) + +target_link_libraries(${target} PRIVATE PerfectHash ${CMAKE_DL_LIBS}) + +# vim:set ts=8 sw=4 sts=4 tw=80 expandtab syntax=cmake : diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.c b/src/PerfectHashClientExe/PerfectHashClientExe.c new file mode 100644 index 00000000..2e456387 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.c @@ -0,0 +1,619 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + PerfectHashClientExe.c + +Abstract: + + This module implements the main entry point for the perfect hash client. + It loads the perfect hash library, creates a client instance, then attempts + to connect and optionally submit a request. + +--*/ + +#include "stdafx.h" +#include + +typedef union _PERFECT_HASH_CLIENT_CLI_FLAGS { + struct { + ULONG EndpointPresent:1; + ULONG CommandLinePresent:1; + ULONG RequestTypePresent:1; + ULONG WaitForServer:1; + ULONG ConnectTimeoutPresent:1; + ULONG Unused:27; + }; + ULONG AsULong; +} PERFECT_HASH_CLIENT_CLI_FLAGS; +typedef PERFECT_HASH_CLIENT_CLI_FLAGS *PPERFECT_HASH_CLIENT_CLI_FLAGS; + +typedef struct _PERFECT_HASH_CLIENT_CLI_OPTIONS { + UNICODE_STRING Endpoint; + UNICODE_STRING CommandLine; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + ULONG ConnectTimeoutInMilliseconds; + PERFECT_HASH_CLIENT_CLI_FLAGS Flags; + ULONG Padding1; +} PERFECT_HASH_CLIENT_CLI_OPTIONS; +typedef PERFECT_HASH_CLIENT_CLI_OPTIONS *PPERFECT_HASH_CLIENT_CLI_OPTIONS; + +static +VOID +PrintUsage( + VOID + ) +{ + wprintf(L"Usage: PerfectHashClient [--Endpoint=] " + L"[--Shutdown|--TableCreate=|--BulkCreate=|" + L"--BulkCreateDirectory=|--Ping]\n"); + wprintf(L" --TableCreate= PerfectHashCreate-style arguments\n"); + wprintf(L" --BulkCreate= PerfectHashBulkCreate-style arguments\n"); + wprintf(L" --BulkCreateDirectory= BulkCreate args with single token\n"); + wprintf(L" --Ping Wait for server readiness\n"); + wprintf(L" --WaitForServer Wait for server to appear\n"); + wprintf(L" --ConnectTimeout= Cap wait time for --WaitForServer\n"); +} + +static +HRESULT +ParseClientArgs( + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _Inout_ PPERFECT_HASH_CLIENT_CLI_OPTIONS Options + ) +{ + ULONG Index; + + Options->Endpoint.Buffer = NULL; + Options->Endpoint.Length = 0; + Options->Endpoint.MaximumLength = 0; + Options->Flags.AsULong = 0; + Options->CommandLine.Buffer = NULL; + Options->CommandLine.Length = 0; + Options->CommandLine.MaximumLength = 0; + Options->RequestType = PerfectHashNullServerRequestType; + Options->ConnectTimeoutInMilliseconds = 0; + + for (Index = 1; Index < NumberOfArguments; Index++) { + PCWSTR Arg = ArgvW[Index]; + + if (!Arg) { + continue; + } + + if (Arg[0] != L'-' || Arg[1] != L'-') { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Arg += 2; + + if (_wcsicmp(Arg, L"Help") == 0 || + _wcsicmp(Arg, L"?") == 0) { + PrintUsage(); + return S_FALSE; + } + + if (_wcsnicmp(Arg, L"Endpoint=", 9) == 0) { + PCWSTR Value; + ULONG Length; + + Value = Arg + 9; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->Endpoint.Buffer = (PWSTR)Value; + Options->Endpoint.Length = (USHORT)Length; + Options->Endpoint.MaximumLength = (USHORT)Length + sizeof(WCHAR); + Options->Flags.EndpointPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"Shutdown") == 0) { + if (Options->Flags.RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + Options->RequestType = PerfectHashShutdownServerRequestType; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"Ping") == 0) { + if (Options->Flags.RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + Options->RequestType = PerfectHashPingServerRequestType; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"WaitForServer") == 0) { + Options->Flags.WaitForServer = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"ConnectTimeout=", 15) == 0) { + PCWSTR Value; + ULONG Timeout; + + Value = Arg + 15; + if (!Value || *Value == L'\0') { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Timeout = wcstoul(Value, NULL, 10); + if (Timeout == 0) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Options->ConnectTimeoutInMilliseconds = Timeout; + Options->Flags.ConnectTimeoutPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"TableCreate=", 12) == 0) { + PCWSTR Value; + ULONG Length; + + if (Options->Flags.RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Value = Arg + 12; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->CommandLine.Buffer = (PWSTR)Value; + Options->CommandLine.Length = (USHORT)Length; + Options->CommandLine.MaximumLength = (USHORT)Length + + sizeof(WCHAR); + Options->Flags.CommandLinePresent = TRUE; + Options->RequestType = PerfectHashTableCreateServerRequestType; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"BulkCreate=", 11) == 0) { + PCWSTR Value; + ULONG Length; + + if (Options->Flags.RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Value = Arg + 11; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->CommandLine.Buffer = (PWSTR)Value; + Options->CommandLine.Length = (USHORT)Length; + Options->CommandLine.MaximumLength = (USHORT)Length + + sizeof(WCHAR); + Options->Flags.CommandLinePresent = TRUE; + Options->RequestType = PerfectHashBulkCreateServerRequestType; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"BulkCreateDirectory=", 20) == 0) { + PCWSTR Value; + ULONG Length; + + if (Options->Flags.RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Value = Arg + 20; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->CommandLine.Buffer = (PWSTR)Value; + Options->CommandLine.Length = (USHORT)Length; + Options->CommandLine.MaximumLength = (USHORT)Length + + sizeof(WCHAR); + Options->Flags.CommandLinePresent = TRUE; + Options->RequestType = + PerfectHashBulkCreateDirectoryServerRequestType; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + return PH_E_INVALID_COMMANDLINE_ARG; + } + + if (Options->Flags.RequestTypePresent) { + if (Options->RequestType == PerfectHashShutdownServerRequestType || + Options->RequestType == PerfectHashPingServerRequestType) { + if (Options->Flags.CommandLinePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + } else if (!Options->Flags.CommandLinePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + } + + if (Options->Flags.ConnectTimeoutPresent && !Options->Flags.WaitForServer) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + return S_OK; +} + +static +HRESULT +PerfectHashClientConnectWithWait( + _In_ PPERFECT_HASH_CLIENT Client, + _In_opt_ PCUNICODE_STRING Endpoint, + _In_ BOOLEAN WaitForServer, + _In_ BOOLEAN TimeoutPresent, + _In_ ULONG TimeoutInMilliseconds + ) +{ + HRESULT Result; + ULONGLONG StartTicks; + + if (!WaitForServer) { + return Client->Vtbl->Connect(Client, Endpoint); + } + + StartTicks = GetTickCount64(); + + for (;;) { + Result = Client->Vtbl->Connect(Client, Endpoint); + if (SUCCEEDED(Result)) { + return S_OK; + } + + if (Result != HRESULT_FROM_WIN32(ERROR_PIPE_BUSY) && + Result != HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) { + return Result; + } + + if (TimeoutPresent) { + ULONGLONG Elapsed = GetTickCount64() - StartTicks; + if (Elapsed >= TimeoutInMilliseconds) { + return HRESULT_FROM_WIN32(WAIT_TIMEOUT); + } + } + + Sleep(50); + } +} + +static +HRESULT +PerfectHashClientPing( + _In_ PPERFECT_HASH_CLIENT Client + ) +{ + HRESULT Result; + ULONG ResponseFlags = 0; + UNICODE_STRING ResponsePayload; + PERFECT_HASH_SERVER_REQUEST Request = { 0 }; + + Request.SizeOfStruct = sizeof(Request); + Request.RequestType = PerfectHashPingServerRequestType; + Request.RequestId = 1; + + Result = Client->Vtbl->SubmitRequest(Client, &Request); + if (FAILED(Result)) { + return Result; + } + + Result = Client->Vtbl->GetLastResponse(Client, + &ResponsePayload, + &ResponseFlags); + if (FAILED(Result)) { + return Result; + } + + if (!(ResponseFlags & PERFECT_HASH_SERVER_RESPONSE_FLAG_PONG)) { + return E_UNEXPECTED; + } + + if (!ResponsePayload.Buffer || + _wcsicmp(ResponsePayload.Buffer, L"PONG") != 0) { + return E_UNEXPECTED; + } + + return S_OK; +} + +static +HRESULT +PerfectHashClientWaitForBulkCreateToken( + _In_ PPERFECT_HASH_CLIENT Client, + _Out_opt_ PPERFECT_HASH_SERVER_BULK_RESULT BulkResult + ) +{ + HRESULT Result; + DWORD WaitResult; + int Count; + ULONG ResponseFlags; + ULONGLONG EventHandleValue = 0; + ULONGLONG ResultHandleValue = 0; + HANDLE EventHandle = NULL; + HANDLE ResultHandle = NULL; + PPERFECT_HASH_SERVER_BULK_RESULT Mapping = NULL; + UNICODE_STRING ResponsePayload; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (ARGUMENT_PRESENT(BulkResult)) { + ZeroMemory(BulkResult, sizeof(*BulkResult)); + } + + Result = Client->Vtbl->GetLastResponse(Client, + &ResponsePayload, + &ResponseFlags); + if (FAILED(Result)) { + return Result; + } + + if (!(ResponseFlags & + PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN)) { + return E_UNEXPECTED; + } + + if (!ResponsePayload.Buffer || ResponsePayload.Length == 0) { + return E_UNEXPECTED; + } + + Count = swscanf_s(ResponsePayload.Buffer, + PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT, + &EventHandleValue, + &ResultHandleValue); + if (Count != 2) { + return E_INVALIDARG; + } + + if (EventHandleValue == 0 || ResultHandleValue == 0) { + return E_INVALIDARG; + } + + EventHandle = (HANDLE)(ULONG_PTR)EventHandleValue; + ResultHandle = (HANDLE)(ULONG_PTR)ResultHandleValue; + + WaitResult = WaitForSingleObject(EventHandle, INFINITE); + if (WaitResult != WAIT_OBJECT_0) { + if (WaitResult == WAIT_FAILED) { + Result = HRESULT_FROM_WIN32(GetLastError()); + } else { + Result = HRESULT_FROM_WIN32(WaitResult); + } + goto End; + } + + Mapping = (PPERFECT_HASH_SERVER_BULK_RESULT)( + MapViewOfFile(ResultHandle, FILE_MAP_READ, 0, 0, 0) + ); + if (!Mapping) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto End; + } + + if (Mapping->SizeOfStruct != sizeof(*Mapping) || + Mapping->Version != PERFECT_HASH_SERVER_BULK_RESULT_VERSION) { + Result = E_UNEXPECTED; + goto End; + } + + if (ARGUMENT_PRESENT(BulkResult)) { + *BulkResult = *Mapping; + Result = S_OK; + } else { + Result = Mapping->Result; + } + +End: + + if (Mapping) { + UnmapViewOfFile(Mapping); + } + + if (ResultHandle) { + CloseHandle(ResultHandle); + } + + if (EventHandle) { + CloseHandle(EventHandle); + } + + return Result; +} + +#ifdef PH_WINDOWS +DECLSPEC_NORETURN +VOID +WINAPI +mainCRTStartup( + VOID + ) +{ + HMODULE Module = NULL; + HRESULT Result = S_OK; + LPWSTR *ArgvW; + LPWSTR CommandLineW; + PICLASSFACTORY ClassFactory; + PPERFECT_HASH_CLIENT Client = NULL; + PERFECT_HASH_CLIENT_CLI_OPTIONS Options = { 0 }; + PERFECT_HASH_SERVER_REQUEST Request = { 0 }; + PERFECT_HASH_SERVER_BULK_RESULT BulkResult = { 0 }; + PPERFECT_HASH_PRINT_ERROR PerfectHashPrintError = NULL; + PPERFECT_HASH_PRINT_MESSAGE PerfectHashPrintMessage; + PICLASSFACTORY_CREATE_INSTANCE CreateInstance; + INT NumberOfArguments = 0; + + CommandLineW = GetCommandLineW(); + ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); + + Result = PerfectHashBootstrap(&ClassFactory, + &PerfectHashPrintError, + &PerfectHashPrintMessage, + &Module); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashBootstrap, Result); + } + goto Error; + } + + CreateInstance = ClassFactory->Vtbl->CreateInstance; + + Result = CreateInstance(ClassFactory, + NULL, + &IID_PERFECT_HASH_CLIENT, + &Client); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashClientCreateInstance, Result); + } + goto Error; + } + + Result = ParseClientArgs(NumberOfArguments, ArgvW, &Options); + if (Result == S_FALSE) { + Result = S_OK; + goto End; + } else if (FAILED(Result)) { + PrintUsage(); + goto Error; + } + + Result = PerfectHashClientConnectWithWait( + Client, + Options.Flags.EndpointPresent ? &Options.Endpoint : NULL, + (BOOLEAN)!!Options.Flags.WaitForServer, + (BOOLEAN)!!Options.Flags.ConnectTimeoutPresent, + Options.ConnectTimeoutInMilliseconds + ); + if (FAILED(Result)) { + goto Error; + } + + if (Options.Flags.WaitForServer || + Options.RequestType == PerfectHashPingServerRequestType) { + Result = PerfectHashClientPing(Client); + if (FAILED(Result)) { + goto Error; + } + + if (Options.RequestType == PerfectHashPingServerRequestType || + !Options.Flags.RequestTypePresent) { + goto End; + } + + Client->Vtbl->Disconnect(Client); + Result = PerfectHashClientConnectWithWait( + Client, + Options.Flags.EndpointPresent ? &Options.Endpoint : NULL, + (BOOLEAN)!!Options.Flags.WaitForServer, + (BOOLEAN)!!Options.Flags.ConnectTimeoutPresent, + Options.ConnectTimeoutInMilliseconds + ); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.Flags.RequestTypePresent) { + Request.SizeOfStruct = sizeof(Request); + Request.RequestType = Options.RequestType; + Request.RequestId = 1; + if (Options.Flags.CommandLinePresent) { + Request.CommandLine = Options.CommandLine; + } + + Result = Client->Vtbl->SubmitRequest(Client, &Request); + if (FAILED(Result)) { + UNICODE_STRING ResponsePayload; + ULONG ResponseFlags = 0; + HRESULT PayloadResult; + + PayloadResult = Client->Vtbl->GetLastResponse(Client, + &ResponsePayload, + &ResponseFlags); + if (SUCCEEDED(PayloadResult) && + ResponsePayload.Buffer && + (ResponseFlags & + PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE)) { + wprintf(L"%.*s\n", + (int)(ResponsePayload.Length / sizeof(WCHAR)), + ResponsePayload.Buffer); + } + goto Error; + } + + if (Options.RequestType == + PerfectHashBulkCreateDirectoryServerRequestType) { + Result = PerfectHashClientWaitForBulkCreateToken(Client, + &BulkResult); + if (FAILED(Result)) { + goto Error; + } + + Result = BulkResult.Result; + if (FAILED(Result)) { + if (BulkResult.TotalFiles || BulkResult.FailedFiles) { + wprintf(L"Bulk result: total=%lu succeeded=%lu failed=%lu " + L"first=0x%08lX\n", + BulkResult.TotalFiles, + BulkResult.SucceededFiles, + BulkResult.FailedFiles, + (ULONG)BulkResult.FirstFailure); + } + if (PerfectHashPrintError != NULL) { + PH_ERROR(BulkCreateDirectory, Result); + } + } + goto End; + } + } + + goto End; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashClient, Result); + } + +End: + + if (Client) { + Client->Vtbl->Release(Client); + } + + if (ClassFactory) { + ClassFactory->Vtbl->Release(ClassFactory); + } + + if (Module) { + FreeLibrary(Module); + } + + ExitProcess((ULONG)Result); +} + +#else // PH_WINDOWS + +int +main( + int NumberOfArguments, + char **ArgvA + ) +{ + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + return 0; +} + +#endif // PH_WINDOWS + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.rc b/src/PerfectHashClientExe/PerfectHashClientExe.rc new file mode 100644 index 00000000..f743c6b0 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.rc @@ -0,0 +1,49 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + PerfectHashClientExe.rc + +Abstract: + + This is the main resource compiler for the perfect hash client executable. + It is responsible for providing version information. + +--*/ + +#include "../PerfectHashVersion.rc" + +#define VER_FILEDESCRIPTION_STR "Perfect Hash Client Application" +#define VER_ORIGINALFILENAME_STR "PerfectHashClient.exe" +#define VER_INTERNALNAME_STR VER_ORIGINALFILENAME_STR + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_PRODUCTVERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +FILEFLAGS VER_FILEFLAGS +FILEOS VOS_NT +FILETYPE VFT_APP +FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + BEGIN + VALUE "CompanyName", VER_COMPANYNAME_STR + VALUE "FileDescription", VER_FILEDESCRIPTION_STR + VALUE "LegalCopyright", VER_LEGALCOPYRIGHT_STR + VALUE "OriginalFilename", VER_ORIGINALFILENAME_STR + VALUE "ProductName", VER_PRODUCTNAME_STR + VALUE "ProductVersion", VER_PRODUCTVERSION_STR + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END + diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj new file mode 100644 index 00000000..9ff29072 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj @@ -0,0 +1,71 @@ + + + + + Debug + x64 + + + Release + x64 + + + PGInstrument + x64 + + + PGUpdate + x64 + + + PGOptimize + x64 + + + + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198} + PerfectHashClientExe + 10.0 + PerfectHashClientExe + + + PerfectHashClient + .exe + + + + Application + + + + + + + + + + + + + + false + + + + + + + + + Create + + + + + + + + + + + diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters new file mode 100644 index 00000000..42882767 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters @@ -0,0 +1,41 @@ + + + + + {F15B494D-F723-4443-9E03-3A04103D2DB9} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {AC13CE39-13FD-4BA5-9F10-CAAACA7CC3EE} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {6F13BC5B-A638-48FA-9F17-58A620D37510} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + + + Resource Files + + + Resource Files + + + diff --git a/src/PerfectHashClientExe/stdafx.c b/src/PerfectHashClientExe/stdafx.c new file mode 100644 index 00000000..fd4f341c --- /dev/null +++ b/src/PerfectHashClientExe/stdafx.c @@ -0,0 +1 @@ +#include "stdafx.h" diff --git a/src/PerfectHashClientExe/stdafx.h b/src/PerfectHashClientExe/stdafx.h new file mode 100644 index 00000000..95bfe414 --- /dev/null +++ b/src/PerfectHashClientExe/stdafx.h @@ -0,0 +1,21 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + stdafx.h + +Abstract: + + This is the precompiled header file for the perfect hash client + executable. + +--*/ + +#pragma once + +#include +#include + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashClientExe/targetver.h b/src/PerfectHashClientExe/targetver.h new file mode 100644 index 00000000..cfae052b --- /dev/null +++ b/src/PerfectHashClientExe/targetver.h @@ -0,0 +1 @@ +#include diff --git a/src/PerfectHashCreateExe/CMakeLists.txt b/src/PerfectHashCreateExe/CMakeLists.txt index 23bb578c..db672b36 100644 --- a/src/PerfectHashCreateExe/CMakeLists.txt +++ b/src/PerfectHashCreateExe/CMakeLists.txt @@ -60,6 +60,10 @@ add_dependencies(${target} PerfectHash ) +if(WIN32) + target_link_libraries(${target} PRIVATE Dbghelp) +endif() + target_link_libraries(${target} PRIVATE PerfectHash ${CMAKE_DL_LIBS}) # vim:set ts=8 sw=4 sts=4 tw=80 expandtab syntax=cmake : diff --git a/src/PerfectHashCreateExe/PerfectHashCreateExe.c b/src/PerfectHashCreateExe/PerfectHashCreateExe.c index b7fe31f4..14e545cb 100644 --- a/src/PerfectHashCreateExe/PerfectHashCreateExe.c +++ b/src/PerfectHashCreateExe/PerfectHashCreateExe.c @@ -17,6 +17,251 @@ Module Name: #include "stdafx.h" +#include +#include + +#ifdef PH_WINDOWS +#pragma warning(push) +#pragma warning(disable: 4820 4255) +#include +#pragma warning(pop) +#endif + +#ifdef PH_WINDOWS +#define PH_CREATE_MAX_STACK_FRAMES 64 + +static +VOID +PerfectHashCreateWriteString( + _In_z_ PCSTR String + ) +/*++ + +Routine Description: + + Writes a null-terminated string to STDERR. + +Arguments: + + String - Supplies a pointer to a NULL-terminated string. + +Return Value: + + None. + +--*/ +{ + DWORD BytesWritten; + size_t Length; + HANDLE ErrorHandle; + + ErrorHandle = GetStdHandle(STD_ERROR_HANDLE); + if (ErrorHandle == INVALID_HANDLE_VALUE || ErrorHandle == NULL) { + return; + } + + Length = strlen(String); + if (Length == 0 || Length > MAXDWORD) { + return; + } + + WriteFile(ErrorHandle, + String, + (DWORD)Length, + &BytesWritten, + NULL); +} + +static +VOID +PerfectHashCreateDumpStack( + _In_ PCONTEXT Context + ) +/*++ + +Routine Description: + + Dumps a best-effort stack trace to STDERR using DbgHelp. Intended for + diagnosing unhandled exceptions during table creation in debug builds. + +Arguments: + + Context - Supplies a pointer to a CONTEXT record. + +Return Value: + + None. + +--*/ +{ + HANDLE Process; + HANDLE Thread; + DWORD MachineType; + DWORD64 Displacement; + DWORD LineDisplacement; + ULONG FrameIndex; + STACKFRAME64 Frame = { 0 }; + PCSTR SymbolPath = NULL; + CHAR ModulePath[MAX_PATH]; + CHAR *LastSlash; + CHAR Buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME] = { 0 }; + PSYMBOL_INFO Symbol = (PSYMBOL_INFO)Buffer; + IMAGEHLP_LINE64 Line = { 0 }; + CHAR LineBuffer[512]; + + if (!ARGUMENT_PRESENT(Context)) { + return; + } + + Process = GetCurrentProcess(); + Thread = GetCurrentThread(); + + SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_UNDNAME | SYMOPT_LOAD_LINES); + + ModulePath[0] = '\0'; + if (GetModuleFileNameA(NULL, ModulePath, ARRAYSIZE(ModulePath)) != 0) { + LastSlash = strrchr(ModulePath, '\\'); + if (LastSlash) { + *LastSlash = '\0'; + SymbolPath = ModulePath; + } + } + + if (!SymInitialize(Process, SymbolPath, TRUE)) { + return; + } + +#if defined(_M_AMD64) || defined(_M_X64) + MachineType = IMAGE_FILE_MACHINE_AMD64; + Frame.AddrPC.Offset = Context->Rip; + Frame.AddrFrame.Offset = Context->Rbp; + Frame.AddrStack.Offset = Context->Rsp; +#elif defined(_M_IX86) + MachineType = IMAGE_FILE_MACHINE_I386; + Frame.AddrPC.Offset = Context->Eip; + Frame.AddrFrame.Offset = Context->Ebp; + Frame.AddrStack.Offset = Context->Esp; +#else + MachineType = IMAGE_FILE_MACHINE_UNKNOWN; +#endif + + Frame.AddrPC.Mode = AddrModeFlat; + Frame.AddrFrame.Mode = AddrModeFlat; + Frame.AddrStack.Mode = AddrModeFlat; + + PerfectHashCreateWriteString("Unhandled exception stack trace:\n"); + + for (FrameIndex = 0; FrameIndex < PH_CREATE_MAX_STACK_FRAMES; FrameIndex++) { + + if (!StackWalk64(MachineType, + Process, + Thread, + &Frame, + Context, + NULL, + SymFunctionTableAccess64, + SymGetModuleBase64, + NULL)) { + break; + } + + if (Frame.AddrPC.Offset == 0) { + break; + } + + Symbol->SizeOfStruct = sizeof(*Symbol); + Symbol->MaxNameLen = MAX_SYM_NAME; + Displacement = 0; + + Line.SizeOfStruct = sizeof(Line); + LineDisplacement = 0; + + if (SymFromAddr(Process, + Frame.AddrPC.Offset, + &Displacement, + Symbol)) { + + if (SymGetLineFromAddr64(Process, + Frame.AddrPC.Offset, + &LineDisplacement, + &Line)) { + sprintf_s(LineBuffer, + sizeof(LineBuffer), + " %02lu: %s + 0x%llx (%s:%lu)\n", + FrameIndex, + Symbol->Name, + (unsigned long long)Displacement, + Line.FileName, + Line.LineNumber); + } else { + sprintf_s(LineBuffer, + sizeof(LineBuffer), + " %02lu: %s + 0x%llx\n", + FrameIndex, + Symbol->Name, + (unsigned long long)Displacement); + } + + } else { + sprintf_s(LineBuffer, + sizeof(LineBuffer), + " %02lu: 0x%llx\n", + FrameIndex, + (unsigned long long)Frame.AddrPC.Offset); + } + + PerfectHashCreateWriteString(LineBuffer); + } + + SymCleanup(Process); +} + +static +LONG +WINAPI +PerfectHashCreateUnhandledExceptionFilter( + _In_ PEXCEPTION_POINTERS ExceptionPointers + ) +/*++ + +Routine Description: + + Unhandled exception filter that logs the exception code and stack trace. + +Arguments: + + ExceptionPointers - Supplies a pointer to the exception information. + +Return Value: + + EXCEPTION_EXECUTE_HANDLER. + +--*/ +{ + CHAR Buffer[128]; + ULONG Code; + + if (!ARGUMENT_PRESENT(ExceptionPointers) || + !ARGUMENT_PRESENT(ExceptionPointers->ExceptionRecord)) { + return EXCEPTION_EXECUTE_HANDLER; + } + + Code = ExceptionPointers->ExceptionRecord->ExceptionCode; + + sprintf_s(Buffer, + sizeof(Buffer), + "Unhandled exception: 0x%08lX\n", + Code); + PerfectHashCreateWriteString(Buffer); + + if (ARGUMENT_PRESENT(ExceptionPointers->ContextRecord)) { + PerfectHashCreateDumpStack(ExceptionPointers->ContextRecord); + } + + return EXCEPTION_EXECUTE_HANDLER; +} +#endif + // // Main entry point. // @@ -43,6 +288,10 @@ mainCRTStartup( CommandLineW = GetCommandLineW(); ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); +#ifdef PH_WINDOWS + SetUnhandledExceptionFilter(PerfectHashCreateUnhandledExceptionFilter); +#endif + Result = PerfectHashBootstrap(&ClassFactory, &PerfectHashPrintError, &PerfectHashPrintMessage, diff --git a/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj b/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj index 8a69ee39..64795d5e 100644 --- a/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj +++ b/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj @@ -50,6 +50,9 @@ false + + Dbghelp.lib;%(AdditionalDependencies) + diff --git a/src/PerfectHashServerExe/CMakeLists.txt b/src/PerfectHashServerExe/CMakeLists.txt new file mode 100644 index 00000000..8a087229 --- /dev/null +++ b/src/PerfectHashServerExe/CMakeLists.txt @@ -0,0 +1,65 @@ +set(target PerfectHashServerExe) + +################################################################################ +# Source groups +################################################################################ +set(Header_Files + "stdafx.h" + "targetver.h" +) +source_group("Header Files" FILES ${Header_Files}) + +set(Source_Files + "PerfectHashServerExe.c" + #"stdafx.c" +) +source_group("Source Files" FILES ${Source_Files}) + +set(ALL_FILES + ${Header_Files} + ${Source_Files} +) + +set(Resource_Files + "PerfectHashServerExe.rc" +) +source_group("Resource Files" FILES ${Resource_Files}) + +if(WIN32) + list( + APPEND ALL_FILES + ${Resource_Files} + ) +endif() + +add_executable(${target} ${ALL_FILES}) + +################################################################################ +# Target +################################################################################ + +perfecthash_apply_common_settings(${target}) + +################################################################################ +# Output executable name +################################################################################ +set_target_properties(${target} PROPERTIES + OUTPUT_NAME "PerfectHashServer" +) +################################################################################ +# Compile definitions +################################################################################ + +################################################################################ +# Compile and link options +################################################################################ +################################################################################ +# Dependencies +################################################################################ +add_dependencies(${target} + PerfectHash +) + +target_link_libraries(${target} PRIVATE PerfectHash ${CMAKE_DL_LIBS}) + +# vim:set ts=8 sw=4 sts=4 tw=80 expandtab syntax=cmake : diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.c b/src/PerfectHashServerExe/PerfectHashServerExe.c new file mode 100644 index 00000000..fc7e9c08 --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.c @@ -0,0 +1,1079 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashServerExe.c + +Abstract: + + This module implements the main entry point for the perfect hash server. + It loads the perfect hash library, creates a server instance, applies any + command line options, then starts and waits on the server. + +--*/ + +#include "stdafx.h" +#include +#include +#include + +#ifdef PH_WINDOWS + +typedef DWORD MINIDUMP_TYPE; + +#define MiniDumpWithDataSegs ((MINIDUMP_TYPE)0x00000001) +#define MiniDumpIgnoreInaccessibleMemory ((MINIDUMP_TYPE)0x00020000) + +#pragma warning(push) +#pragma warning(disable: 4820) + +typedef struct _MINIDUMP_EXCEPTION_INFORMATION { + DWORD ThreadId; + PEXCEPTION_POINTERS ExceptionPointers; + BOOL ClientPointers; +} MINIDUMP_EXCEPTION_INFORMATION, *PMINIDUMP_EXCEPTION_INFORMATION; + +#pragma warning(pop) + +typedef PVOID PMINIDUMP_USER_STREAM_INFORMATION; +typedef PVOID PMINIDUMP_CALLBACK_INFORMATION; + +#endif + +typedef struct _PERFECT_HASH_SERVER_CLI_OPTIONS { + UNICODE_STRING Endpoint; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; + ULONG IocpConcurrency; + ULONG MaxThreads; + BOOLEAN EndpointPresent; + BOOLEAN IocpConcurrencyPresent; + BOOLEAN MaxThreadsPresent; + BOOLEAN NumaNodeMaskPresent; + BOOLEAN LocalOnly; + BOOLEAN LocalOnlyPresent; + BOOLEAN Verbose; + BOOLEAN VerbosePresent; + BOOLEAN NoFileIo; + BOOLEAN NoFileIoPresent; + BOOLEAN IocpBufferGuardPages; + BOOLEAN IocpBufferGuardPagesPresent; + UCHAR Padding1[4]; +} PERFECT_HASH_SERVER_CLI_OPTIONS; +typedef PERFECT_HASH_SERVER_CLI_OPTIONS *PPERFECT_HASH_SERVER_CLI_OPTIONS; + +static PPERFECT_HASH_SERVER GlobalServer; + +#ifdef PH_WINDOWS + +typedef BOOL (WINAPI *PMINIDUMP_WRITE_DUMP)( + HANDLE Process, + DWORD ProcessId, + HANDLE File, + MINIDUMP_TYPE DumpType, + PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam, + PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam, + PMINIDUMP_CALLBACK_INFORMATION CallbackParam + ); + +static HMODULE ServerDbgHelpModule = NULL; +static PMINIDUMP_WRITE_DUMP ServerMiniDumpWriteDump = NULL; +static WCHAR ServerCrashDir[MAX_PATH]; + +static +VOID +BuildCrashPath( + _In_opt_ PCWSTR CrashDir, + _In_ PCWSTR FileName, + _Out_writes_(BufferChars) PWSTR Buffer, + _In_ ULONG BufferChars + ) +{ + size_t Length; + + if (!CrashDir || !*CrashDir) { + wcscpy_s(Buffer, BufferChars, FileName); + return; + } + + Length = wcslen(CrashDir); + if (CrashDir[Length - 1] == L'\\' || CrashDir[Length - 1] == L'/') { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, FileName); + } else { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, L"\\"); + wcscat_s(Buffer, BufferChars, FileName); + } +} + +static +VOID +InitializeServerCrashDir( + VOID + ) +{ + DWORD Length; + + ServerCrashDir[0] = L'\0'; + Length = GetEnvironmentVariableW(L"PH_SERVER_CRASH_DIR", + ServerCrashDir, + ARRAYSIZE(ServerCrashDir)); + if (Length == 0 || Length >= ARRAYSIZE(ServerCrashDir)) { + ServerCrashDir[0] = L'\0'; + } else { + ServerCrashDir[Length] = L'\0'; + } +} + +static +VOID +PreloadServerDbgHelp( + VOID + ) +{ + if (ServerMiniDumpWriteDump) { + return; + } + + ServerDbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); + if (!ServerDbgHelpModule) { + return; + } + +#pragma warning(push) +#pragma warning(disable: 4191) + ServerMiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( + GetProcAddress(ServerDbgHelpModule, "MiniDumpWriteDump") + ); +#pragma warning(pop) +} + +static +LONG +WINAPI +PerfectHashServerUnhandledExceptionFilter( + _In_ struct _EXCEPTION_POINTERS *ExceptionPointers + ) +{ + HMODULE DbgHelpModule; + HANDLE FileHandle = INVALID_HANDLE_VALUE; + PMINIDUMP_WRITE_DUMP MiniDumpWriteDump; + MINIDUMP_EXCEPTION_INFORMATION ExceptionInfo; + PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam = NULL; + MINIDUMP_TYPE DumpType; + BOOL DumpResult = FALSE; + BOOL FallbackResult = FALSE; + BOOL TriedFallback = FALSE; + DWORD DumpError = 0; + DWORD FallbackError = 0; + HANDLE LogHandle = INVALID_HANDLE_VALUE; + DWORD BytesWritten; + PVOID BackTrace[64]; + USHORT Index; + USHORT Frames; + CHAR Buffer[256]; + HMODULE Module; + HMODULE ExeBase; + ULONG_PTR Offset; + BOOLEAN ForceFallback = FALSE; + WCHAR ModulePathW[MAX_PATH]; + CHAR ModulePathA[MAX_PATH]; + DWORD ModulePathLength; + INT ModulePathChars; + WCHAR DumpPath[MAX_PATH]; + WCHAR LogPath[MAX_PATH]; + + BuildCrashPath(ServerCrashDir, + L"PerfectHashServerCrash.dmp", + DumpPath, + ARRAYSIZE(DumpPath)); + + BuildCrashPath(ServerCrashDir, + L"PerfectHashServerCrash.log", + LogPath, + ARRAYSIZE(LogPath)); + + LogHandle = CreateFileW(LogPath, + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (LogHandle != INVALID_HANDLE_VALUE) { + if (ExceptionPointers) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + Module = NULL; + if (GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)ExceptionPointers->ExceptionRecord->ExceptionAddress, + &Module)) { + Offset = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress - + (ULONG_PTR)Module; + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionModule: 0x%p Offset: 0x%Ix\r\n", + Module, + Offset); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + } else { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ThreadId: %lu\r\n", + GetCurrentThreadId()); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + if (!ServerMiniDumpWriteDump) { + PreloadServerDbgHelp(); + } + + DbgHelpModule = ServerDbgHelpModule; + MiniDumpWriteDump = ServerMiniDumpWriteDump; + + if (!MiniDumpWriteDump) { + DumpError = GetLastError(); + goto LogOnly; + } + + FileHandle = CreateFileW(DumpPath, + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (FileHandle == INVALID_HANDLE_VALUE) { + DumpError = GetLastError(); + goto LogOnly; + } + + if (ExceptionPointers) { + ExceptionInfo.ThreadId = GetCurrentThreadId(); + ExceptionInfo.ExceptionPointers = ExceptionPointers; + ExceptionInfo.ClientPointers = FALSE; + ExceptionParam = &ExceptionInfo; + } + + DumpType = MiniDumpWithDataSegs | MiniDumpIgnoreInaccessibleMemory; + + ForceFallback = (GetEnvironmentVariableW( + L"PH_SERVER_MINIDUMP_FORCE_FALLBACK", + NULL, + 0 + ) > 0); + + if (!ForceFallback) { + DumpResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + ExceptionParam, + NULL, + NULL); + } else { + DumpResult = FALSE; + DumpError = ERROR_NOACCESS; + } + + if (!DumpResult) { + if (!ForceFallback) { + DumpError = GetLastError(); + } + + if (DumpError == ERROR_NOACCESS && ExceptionParam) { + DumpResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + NULL, + NULL, + NULL); + if (!DumpResult) { + DumpError = GetLastError(); + } + } + + if (DumpResult) { + goto LogOnly; + } + + TriedFallback = TRUE; + + CloseHandle(FileHandle); + FileHandle = CreateFileW(DumpPath, + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (FileHandle != INVALID_HANDLE_VALUE) { + DumpType = MiniDumpIgnoreInaccessibleMemory; + FallbackResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + ExceptionParam, + NULL, + NULL); + if (!FallbackResult) { + FallbackError = GetLastError(); + } + } + } + +LogOnly: + + if (LogHandle == INVALID_HANDLE_VALUE) { + LogHandle = CreateFileW(LogPath, + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + } + + if (LogHandle == INVALID_HANDLE_VALUE) { + goto End; + } + + SetFilePointer(LogHandle, 0, NULL, FILE_END); + + ExeBase = GetModuleHandleW(NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExeBase: 0x%p\r\n", + ExeBase); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "MiniDumpForceFallback: %s\r\n", + ForceFallback ? "true" : "false"); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "MiniDumpWriteDump: %s (Error=0x%08lX Win32=%lu)\r\n", + DumpResult ? "OK" : "FAILED", + DumpError, + (DumpError & 0xFFFF)); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + if (TriedFallback) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "MiniDumpWriteDumpFallback: %s (Error=0x%08lX Win32=%lu)\r\n", + FallbackResult ? "OK" : "FAILED", + FallbackError, + (FallbackError & 0xFFFF)); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + Frames = RtlCaptureStackBackTrace(0, + ARRAYSIZE(BackTrace), + BackTrace, + NULL); + + for (Index = 0; Index < Frames; Index++) { + Module = NULL; + Offset = 0; + ModulePathLength = 0; + ModulePathChars = 0; + ModulePathA[0] = '?'; + ModulePathA[1] = '\0'; + + if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)BackTrace[Index], + &Module)) { + Offset = (ULONG_PTR)BackTrace[Index] - (ULONG_PTR)Module; + + ModulePathLength = GetModuleFileNameW(Module, + ModulePathW, + ARRAYSIZE(ModulePathW)); + + if (ModulePathLength > 0) { + ModulePathW[ARRAYSIZE(ModulePathW) - 1] = L'\0'; + ModulePathChars = WideCharToMultiByte(CP_ACP, + 0, + ModulePathW, + ModulePathLength, + ModulePathA, + ARRAYSIZE(ModulePathA) - 1, + NULL, + NULL); + + if (ModulePathChars > 0) { + ModulePathA[ModulePathChars] = '\0'; + } + } + } + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Frame %02hu: 0x%p (base=0x%p offset=0x%Ix module=%s)\r\n", + Index, + BackTrace[Index], + Module, + Offset, + ModulePathA); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + +End: + + if (FileHandle != INVALID_HANDLE_VALUE) { + CloseHandle(FileHandle); + } + + if (LogHandle != INVALID_HANDLE_VALUE) { + CloseHandle(LogHandle); + } + + if (DbgHelpModule) { + FreeLibrary(DbgHelpModule); + } + + return EXCEPTION_EXECUTE_HANDLER; +} + +static +VOID +InstallPerfectHashServerCrashHandler( + VOID + ) +{ + DWORD Length; + + Length = GetEnvironmentVariableW(L"PH_LOG_SERVER_CRASH", NULL, 0); + if (Length == 0) { + return; + } + + InitializeServerCrashDir(); + PreloadServerDbgHelp(); + + SetUnhandledExceptionFilter(PerfectHashServerUnhandledExceptionFilter); +} + +static +VOID +TriggerServerCrashTest( + _In_ ULONG ExceptionCode + ) +{ + UNREFERENCED_PARAMETER(ExceptionCode); + PerfectHashServerUnhandledExceptionFilter(NULL); + ExitProcess((ULONG)E_UNEXPECTED); +} + +static +VOID +MaybeTriggerServerCrashTest( + VOID + ) +{ + DWORD Length; + WCHAR Buffer[32]; + Length = GetEnvironmentVariableW(L"PH_SERVER_CRASH_TEST", + Buffer, + ARRAYSIZE(Buffer)); + + if (Length == 0 || Length >= ARRAYSIZE(Buffer)) { + return; + } + + Buffer[Length] = L'\0'; + + InitializeServerCrashDir(); + + if (_wcsicmp(Buffer, L"AV") == 0 || + _wcsicmp(Buffer, L"ACCESS_VIOLATION") == 0) { + TriggerServerCrashTest(STATUS_ACCESS_VIOLATION); + } else { + TriggerServerCrashTest(0xE0000001); + } +} + +#endif +static +VOID +PrintUsage( + VOID + ) +{ + wprintf(L"Usage: PerfectHashServer [--IocpConcurrency=] " + L"[--MaxThreads=] [--Numa=All|] [--Endpoint=] " + L"[--AllowRemote|--LocalOnly] [--Verbose] [--NoFileIo] " + L"[--IocpBufferGuardPages]\n"); + wprintf(L" --IocpConcurrency= IOCP concurrency per NUMA node\n"); + wprintf(L" --MaxConcurrency= Alias for --IocpConcurrency\n"); + wprintf(L" --MaxThreads= Worker threads per NUMA node\n"); + wprintf(L" Default: IocpConcurrency * 2 when set\n"); + wprintf(L" --Numa=All|0,1|0-3 NUMA node selection mask\n"); + wprintf(L" --Endpoint= Named pipe endpoint\n"); + wprintf(L" --AllowRemote Allow remote named pipe clients\n"); + wprintf(L" --LocalOnly Reject remote named pipe clients\n"); + wprintf(L" --Verbose Enable per-request console output\n"); + wprintf(L" --NoFileIo Disable file I/O for requests\n"); + wprintf(L" --IocpBufferGuardPages Enable guard pages for IOCP buffers\n"); +} + +static +HRESULT +ParseUnsignedInteger( + _In_ PCWSTR Value, + _Out_ PULONG Result + ) +{ + PWSTR End = NULL; + ULONG Local; + + Local = wcstoul(Value, &End, 10); + if (!End || End == Value || *End != L'\0') { + return E_INVALIDARG; + } + + *Result = Local; + return S_OK; +} + +static +HRESULT +ParseNumaNodeMaskValue( + _In_ PCWSTR Value, + _Out_ PPERFECT_HASH_NUMA_NODE_MASK Mask + ) +{ + PCWSTR Ptr; + PERFECT_HASH_NUMA_NODE_MASK LocalMask; + + if (_wcsicmp(Value, L"All") == 0) { + *Mask = PERFECT_HASH_NUMA_NODE_MASK_ALL; + return S_OK; + } + + LocalMask = 0; + Ptr = Value; + + while (*Ptr) { + ULONG Start; + ULONG End; + PWSTR Next = NULL; + + Start = wcstoul(Ptr, &Next, 10); + if (!Next || Next == Ptr) { + return E_INVALIDARG; + } + + End = Start; + + if (*Next == L'-') { + Ptr = Next + 1; + End = wcstoul(Ptr, &Next, 10); + if (!Next || Next == Ptr) { + return E_INVALIDARG; + } + } + + if (End < Start) { + return E_INVALIDARG; + } + + if (End >= 64) { + return E_INVALIDARG; + } + + for (; Start <= End; Start++) { + LocalMask |= (1ULL << Start); + } + + if (*Next == L',') { + Ptr = Next + 1; + continue; + } else if (*Next == L'\0') { + break; + } else { + return E_INVALIDARG; + } + } + + if (LocalMask == 0) { + return E_INVALIDARG; + } + + *Mask = LocalMask; + return S_OK; +} + +static +HRESULT +ParseServerArgs( + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _Inout_ PPERFECT_HASH_SERVER_CLI_OPTIONS Options + ) +{ + ULONG Index; + + Options->IocpConcurrency = 0; + Options->MaxThreads = 0; + Options->NumaNodeMask = PERFECT_HASH_NUMA_NODE_MASK_ALL; + Options->Endpoint.Buffer = NULL; + Options->Endpoint.Length = 0; + Options->Endpoint.MaximumLength = 0; + Options->EndpointPresent = FALSE; + Options->IocpConcurrencyPresent = FALSE; + Options->MaxThreadsPresent = FALSE; + Options->NumaNodeMaskPresent = FALSE; + Options->LocalOnly = TRUE; + Options->LocalOnlyPresent = FALSE; + Options->Verbose = FALSE; + Options->VerbosePresent = FALSE; + Options->NoFileIo = FALSE; + Options->NoFileIoPresent = FALSE; + Options->IocpBufferGuardPages = FALSE; + Options->IocpBufferGuardPagesPresent = FALSE; + + for (Index = 1; Index < NumberOfArguments; Index++) { + PCWSTR Arg = ArgvW[Index]; + + if (!Arg) { + continue; + } + + if (Arg[0] != L'-' || Arg[1] != L'-') { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Arg += 2; + + if (_wcsicmp(Arg, L"Help") == 0 || + _wcsicmp(Arg, L"?") == 0) { + PrintUsage(); + return S_FALSE; + } + + if (_wcsnicmp(Arg, L"IocpConcurrency=", 16) == 0) { + HRESULT Result; + Arg += 16; + Result = ParseUnsignedInteger(Arg, &Options->IocpConcurrency); + if (FAILED(Result)) { + return Result; + } + Options->IocpConcurrencyPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"MaxConcurrency=", 15) == 0) { + HRESULT Result; + Arg += 15; + Result = ParseUnsignedInteger(Arg, &Options->IocpConcurrency); + if (FAILED(Result)) { + return Result; + } + Options->IocpConcurrencyPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"MaxThreads=", 11) == 0) { + HRESULT Result; + Arg += 11; + Result = ParseUnsignedInteger(Arg, &Options->MaxThreads); + if (FAILED(Result)) { + return Result; + } + Options->MaxThreadsPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"Numa=", 5) == 0) { + HRESULT Result; + Arg += 5; + Result = ParseNumaNodeMaskValue(Arg, &Options->NumaNodeMask); + if (FAILED(Result)) { + return Result; + } + Options->NumaNodeMaskPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"Endpoint=", 9) == 0) { + PCWSTR Value; + ULONG Length; + + Value = Arg + 9; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->Endpoint.Buffer = (PWSTR)Value; + Options->Endpoint.Length = (USHORT)Length; + Options->Endpoint.MaximumLength = (USHORT)Length + sizeof(WCHAR); + Options->EndpointPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"AllowRemote") == 0) { + Options->LocalOnly = FALSE; + Options->LocalOnlyPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"LocalOnly") == 0) { + Options->LocalOnly = TRUE; + Options->LocalOnlyPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"Verbose") == 0) { + Options->Verbose = TRUE; + Options->VerbosePresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"NoFileIo") == 0) { + Options->NoFileIo = TRUE; + Options->NoFileIoPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"IocpBufferGuardPages") == 0) { + Options->IocpBufferGuardPages = TRUE; + Options->IocpBufferGuardPagesPresent = TRUE; + continue; + } + + return PH_E_INVALID_COMMANDLINE_ARG; + } + + return S_OK; +} + +#ifdef PH_WINDOWS +static +BOOL +WINAPI +PerfectHashServerConsoleCtrlHandler( + _In_ DWORD ControlType + ) +{ + switch (ControlType) { + case CTRL_C_EVENT: + case CTRL_CLOSE_EVENT: + case CTRL_SHUTDOWN_EVENT: + if (GlobalServer) { + GlobalServer->Vtbl->Stop(GlobalServer); + } + return TRUE; + default: + return FALSE; + } +} +#endif + +// +// Main entry point. +// + +#ifdef PH_WINDOWS +DECLSPEC_NORETURN +VOID +WINAPI +mainCRTStartup( + VOID + ) +{ + HMODULE Module = NULL; + HRESULT Result = S_OK; + LPWSTR *ArgvW; + LPWSTR CommandLineW; + PICLASSFACTORY ClassFactory; + PPERFECT_HASH_SERVER Server = NULL; + PERFECT_HASH_SERVER_CLI_OPTIONS Options = { 0 }; + PPERFECT_HASH_PRINT_ERROR PerfectHashPrintError = NULL; + PPERFECT_HASH_PRINT_MESSAGE PerfectHashPrintMessage; + PICLASSFACTORY_CREATE_INSTANCE CreateInstance; + INT NumberOfArguments = 0; + + CommandLineW = GetCommandLineW(); + ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); + + InstallPerfectHashServerCrashHandler(); + MaybeTriggerServerCrashTest(); + + Result = PerfectHashBootstrap(&ClassFactory, + &PerfectHashPrintError, + &PerfectHashPrintMessage, + &Module); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashBootstrap, Result); + } + goto Error; + } + + CreateInstance = ClassFactory->Vtbl->CreateInstance; + + Result = CreateInstance(ClassFactory, + NULL, + &IID_PERFECT_HASH_SERVER, + &Server); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashServerCreateInstance, Result); + } + goto Error; + } + + Result = ParseServerArgs(NumberOfArguments, ArgvW, &Options); + if (Result == S_FALSE) { + Result = S_OK; + goto End; + } else if (FAILED(Result)) { + PrintUsage(); + goto Error; + } + + if (Options.IocpConcurrencyPresent) { + Result = Server->Vtbl->SetMaximumConcurrency( + Server, + Options.IocpConcurrency + ); + if (FAILED(Result)) { + goto Error; + } + + if (!Options.MaxThreadsPresent) { + ULONGLONG DefaultThreads; + + DefaultThreads = (ULONGLONG)Options.IocpConcurrency * 2; + if (DefaultThreads == 0 || DefaultThreads > ULONG_MAX) { + Result = E_INVALIDARG; + goto Error; + } + + Options.MaxThreads = (ULONG)DefaultThreads; + Options.MaxThreadsPresent = TRUE; + } + } + + if (Options.MaxThreadsPresent) { + Result = Server->Vtbl->SetMaximumThreads( + Server, + Options.MaxThreads + ); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.NumaNodeMaskPresent) { + Result = Server->Vtbl->SetNumaNodeMask(Server, + Options.NumaNodeMask); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.EndpointPresent) { + Result = Server->Vtbl->SetEndpoint(Server, &Options.Endpoint); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.LocalOnlyPresent) { + Result = Server->Vtbl->SetLocalOnly(Server, Options.LocalOnly); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.VerbosePresent) { + Result = Server->Vtbl->SetVerbose(Server, Options.Verbose); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.NoFileIoPresent) { + Result = Server->Vtbl->SetNoFileIo(Server, Options.NoFileIo); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.IocpBufferGuardPagesPresent) { + Result = Server->Vtbl->SetIocpBufferGuardPages( + Server, + Options.IocpBufferGuardPages + ); + if (FAILED(Result)) { + goto Error; + } + } + + GlobalServer = Server; + SetConsoleCtrlHandler(PerfectHashServerConsoleCtrlHandler, TRUE); + + Result = Server->Vtbl->Start(Server); + if (FAILED(Result)) { + goto Error; + } + + Result = Server->Vtbl->Wait(Server); + if (FAILED(Result)) { + goto Error; + } + + goto End; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashServer, Result); + } + +End: + + if (Server) { + Server->Vtbl->Release(Server); + } + + if (ClassFactory) { + ClassFactory->Vtbl->Release(ClassFactory); + } + + if (Module) { + FreeLibrary(Module); + } + + ExitProcess((ULONG)Result); +} + +#else // PH_WINDOWS + +int +main( + int NumberOfArguments, + char **ArgvA + ) +{ + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + + return 0; +} + +#endif // PH_WINDOWS + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.rc b/src/PerfectHashServerExe/PerfectHashServerExe.rc new file mode 100644 index 00000000..02be9198 --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.rc @@ -0,0 +1,49 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + PerfectHashServerExe.rc + +Abstract: + + This is the main resource compiler for the perfect hash server executable. + It is responsible for providing version information. + +--*/ + +#include "../PerfectHashVersion.rc" + +#define VER_FILEDESCRIPTION_STR "Perfect Hash Server Application" +#define VER_ORIGINALFILENAME_STR "PerfectHashServer.exe" +#define VER_INTERNALNAME_STR VER_ORIGINALFILENAME_STR + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_PRODUCTVERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +FILEFLAGS VER_FILEFLAGS +FILEOS VOS_NT +FILETYPE VFT_APP +FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + BEGIN + VALUE "CompanyName", VER_COMPANYNAME_STR + VALUE "FileDescription", VER_FILEDESCRIPTION_STR + VALUE "LegalCopyright", VER_LEGALCOPYRIGHT_STR + VALUE "OriginalFilename", VER_ORIGINALFILENAME_STR + VALUE "ProductName", VER_PRODUCTNAME_STR + VALUE "ProductVersion", VER_PRODUCTVERSION_STR + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END + diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj new file mode 100644 index 00000000..469f31ec --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj @@ -0,0 +1,71 @@ + + + + + Debug + x64 + + + Release + x64 + + + PGInstrument + x64 + + + PGUpdate + x64 + + + PGOptimize + x64 + + + + {B269AB7F-EF35-490A-B66D-031E431C4481} + PerfectHashServerExe + 10.0 + PerfectHashServerExe + + + PerfectHashServer + .exe + + + + Application + + + + + + + + + + + + + + false + + + + + + + + + Create + + + + + + + + + + + diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters new file mode 100644 index 00000000..0373fa97 --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters @@ -0,0 +1,41 @@ + + + + + {F15B494D-F723-4443-9E03-3A04103D2DB9} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {AC13CE39-13FD-4BA5-9F10-CAAACA7CC3EE} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {6F13BC5B-A638-48FA-9F17-58A620D37510} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + + + Resource Files + + + Resource Files + + + diff --git a/src/PerfectHashServerExe/stdafx.c b/src/PerfectHashServerExe/stdafx.c new file mode 100644 index 00000000..fd4f341c --- /dev/null +++ b/src/PerfectHashServerExe/stdafx.c @@ -0,0 +1 @@ +#include "stdafx.h" diff --git a/src/PerfectHashServerExe/stdafx.h b/src/PerfectHashServerExe/stdafx.h new file mode 100644 index 00000000..3790ffec --- /dev/null +++ b/src/PerfectHashServerExe/stdafx.h @@ -0,0 +1,21 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + stdafx.h + +Abstract: + + This is the precompiled header file for the perfect hash server + executable. + +--*/ + +#pragma once + +#include +#include + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashServerExe/targetver.h b/src/PerfectHashServerExe/targetver.h new file mode 100644 index 00000000..cfae052b --- /dev/null +++ b/src/PerfectHashServerExe/targetver.h @@ -0,0 +1 @@ +#include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 37133fe3..2803610c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,9 @@ endif() add_executable(perfecthash_unit_tests PerfectHashArgvTests.cpp + PerfectHashIocpBufferPoolTests.cpp + ${PERFECTHASH_SRC_DIR}/PerfectHash/PerfectHashIocpBufferPool.c + ${PERFECTHASH_SRC_DIR}/PerfectHash/RtlBuffers.c ) target_link_libraries(perfecthash_unit_tests @@ -26,6 +29,18 @@ target_link_libraries(perfecthash_unit_tests perfecthash_apply_common_definitions(perfecthash_unit_tests) +set_source_files_properties( + ${PERFECTHASH_SRC_DIR}/PerfectHash/PerfectHashIocpBufferPool.c + PROPERTIES + COMPILE_DEFINITIONS _PERFECT_HASH_INTERNAL_BUILD +) + +set_source_files_properties( + ${PERFECTHASH_SRC_DIR}/PerfectHash/RtlBuffers.c + PROPERTIES + COMPILE_DEFINITIONS _PERFECT_HASH_INTERNAL_BUILD +) + set_target_properties(perfecthash_unit_tests PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED YES diff --git a/tests/PerfectHashIocpBufferPoolTests.cpp b/tests/PerfectHashIocpBufferPoolTests.cpp new file mode 100644 index 00000000..d17fc072 --- /dev/null +++ b/tests/PerfectHashIocpBufferPoolTests.cpp @@ -0,0 +1,187 @@ +#include + +#include + +#define _PERFECT_HASH_INTERNAL_BUILD +#include "PerfectHash/stdafx.h" +#include "PerfectHash/PerfectHashIocpBufferPool.h" + +namespace { + +void NTAPI +TestRtlFillMemory( + PVOID Destination, + ULONG_PTR Length, + BYTE Fill + ) +{ + std::memset(Destination, Fill, static_cast(Length)); +} + +void NTAPI +TestRtlZeroMemory( + PVOID Destination, + ULONG_PTR Length + ) +{ + std::memset(Destination, 0, static_cast(Length)); +} + +ULONGLONG +TestRoundUpPowerOfTwo64( + ULONGLONG Value + ) +{ + if (Value == 0) { + return 0; + } + + Value--; + Value |= Value >> 1; + Value |= Value >> 2; + Value |= Value >> 4; + Value |= Value >> 8; + Value |= Value >> 16; + Value |= Value >> 32; + Value++; + + return Value; +} + +ULONGLONG +TestTrailingZeros64( + ULONGLONG Value + ) +{ + ULONGLONG Count = 0; + + if (Value == 0) { + return 64; + } + + while ((Value & 1) == 0) { + Count++; + Value >>= 1; + } + + return Count; +} + +class PerfectHashIocpBufferPoolTests : public ::testing::Test { +protected: + void SetUp() override { + std::memset(&rtl_, 0, sizeof(rtl_)); + std::memset(&vtbl_, 0, sizeof(vtbl_)); + vtbl_.CreateBuffer = RtlCreateBuffer; + vtbl_.DestroyBuffer = RtlDestroyBuffer; + rtl_.Vtbl = &vtbl_; + rtl_.RtlFillMemory = TestRtlFillMemory; + rtl_.RtlZeroMemory = TestRtlZeroMemory; + rtl_.RoundUpPowerOfTwo64 = TestRoundUpPowerOfTwo64; + rtl_.TrailingZeros64 = TestTrailingZeros64; + } + + void TearDown() override { + rtl_.Vtbl = nullptr; + } + + RTL rtl_; + RTL_VTBL vtbl_; +}; + +TEST_F(PerfectHashIocpBufferPoolTests, InitializeAcquireRelease) { + PERFECT_HASH_IOCP_BUFFER_POOL pool; + std::memset(&pool, 0, sizeof(pool)); + EXPECT_EQ(reinterpret_cast(&pool) & 0xF, 0u); + + HRESULT result = PerfectHashIocpBufferPoolInitialize( + &rtl_, + &pool, + 4096, + 0, + 0, + nullptr, + nullptr); + + ASSERT_GE(result, 0); + EXPECT_GT(pool.PayloadSize, 0u); + EXPECT_GT(pool.AllocationSize, 0u); + + PPERFECT_HASH_IOCP_BUFFER buffer = nullptr; + result = PerfectHashIocpBufferPoolAcquire(&rtl_, + nullptr, + &pool, + &buffer); + ASSERT_GE(result, 0); + ASSERT_NE(buffer, nullptr); + EXPECT_EQ(buffer->PayloadSize, pool.PayloadSize); + EXPECT_EQ(buffer->PayloadOffset, PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE); + + PVOID payload = PerfectHashIocpBufferPayload(buffer); + EXPECT_EQ(reinterpret_cast(payload) - + reinterpret_cast(buffer), + buffer->PayloadOffset); + + PerfectHashIocpBufferPoolRelease(&pool, buffer); + + PPERFECT_HASH_IOCP_BUFFER second = nullptr; + result = PerfectHashIocpBufferPoolAcquire(&rtl_, + nullptr, + &pool, + &second); + ASSERT_GE(result, 0); + ASSERT_NE(second, nullptr); + EXPECT_EQ(second, buffer); + PerfectHashIocpBufferPoolRelease(&pool, second); + + PerfectHashIocpBufferPoolRundown(&rtl_, nullptr, &pool); +} + +TEST_F(PerfectHashIocpBufferPoolTests, GuardPagesFlagPropagates) { + PERFECT_HASH_IOCP_BUFFER_POOL pool; + std::memset(&pool, 0, sizeof(pool)); + + HRESULT result = PerfectHashIocpBufferPoolInitialize( + &rtl_, + &pool, + 4096, + 0, + PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES, + nullptr, + nullptr); + + ASSERT_GE(result, 0); + + PPERFECT_HASH_IOCP_BUFFER buffer = nullptr; + result = PerfectHashIocpBufferPoolAcquire(&rtl_, + nullptr, + &pool, + &buffer); + ASSERT_GE(result, 0); + ASSERT_NE(buffer, nullptr); + EXPECT_NE(buffer->Flags & PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES, 0u); + + PerfectHashIocpBufferPoolRelease(&pool, buffer); + PerfectHashIocpBufferPoolRundown(&rtl_, nullptr, &pool); +} + +TEST_F(PerfectHashIocpBufferPoolTests, SizeClassHelpers) { + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 0), 0); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 1), 0); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 4096), 0); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 4097), 1); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 8192), 1); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex( + &rtl_, + PERFECT_HASH_IOCP_BUFFER_MAX_SIZE), + PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT - 1); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex( + &rtl_, + PERFECT_HASH_IOCP_BUFFER_MAX_SIZE + 1), + -1); + + EXPECT_EQ(PerfectHashIocpBufferGetPayloadSizeFromClassIndex(0), 4096u); + EXPECT_EQ(PerfectHashIocpBufferGetPayloadSizeFromClassIndex(1), 8192u); +} + +} // namespace