From 174c46965649594850bd25dc63cb1029c4be46dd Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Tue, 20 Jan 2026 21:57:27 -0800 Subject: [PATCH 1/9] Checkpoint commit of IOCP progress. --- AGENTS.md | 3 + CMakeLists.txt | 2 + IOCP-LOGS.md | 60 + IOCP-PROMPT.md | 135 + IOCP-TOOD.md | 19 + include/PerfectHash.h | 541 +++- include/PerfectHashErrors.h | 20 +- scripts/iocp-smoke.ps1 | 101 + scripts/stress-sys32-iocp.ps1 | 107 + scripts/stress-sys32.ps1 | 51 + scripts/stress-sys32.sh | 37 + src/CMakeLists.txt | 4 + src/PerfectHash.sln | 30 + src/PerfectHash/CMakeLists.txt | 10 + src/PerfectHash/Chm01.c | 21 +- src/PerfectHash/Chm01Async.c | 1833 +++++++++++ src/PerfectHash/Chm01Async.h | 137 + src/PerfectHash/Chm01Compat.c | 5 - src/PerfectHash/Chm01FileWork.c | 30 +- src/PerfectHash/Chm01FileWork.h | 3 +- .../Chm01FileWorkCppHeaderOnlyFile.c | 70 + src/PerfectHash/Chm02.c | 10 +- src/PerfectHash/Chm02Shared.c | 5 - src/PerfectHash/PerfectHash.vcxproj | 10 + src/PerfectHash/PerfectHash.vcxproj.filters | 30 + src/PerfectHash/PerfectHashAsync.c | 296 ++ src/PerfectHash/PerfectHashAsync.h | 122 + src/PerfectHash/PerfectHashClient.c | 514 +++ src/PerfectHash/PerfectHashClient.h | 89 + src/PerfectHash/PerfectHashConstants.c | 91 +- src/PerfectHash/PerfectHashContext.c | 309 +- src/PerfectHash/PerfectHashContext.h | 44 +- src/PerfectHash/PerfectHashContextIocp.c | 1250 +++++++ src/PerfectHash/PerfectHashContextIocp.h | 239 ++ src/PerfectHash/PerfectHashDirectory.c | 8 +- src/PerfectHash/PerfectHashErrors.dbg | 2 + src/PerfectHash/PerfectHashErrors.mc | 18 +- src/PerfectHash/PerfectHashErrors_English.bin | Bin 117448 -> 117620 bytes src/PerfectHash/PerfectHashServer.c | 2863 +++++++++++++++++ src/PerfectHash/PerfectHashServer.h | 135 + src/PerfectHash/PerfectHashTable.c | 24 + src/PerfectHash/PerfectHashTableCreate.c | 44 + src/PerfectHash/PerfectHashTls.h | 4 +- src/PerfectHash/stdafx.h | 6 + src/PerfectHashClientExe/CMakeLists.txt | 65 + .../PerfectHashClientExe.c | 450 +++ .../PerfectHashClientExe.rc | 49 + .../PerfectHashClientExe.vcxproj | 71 + .../PerfectHashClientExe.vcxproj.filters | 41 + src/PerfectHashClientExe/stdafx.c | 1 + src/PerfectHashClientExe/stdafx.h | 21 + src/PerfectHashClientExe/targetver.h | 1 + src/PerfectHashCreateExe/CMakeLists.txt | 4 + .../PerfectHashCreateExe.c | 249 ++ .../PerfectHashCreateExe.vcxproj | 3 + src/PerfectHashServerExe/CMakeLists.txt | 65 + .../PerfectHashServerExe.c | 795 +++++ .../PerfectHashServerExe.rc | 49 + .../PerfectHashServerExe.vcxproj | 71 + .../PerfectHashServerExe.vcxproj.filters | 41 + src/PerfectHashServerExe/stdafx.c | 1 + src/PerfectHashServerExe/stdafx.h | 21 + src/PerfectHashServerExe/targetver.h | 1 + 63 files changed, 11269 insertions(+), 62 deletions(-) create mode 100644 IOCP-LOGS.md create mode 100644 IOCP-PROMPT.md create mode 100644 IOCP-TOOD.md create mode 100644 scripts/iocp-smoke.ps1 create mode 100644 scripts/stress-sys32-iocp.ps1 create mode 100644 scripts/stress-sys32.ps1 create mode 100644 scripts/stress-sys32.sh create mode 100644 src/PerfectHash/Chm01Async.c create mode 100644 src/PerfectHash/Chm01Async.h create mode 100644 src/PerfectHash/PerfectHashAsync.c create mode 100644 src/PerfectHash/PerfectHashAsync.h create mode 100644 src/PerfectHash/PerfectHashClient.c create mode 100644 src/PerfectHash/PerfectHashClient.h create mode 100644 src/PerfectHash/PerfectHashContextIocp.c create mode 100644 src/PerfectHash/PerfectHashContextIocp.h create mode 100644 src/PerfectHash/PerfectHashServer.c create mode 100644 src/PerfectHash/PerfectHashServer.h create mode 100644 src/PerfectHashClientExe/CMakeLists.txt create mode 100644 src/PerfectHashClientExe/PerfectHashClientExe.c create mode 100644 src/PerfectHashClientExe/PerfectHashClientExe.rc create mode 100644 src/PerfectHashClientExe/PerfectHashClientExe.vcxproj create mode 100644 src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters create mode 100644 src/PerfectHashClientExe/stdafx.c create mode 100644 src/PerfectHashClientExe/stdafx.h create mode 100644 src/PerfectHashClientExe/targetver.h create mode 100644 src/PerfectHashServerExe/CMakeLists.txt create mode 100644 src/PerfectHashServerExe/PerfectHashServerExe.c create mode 100644 src/PerfectHashServerExe/PerfectHashServerExe.rc create mode 100644 src/PerfectHashServerExe/PerfectHashServerExe.vcxproj create mode 100644 src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters create mode 100644 src/PerfectHashServerExe/stdafx.c create mode 100644 src/PerfectHashServerExe/stdafx.h create mode 100644 src/PerfectHashServerExe/targetver.h diff --git a/AGENTS.md b/AGENTS.md index 7f098774..dd88a486 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,3 +15,6 @@ ## File enum capacity - File enums are tracked via a 64-bit bitmap. When we run out of bits, add a new enum group with a `_2`, `_3`, etc. suffix and keep the existing enum ordering rules intact. + +## Auto-generated files +- `include/PerfectHashEvents.h` is auto-generated; discard local changes before rebasing or merging. diff --git a/CMakeLists.txt b/CMakeLists.txt index ba18621c..cf67c01b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,8 @@ if(PERFECTHASH_ENABLE_INSTALL) PerfectHash PerfectHashCreateExe PerfectHashBulkCreateExe + PerfectHashServerExe + PerfectHashClientExe RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md new file mode 100644 index 00000000..18549f4c --- /dev/null +++ b/IOCP-LOGS.md @@ -0,0 +1,60 @@ +# IOCP Logs + +## 2026-01-19 +- Added IOCP context/server/client interfaces and COM scaffolding. +- Added NUMA-aware IOCP runtime skeleton (per-node IOCPs, worker threads with affinity). +- Wired new components into PerfectHash constants/build definitions. +- Added server/client executables with basic CLI parsing and IOCP startup wiring. +- Fixed /Wall padding warnings and confirmed Debug builds for server/client targets. +- Implemented named-pipe IOCP transport in the server with per-node pipe instances and a request/response state machine. +- Implemented client-side named-pipe connect and request submission with wire protocol headers. +- Fixed /Wall issues in new IOCP code (Rtl-backed ZeroMemory usage, struct padding, enum exhaustiveness) and rebuilt server/client executables. +- Wired server request dispatch to parse command-line payloads and call IOCP context bulk/table create entrypoints. +- Implemented IOCP ArgvW entrypoints by delegating to legacy PerfectHashContext workflows. +- Added server endpoint/local-only API + CLI configuration, plus error-response payload handling and client-side retrieval/printing. +- Added a PowerShell smoke test script for server/client shutdown round-trips. +- Ran `scripts/iocp-smoke.ps1` after fixes to confirm clean startup/shutdown. +- Diagnosed a Debug breakpoint during table create to a TLS global-component bitfield mismatch; realigned TLS flag bit positions to interface IDs and removed fragile asserts. +- Added a create-exe unhandled-exception stack trace hook (DbgHelp) to aid future diagnostics. +- Hardened `scripts/iocp-smoke.ps1` to handle missing exit codes after timed waits; smoke test now completes successfully. +- Added IOCP work item header/signature and per-item completion dispatch in the IOCP worker loop. +- Updated server pipe handling to use per-pipe IOCP work items instead of global completion callbacks. +- Added stress scripts for sys32 bulk create (direct and IOCP variants). +- Rebuilt PerfectHashServerExe and reran `scripts/iocp-smoke.ps1` successfully. +- Added bulk-create directory server request with per-request completion token (event + mapping). +- Implemented server-side directory walk and per-file IOCP work item dispatch for bulk requests. +- Added client-side `--BulkCreateDirectory` support with token wait and bulk result handling. +- Fixed bulk enqueue completion accounting and failure handling to avoid double-decrement. +- Updated IOCP sys32 stress script to use bulk-create directory request. +- Regenerated PerfectHash error headers/resources for new status codes. +- Rebuilt PerfectHashServerExe/PerfectHashClientExe with `/p:CL_MPCount=1` and reran `scripts/iocp-smoke.ps1`. +- Built Release targets for PerfectHashBulkCreate/Server/Client with `/p:CL_MPCount=1`. +- BulkCreate Release sys32 run at concurrency 32 crashed (`0xC0000005`); repeated `FlushConsoleInputBuffer` errors observed. +- BulkCreateDirectory argv handling fixed to always inject a program name for parsing. +- IOCP Release sys32 run at concurrency 32 fails with `SetThreadpoolThreadMinimum` access denied during per-file context init. +- Created `D:\\src\\perfecthash-main` worktree, built Release `PerfectHashBulkCreate.exe` and tested baseline bulk-create. +- `hard2` run succeeded but printed repeated `FlushConsoleInputBuffer` errors. +- `sys32` run crashed with access violation (`0xC0000005`) and the same flush errors, indicating the crash exists on `main`. + +## 2026-01-20 +- Rebased `iocp-dev` onto `origin/main` and resolved `PerfectHashBulkCreateExe.c` conflict by keeping upstream minidump handler. +- Ignored `ERROR_ACCESS_DENIED` from `SetMaximumConcurrency` in IOCP per-file work items to allow bulk runs to continue. +- Diagnosed IOCP bulk-create server crash (`0xC0000005`) to use-after-free of `CommandLineToArgvW` buffers referenced by table-create parameters. +- Retained bulk-create directory `ArgvW` buffers in request state (free on completion) and moved argv adjustment into server dispatch to keep string pointers alive. +- Fixed IOCP bulk work-item callback use-after-free (request freed before work item allocator cleanup). +- Mapped IOCP bulk-create crash to `PrepareCHeaderFileChm01` using `Context->CommandLineW`; crash was NULL due to direct `TableCreate` usage. +- Preserved request command line and seeded per-file contexts with `Context->CommandLineW` to keep header generation stable. +- Updated IOCP stress script to accept `PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED` as a success exit code. +- Ran `scripts/stress-sys32-iocp.ps1` against a single-file keys dir (Release, max concurrency 1); completed successfully. +- Added IOCP-backed file work dispatch hooks: context submit/wait helpers with outstanding-event tracking, per-file context IOCP port wiring, and file-work event signaling for non-threadpool callbacks. +- Added async work framework scaffolding and CHM01 async state machine skeleton (new modules, build integration). +- Added CHM01 async bridge that runs the legacy CHM01 routine on a worker thread and polls completion via IOCP, plus an env-gated async entrypoint. +- Replaced the CHM01 async bridge with IOCP-driven solver slices, graph work items, and a state-machine finalize path for save/close/verify. +- Fixed CHM01 async header macro usage inside a struct (removed extra semicolons) and prevented double-include macro expansion by adding `#pragma once` to `Chm01FileWork.h`. +- Added explicit padding to CHM01 async graph work items and undefed `EXPAND_AS_CHECK_ERRORS` before redefining to satisfy `/Wall` with warnings-as-errors. +- Rebuilt `PerfectHashServerExe` Debug successfully after the async/header fixes. +- Ran `scripts/stress-sys32-iocp.ps1` with `PERFECT_HASH_IOCP_ASYNC_CHM01=1` against `perfecthash-keys\\hard` (Debug, max concurrency 32); run timed out with client hung waiting for completion (server already exited), leaving partial output in `build-win\\iocp-stress-hard`. +- Added IOCP work-item flags and a CHM01 async pump loop so IOCP completions continue while `Chm01AsyncWaitJob()` waits; file work/async/pipe items are dispatched inline, bulk work items are requeued. +- Added a CHM01 async prepare-wait state for the try-larger-table-size path (still returns `PH_E_NOT_IMPLEMENTED` but now waits for prepare file work before closing). +- Stress runs against `perfecthash-keys\\hard` still time out; observed repeated `SetEndOfFile`/`PerfectHashFileTruncate` failures (error 1224: mapped section open) and `PrepareFileChm01` failures during IOCP bulk runs. +- Added a context flag to skip context-level file work for per-file IOCP contexts; CHM01 async prepare/save/close now bypass context file items and signal events directly to avoid `ERROR_USER_MAPPED_FILE` conflicts. diff --git a/IOCP-PROMPT.md b/IOCP-PROMPT.md new file mode 100644 index 00000000..414c451b --- /dev/null +++ b/IOCP-PROMPT.md @@ -0,0 +1,135 @@ +# IOCP Backend Status (PerfectHash) + +This prompt summarizes the current IOCP backend work for the perfecthash server/client, including what is implemented, recent fixes, and next steps. + +## Current State + +The IOCP backend exists as a parallel, additive path (does not replace existing executables). It provides: +- A NUMA-aware IOCP runtime with one completion port per node and manually created worker threads pinned to node affinity. +- A named-pipe server with an IOCP state machine for request/response. +- A client that submits requests and optionally waits on bulk completion tokens. +- A bulk-create directory request that fans out per-file work items to IOCP nodes. + +The core IOCP pipeline runs on Windows with IOCP worker threads (no Windows threadpools for IOCP workers). Per-file perfect hash work still uses legacy `PERFECT_HASH_CONTEXT` / table-create logic, including its internal threadpool usage. + +## Key Components / Files + +IOCP runtime + NUMA: +- `src/PerfectHash/PerfectHashContextIocp.c` +- `src/PerfectHash/PerfectHashContextIocp.h` + +Server / pipe IOCP: +- `src/PerfectHash/PerfectHashServer.c` +- `src/PerfectHash/PerfectHashServer.h` + +Client: +- `src/PerfectHash/PerfectHashClient.c` +- `src/PerfectHash/PerfectHashClient.h` + +Server exe: +- `src/PerfectHashServerExe/PerfectHashServerExe.c` + +Client exe: +- `src/PerfectHashClientExe/PerfectHashClientExe.c` + +Protocol constants and bulk result struct: +- `include/PerfectHash.h` + +Scripts: +- `scripts/iocp-smoke.ps1` (table create over IOCP) +- `scripts/stress-sys32-iocp.ps1` (bulk-create directory request) +- `scripts/stress-sys32.ps1` (baseline bulk create) + +Logs / TODOs: +- `IOCP-LOGS.md` +- `IOCP-TOOD.md` + +## Implemented Features (IOCP Backend) + +- NUMA-aware IOCP runtime: + - Enumerates NUMA nodes. + - Creates one IOCP per node. + - Spawns worker threads per node with group affinity. + - Configurable max concurrency and NUMA node mask. + - Implemented in `src/PerfectHash/PerfectHashContextIocp.c`. + +- Server named-pipe IOCP transport: + - Per-node pipe instances tied to the node IOCP. + - IOCP state machine for connect, read header/payload, write response. + - Local-only vs allow-remote configuration. + - Implemented in `src/PerfectHash/PerfectHashServer.c`. + +- Client request handling: + - Supports `--TableCreate`, `--BulkCreate`, `--BulkCreateDirectory`, `--Shutdown`. + - Waits on bulk-create completion tokens via event + mapping. + - Implemented in `src/PerfectHashClientExe/PerfectHashClientExe.c` and `src/PerfectHash/PerfectHashClient.c`. + +- Bulk-create directory request: + - Server walks a directory and queues per-file IOCP work items. + - Uses round-robin dispatch across NUMA nodes. + - Completion logic tracks per-node counts and outstanding totals to signal a single completion token. + - Implemented in `src/PerfectHash/PerfectHashServer.c`. + +## Recent Fixes / Diagnostics + +- Fixed a crash in IOCP bulk-create caused by `Context->CommandLineW` being NULL when `PrepareCHeaderFileChm01` emits the command line. + - Bulk request now keeps a copy of the command line and assigns it to each per-file context. + - Files: `src/PerfectHash/PerfectHashServer.c` + +- Fixed use-after-free in bulk work callback (request freed before work item cleanup). + - Files: `src/PerfectHash/PerfectHashServer.c` + +- Fixed `CommandLineToArgvW` lifetime for bulk directory requests: + - `ArgvW` is now retained for the request lifetime, and freed on completion. + - Files: `src/PerfectHash/PerfectHashServer.c` + +- Added richer crash logging for server process: + - Records exception address, module base, offset, and thread ID before minidump attempt. + - Files: `src/PerfectHashServerExe/PerfectHashServerExe.c` + - Use env var `PH_LOG_SERVER_CRASH=1` to enable. + - `PH_SERVER_MINIDUMP_FORCE_FALLBACK=1` forces fallback minidump behavior. + +- Adjusted IOCP stress script to accept success exit code: + - `PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED` (0x2004000F). + - Files: `scripts/stress-sys32-iocp.ps1` + +## What Works Now (Observed) + +- `scripts/iocp-smoke.ps1` succeeds (table create via IOCP server/client). +- `scripts/stress-sys32-iocp.ps1` succeeds on a small directory (single keys file) after crash fixes. + +## Known Gaps / Risks + +- IOCP bulk-create directory with full `..\perfecthash-keys\sys32` has not been rerun post-fix; expected to be next. +- Per-file work still uses legacy table-create which relies on its internal threadpool and console hooks; IOCP orchestration does not yet replace that internal compute pipeline. +- Backpressure/queueing policies are still “post everything immediately” (no high-water mark). +- Linux/`io_uring` path is not implemented; current IOCP design is Windows-only. + +## Next Steps (Plan) + +1. Run full IOCP stress: + - `scripts/stress-sys32-iocp.ps1` against `..\perfecthash-keys\sys32` with Release and desired concurrency. +2. Performance comparison: + - Compare baseline `scripts/stress-sys32.ps1` vs IOCP run (same hash/mask/concurrency). +3. Native IOCP pipeline: + - Replace legacy context delegation with explicit IOCP-driven phases (key load, graph solve, file work). + - Add proper queue/backpressure per request and per node. +4. Multi-node tracking polish: + - Expand per-node completion tracking if needed for more granular results. +5. Docs: + - Draft `IOCP-README.md` once behavior and protocol stabilize. + +## Suggested Commands + +Build: +- `cmake --build build-win --config Release --target PerfectHashServerExe PerfectHashClientExe` + +Smoke: +- `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\iocp-smoke.ps1 -BuildDir build-win -Config Release` + +IOCP stress (sys32): +- `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32-iocp.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` + +Baseline: +- `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` + diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md new file mode 100644 index 00000000..cb9afe5d --- /dev/null +++ b/IOCP-TOOD.md @@ -0,0 +1,19 @@ +# IOCP TODO + +- Validate bulk-create directory request end-to-end (sys32 stress) and tune per-file concurrency defaults. +- Run IOCP sys32 stress pass (Release, max concurrency) after crash fixes. +- Investigate Release bulk-create crash in `PerfectHashBulkCreate.exe` (0xC0000005) during sys32 stress. +- Track down `FlushConsoleInputBuffer` failures during bulk create (seen on `main` and iocp-dev). +- Validate access-denied fallback for per-file context threadpool minimum failures at higher concurrency. +- Recheck named-pipe endpoint handling if `PerfectHashServer-StressSys32` continues to fail. +- Decide whether BulkCreateDirectory should accept a single-directory short form or keep output dir required. +- Exercise IOCP file work dispatch path (bulk create) to confirm outstanding event signaling and non-threadpool file work callbacks. +- Validate CHM01 async path with smaller keysets (e.g. `hard`) and record timing vs legacy. +- Investigate IOCP bulk-create async hang on `hard` keyset (client wait never completes). +- Validate context file work skip fix for `SetEndOfFile`/`PerfectHashFileTruncate` 1224 failures; decide whether to generate context files once per bulk request. +- Handle CHM01 async resize/try-larger-table-size path (currently returns `PH_E_NOT_IMPLEMENTED`). +- Wire IOCP server bulk path to CHM01 async jobs and request completion callbacks. +- Complete native IOCP table/bulk create workflows (replace legacy-context delegation). +- Add IOCP work item queueing/state to drive per-request pipelines. +- Integrate per-request concurrency caps and queueing policies. +- Draft IOCP-README.md once protocol and dispatch behavior settle. diff --git a/include/PerfectHash.h b/include/PerfectHash.h index 1bfbcdc9..b2912f76 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -479,6 +479,33 @@ PerfectHashGetCurrentCpuArch( ) \ ) \ \ + ENTRY( \ + ContextIocp, \ + CONTEXT_IOCP, \ + GUID_EX( \ + 0x72bf14d6, 0xa312, 0x4bf5, \ + 0xb0, 0x51, 0x9e, 0xef, 0xa5, 0xfd, 0x8f, 0x59 \ + ) \ + ) \ + \ + ENTRY( \ + Server, \ + SERVER, \ + GUID_EX( \ + 0xbba77b60, 0xb38e, 0x4396, \ + 0xaa, 0x4b, 0xdc, 0xa5, 0x8e, 0xdf, 0xcf, 0x08 \ + ) \ + ) \ + \ + ENTRY( \ + Client, \ + CLIENT, \ + GUID_EX( \ + 0x997e42aa, 0x035f, 0x4d40, \ + 0x82, 0x9f, 0x6c, 0x4c, 0xe2, 0xd2, 0xeb, 0x44 \ + ) \ + ) \ + \ ENTRY( \ Table, \ TABLE, \ @@ -708,7 +735,7 @@ PerfectHashInterfaceGuidToId( for (Index = 1; Index < Count; Index++) { #ifdef __cplusplus const GUID *Entry = PerfectHashInterfaceGuids[Index]; - if (Entry && InlineIsEqualGUID(Guid, Entry)) { + if (Entry && InlineIsEqualGUID(Guid, *Entry)) { Id = (PERFECT_HASH_INTERFACE_ID)Index; break; } @@ -2656,7 +2683,20 @@ IsModulusMasking( // PERFECT_HASH_TABLE interface's creation routine. // +// +// Define NUMA node mask handling. A mask of all 1s indicates "all nodes"; +// callers should intersect it with the machine's available NUMA nodes. +// + +typedef ULONGLONG PERFECT_HASH_NUMA_NODE_MASK; +typedef PERFECT_HASH_NUMA_NODE_MASK *PPERFECT_HASH_NUMA_NODE_MASK; + +#define PERFECT_HASH_NUMA_NODE_MASK_ALL ((PERFECT_HASH_NUMA_NODE_MASK)~0ULL) + DECLARE_COMPONENT(Context, PERFECT_HASH_CONTEXT); +DECLARE_COMPONENT(ContextIocp, PERFECT_HASH_CONTEXT_IOCP); +DECLARE_COMPONENT(Server, PERFECT_HASH_SERVER); +DECLARE_COMPONENT(Client, PERFECT_HASH_CLIENT); typedef HRESULT @@ -4210,6 +4250,505 @@ typedef struct _PERFECT_HASH_CONTEXT { typedef PERFECT_HASH_CONTEXT *PPERFECT_HASH_CONTEXT; #endif +// +// Define the PERFECT_HASH_CONTEXT_IOCP interface. +// + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG MaximumConcurrency + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Out_ PULONG MaximumConcurrency + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK + *PPERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Out_ PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK + *PPERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PCUNICODE_STRING BaseOutputDirectory + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY + *PPERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Inout_ PPERFECT_HASH_DIRECTORY *BaseOutputDirectory + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY + *PPERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PCUNICODE_STRING KeysDirectory, + _In_ PCUNICODE_STRING BaseOutputDirectory, + _In_ PERFECT_HASH_ALGORITHM_ID AlgorithmId, + _In_ PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _In_ PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _In_opt_ PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + _In_opt_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _In_opt_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _In_opt_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_opt_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE + *PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW + ); +typedef PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW, + _In_ PUNICODE_STRING KeysDirectory, + _In_ PUNICODE_STRING BaseOutputDirectory, + _Inout_ PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + _Inout_ PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _Inout_ PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _Inout_ PULONG MaximumConcurrency, + _Inout_ PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + _Inout_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _Inout_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _Inout_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ PCUNICODE_STRING KeysPath, + _In_ PCUNICODE_STRING BaseOutputDirectory, + _In_ PERFECT_HASH_ALGORITHM_ID AlgorithmId, + _In_ PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _In_ PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _In_opt_ PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS + ContextTableCreateFlags, + _In_opt_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _In_opt_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _In_opt_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE + *PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW + ); +typedef PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE + PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW, + _In_ PUNICODE_STRING KeysPath, + _In_ PUNICODE_STRING BaseOutputDirectory, + _Inout_ PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + _Inout_ PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + _Inout_ PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + _Inout_ PULONG MaximumConcurrency, + _Inout_ PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS + ContextTableCreateFlags, + _Inout_ PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + _Inout_ PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + _Inout_ PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + _In_ PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ); +typedef PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + *PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPSTR *ArgvA + ); +typedef PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA + *PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA; + +typedef +_Success_(return >= 0) +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPSTR *ArgvA + ); +typedef PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA + *PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA; + +typedef struct _PERFECT_HASH_CONTEXT_IOCP_VTBL { + DECLARE_COMPONENT_VTBL_HEADER(PERFECT_HASH_CONTEXT_IOCP); + + PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY SetMaximumConcurrency; + PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY GetMaximumConcurrency; + PPERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK SetNumaNodeMask; + PPERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK GetNumaNodeMask; + PPERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY SetBaseOutputDirectory; + PPERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY GetBaseOutputDirectory; + + PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE BulkCreate; + PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW BulkCreateArgvW; + PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + ExtractBulkCreateArgsFromArgvW; + + PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE TableCreate; + + PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW TableCreateArgvW; + PPERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + ExtractTableCreateArgsFromArgvW; + + // + // N.B. These two routines will only be present on non-Windows platforms. + // + + PPERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA TableCreateArgvA; + PPERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA BulkCreateArgvA; + +} PERFECT_HASH_CONTEXT_IOCP_VTBL; +typedef PERFECT_HASH_CONTEXT_IOCP_VTBL *PPERFECT_HASH_CONTEXT_IOCP_VTBL; + +#ifndef _PERFECT_HASH_INTERNAL_BUILD +typedef struct _PERFECT_HASH_CONTEXT_IOCP { + PPERFECT_HASH_CONTEXT_IOCP_VTBL Vtbl; +} PERFECT_HASH_CONTEXT_IOCP; +typedef PERFECT_HASH_CONTEXT_IOCP *PPERFECT_HASH_CONTEXT_IOCP; +#endif + +// +// Define the PERFECT_HASH_SERVER interface. +// + +typedef enum _PERFECT_HASH_SERVER_REQUEST_TYPE { + PerfectHashNullServerRequestType = 0, + PerfectHashTableCreateServerRequestType, + PerfectHashBulkCreateServerRequestType, + PerfectHashBulkCreateDirectoryServerRequestType, + PerfectHashShutdownServerRequestType, + PerfectHashInvalidServerRequestType +} PERFECT_HASH_SERVER_REQUEST_TYPE; + +FORCEINLINE +BOOLEAN +IsValidPerfectHashServerRequestType( + _In_ PERFECT_HASH_SERVER_REQUEST_TYPE RequestType + ) +{ + return ( + RequestType > PerfectHashNullServerRequestType && + RequestType < PerfectHashInvalidServerRequestType + ); +} + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_SERVER_REQUEST { + ULONG SizeOfStruct; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + ULONG Padding1; + ULONG Padding2; + ULONGLONG RequestId; + UNICODE_STRING CommandLine; +} PERFECT_HASH_SERVER_REQUEST; +typedef PERFECT_HASH_SERVER_REQUEST *PPERFECT_HASH_SERVER_REQUEST; + +// +// Wire protocol structures for client/server IPC. +// + +#define PERFECT_HASH_SERVER_MESSAGE_VERSION 1 +#define PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE (1024 * 1024) + +#define PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE 0x00000001 +#define PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN 0x00000002 +#define PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT \ + L"EventHandle=%llu ResultHandle=%llu" + +#define PERFECT_HASH_SERVER_BULK_RESULT_VERSION 1 + +typedef struct _Struct_size_bytes_(SizeOfStruct) +_PERFECT_HASH_SERVER_BULK_RESULT { + ULONG SizeOfStruct; + ULONG Version; + HRESULT Result; + ULONG Flags; + ULONG TotalFiles; + ULONG SucceededFiles; + ULONG FailedFiles; + HRESULT FirstFailure; +} PERFECT_HASH_SERVER_BULK_RESULT; +typedef PERFECT_HASH_SERVER_BULK_RESULT *PPERFECT_HASH_SERVER_BULK_RESULT; + +typedef struct _Struct_size_bytes_(SizeOfStruct) +_PERFECT_HASH_SERVER_REQUEST_HEADER { + ULONG SizeOfStruct; + ULONG Version; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + ULONG Flags; + ULONGLONG RequestId; + ULONG PayloadLength; + ULONG Reserved; +} PERFECT_HASH_SERVER_REQUEST_HEADER; +typedef PERFECT_HASH_SERVER_REQUEST_HEADER + *PPERFECT_HASH_SERVER_REQUEST_HEADER; + +typedef struct _Struct_size_bytes_(SizeOfStruct) +_PERFECT_HASH_SERVER_RESPONSE_HEADER { + ULONG SizeOfStruct; + ULONG Version; + HRESULT Result; + ULONG Flags; + ULONGLONG RequestId; + ULONG PayloadLength; + ULONG Reserved; +} PERFECT_HASH_SERVER_RESPONSE_HEADER; +typedef PERFECT_HASH_SERVER_RESPONSE_HEADER + *PPERFECT_HASH_SERVER_RESPONSE_HEADER; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ ULONG MaximumConcurrency + ); +typedef PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PULONG MaximumConcurrency + ); +typedef PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY + *PPERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK + *PPERFECT_HASH_SERVER_SET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ); +typedef PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK + *PPERFECT_HASH_SERVER_GET_NUMA_NODE_MASK; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_ENDPOINT)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ PCUNICODE_STRING Endpoint + ); +typedef PERFECT_HASH_SERVER_SET_ENDPOINT + *PPERFECT_HASH_SERVER_SET_ENDPOINT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_ENDPOINT)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PUNICODE_STRING Endpoint + ); +typedef PERFECT_HASH_SERVER_GET_ENDPOINT + *PPERFECT_HASH_SERVER_GET_ENDPOINT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_LOCAL_ONLY)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN LocalOnly + ); +typedef PERFECT_HASH_SERVER_SET_LOCAL_ONLY + *PPERFECT_HASH_SERVER_SET_LOCAL_ONLY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_LOCAL_ONLY)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN LocalOnly + ); +typedef PERFECT_HASH_SERVER_GET_LOCAL_ONLY + *PPERFECT_HASH_SERVER_GET_LOCAL_ONLY; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_START)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_START *PPERFECT_HASH_SERVER_START; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_STOP)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_STOP *PPERFECT_HASH_SERVER_STOP; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_WAIT)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_WAIT *PPERFECT_HASH_SERVER_WAIT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SUBMIT_REQUEST)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ PPERFECT_HASH_SERVER_REQUEST Request + ); +typedef PERFECT_HASH_SERVER_SUBMIT_REQUEST + *PPERFECT_HASH_SERVER_SUBMIT_REQUEST; + +typedef struct _PERFECT_HASH_SERVER_VTBL { + DECLARE_COMPONENT_VTBL_HEADER(PERFECT_HASH_SERVER); + + PPERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY SetMaximumConcurrency; + PPERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY GetMaximumConcurrency; + PPERFECT_HASH_SERVER_SET_NUMA_NODE_MASK SetNumaNodeMask; + PPERFECT_HASH_SERVER_GET_NUMA_NODE_MASK GetNumaNodeMask; + PPERFECT_HASH_SERVER_SET_ENDPOINT SetEndpoint; + PPERFECT_HASH_SERVER_GET_ENDPOINT GetEndpoint; + PPERFECT_HASH_SERVER_SET_LOCAL_ONLY SetLocalOnly; + PPERFECT_HASH_SERVER_GET_LOCAL_ONLY GetLocalOnly; + PPERFECT_HASH_SERVER_START Start; + PPERFECT_HASH_SERVER_STOP Stop; + PPERFECT_HASH_SERVER_WAIT Wait; + PPERFECT_HASH_SERVER_SUBMIT_REQUEST SubmitRequest; +} PERFECT_HASH_SERVER_VTBL; +typedef PERFECT_HASH_SERVER_VTBL *PPERFECT_HASH_SERVER_VTBL; + +#ifndef _PERFECT_HASH_INTERNAL_BUILD +typedef struct _PERFECT_HASH_SERVER { + PPERFECT_HASH_SERVER_VTBL Vtbl; +} PERFECT_HASH_SERVER; +typedef PERFECT_HASH_SERVER *PPERFECT_HASH_SERVER; +#endif + +// +// Define the PERFECT_HASH_CLIENT interface. +// + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_CONNECT)( + _In_ PPERFECT_HASH_CLIENT Client, + _In_opt_ PCUNICODE_STRING Endpoint + ); +typedef PERFECT_HASH_CLIENT_CONNECT *PPERFECT_HASH_CLIENT_CONNECT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_DISCONNECT)( + _In_ PPERFECT_HASH_CLIENT Client + ); +typedef PERFECT_HASH_CLIENT_DISCONNECT *PPERFECT_HASH_CLIENT_DISCONNECT; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_SUBMIT_REQUEST)( + _In_ PPERFECT_HASH_CLIENT Client, + _In_ PPERFECT_HASH_SERVER_REQUEST Request + ); +typedef PERFECT_HASH_CLIENT_SUBMIT_REQUEST + *PPERFECT_HASH_CLIENT_SUBMIT_REQUEST; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CLIENT_GET_LAST_RESPONSE)( + _In_ PPERFECT_HASH_CLIENT Client, + _Out_ PUNICODE_STRING ResponsePayload, + _Out_opt_ PULONG ResponseFlags + ); +typedef PERFECT_HASH_CLIENT_GET_LAST_RESPONSE + *PPERFECT_HASH_CLIENT_GET_LAST_RESPONSE; + +typedef struct _PERFECT_HASH_CLIENT_VTBL { + DECLARE_COMPONENT_VTBL_HEADER(PERFECT_HASH_CLIENT); + + PPERFECT_HASH_CLIENT_CONNECT Connect; + PPERFECT_HASH_CLIENT_DISCONNECT Disconnect; + PPERFECT_HASH_CLIENT_SUBMIT_REQUEST SubmitRequest; + PPERFECT_HASH_CLIENT_GET_LAST_RESPONSE GetLastResponse; +} PERFECT_HASH_CLIENT_VTBL; +typedef PERFECT_HASH_CLIENT_VTBL *PPERFECT_HASH_CLIENT_VTBL; + +#ifndef _PERFECT_HASH_INTERNAL_BUILD +typedef struct _PERFECT_HASH_CLIENT { + PPERFECT_HASH_CLIENT_VTBL Vtbl; +} PERFECT_HASH_CLIENT; +typedef PERFECT_HASH_CLIENT *PPERFECT_HASH_CLIENT; +#endif + // // Define the PERFECT_HASH_TABLE interface. // diff --git a/include/PerfectHashErrors.h b/include/PerfectHashErrors.h index 46c5cf90..5a08ca25 100644 --- a/include/PerfectHashErrors.h +++ b/include/PerfectHashErrors.h @@ -189,6 +189,15 @@ Module Name: // #define PH_S_CU_KERNEL_RUNTIME_TARGET_REACHED ((HRESULT)0xE004000EL) +// +// MessageId: PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED +// +// MessageText: +// +// Bulk create request completed successfully. +// +#define PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED ((HRESULT)0x2004000FL) + //////////////////////////////////////////////////////////////////////////////// // PH_SEVERITY_INFORMATIONAL @@ -2538,7 +2547,16 @@ Module Name: #define PH_E_ERROR_DURING_PREPARE_VCPROPS_COMPILED_PERFECT_HASH_FILE ((HRESULT)0xE004028FL) // -// Spare IDs: 0x290, 0x291. +// MessageId: PH_E_SERVER_BULK_CREATE_FAILED +// +// MessageText: +// +// Bulk create request failed. +// +#define PH_E_SERVER_BULK_CREATE_FAILED ((HRESULT)0xE0040290L) + +// +// Spare ID: 0x291. // // // MessageId: PH_E_ERROR_DURING_SAVE_VCPROJECT_DLL_FILE diff --git a/scripts/iocp-smoke.ps1 b/scripts/iocp-smoke.ps1 new file mode 100644 index 00000000..2e9051b1 --- /dev/null +++ b/scripts/iocp-smoke.ps1 @@ -0,0 +1,101 @@ +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$Endpoint = "\\.\\pipe\\PerfectHashServer-Smoke", + [int]$TimeoutSeconds = 10 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) +$clientExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashClient.exe" -f $Config) +$workDir = Join-Path $BuildDir "iocp-smoke" +$keysFile = Join-Path $workDir "smoke_keys_16.keys" +$outputDir = Join-Path $workDir "output" + +if (-not (Test-Path $serverExe)) { + throw "Server exe not found: $serverExe" +} + +if (-not (Test-Path $clientExe)) { + throw "Client exe not found: $clientExe" +} + +New-Item -ItemType Directory -Path $workDir -Force | Out-Null +New-Item -ItemType Directory -Path $outputDir -Force | Out-Null + +$keys = 1..128 +$stream = [System.IO.File]::Open($keysFile, + [System.IO.FileMode]::Create, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None) +$writer = New-Object System.IO.BinaryWriter($stream) +foreach ($key in $keys) { + $writer.Write([UInt32]$key) +} +$writer.Flush() +$writer.Dispose() +$stream.Dispose() + +$keysPath = (Resolve-Path $keysFile).Path +$outputPath = (Resolve-Path $outputDir).Path + +$createCommand = "PerfectHashCreate.exe $keysPath $outputPath " + + "Chm01 Mulshrolate1RX And 0 --DisableCsvOutputFile" +$tableArg = "--TableCreate=$createCommand" + +$serverArgs = @("--Endpoint=$Endpoint") +$clientArgs = @("--Endpoint=$Endpoint", "--Shutdown") + +$server = Start-Process -FilePath $serverExe ` + -ArgumentList $serverArgs ` + -PassThru ` + -NoNewWindow + +Start-Sleep -Milliseconds 500 + +$clientCreate = Start-Process -FilePath $clientExe ` + -ArgumentList @("--Endpoint=$Endpoint", + "`"$tableArg`"") ` + -PassThru ` + -NoNewWindow ` + -Wait + +if ($clientCreate.ExitCode -ne 0) { + throw ("Client create exited with code {0}" -f $clientCreate.ExitCode) +} + +$clientShutdown = Start-Process -FilePath $clientExe ` + -ArgumentList $clientArgs ` + -PassThru ` + -NoNewWindow ` + -Wait + +if ($clientShutdown.ExitCode -ne 0) { + throw ("Client shutdown exited with code {0}" -f $clientShutdown.ExitCode) +} + +if (-not $server.WaitForExit($TimeoutSeconds * 1000)) { + $server.Kill() + throw ("Server did not exit within {0}s" -f $TimeoutSeconds) +} + +$server.WaitForExit() +$server.Refresh() + +if ($null -eq $server.ExitCode) { + $serverProcess = Get-Process -Id $server.Id -ErrorAction SilentlyContinue + if ($null -ne $serverProcess) { + throw "Server exit code unavailable after wait." + } + $serverExitCode = 0 +} else { + $serverExitCode = $server.ExitCode +} + +if ($serverExitCode -ne 0) { + throw ("Server exited with code {0}" -f $serverExitCode) +} + +Write-Host "IOCP smoke test succeeded." diff --git a/scripts/stress-sys32-iocp.ps1 b/scripts/stress-sys32-iocp.ps1 new file mode 100644 index 00000000..f6ab2089 --- /dev/null +++ b/scripts/stress-sys32-iocp.ps1 @@ -0,0 +1,107 @@ +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$Endpoint = "\\.\\pipe\\PerfectHashServer-StressSys32", + [string]$KeysDir = "..\\perfecthash-keys\\sys32", + [string]$OutputDir = "build-win\\iocp-stress-sys32", + [string]$Algorithm = "Chm01", + [string]$HashFunction = "Mulshrolate4RX", + [string]$MaskFunction = "And", + [int]$MaximumConcurrency = 0, + [string[]]$ExtraArgs = @(), + [int]$TimeoutSeconds = 300 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) +$clientExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashClient.exe" -f $Config) + +if (-not (Test-Path $serverExe)) { + throw "Server exe not found: $serverExe" +} + +if (-not (Test-Path $clientExe)) { + throw "Client exe not found: $clientExe" +} + +if (-not (Test-Path $KeysDir)) { + throw "Keys dir not found: $KeysDir" +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +$keysPath = (Resolve-Path $KeysDir).Path +$outputPath = (Resolve-Path $OutputDir).Path + +if ($MaximumConcurrency -le 0) { + $MaximumConcurrency = [Environment]::ProcessorCount +} + +$bulkCommand = ('"{0}" "{1}" {2} {3} {4} {5}' -f $keysPath, + $outputPath, + $Algorithm, + $HashFunction, + $MaskFunction, + $MaximumConcurrency) + +if ($ExtraArgs -and $ExtraArgs.Count -gt 0) { + $bulkCommand += " " + ($ExtraArgs -join " ") +} + +$bulkArg = "--BulkCreateDirectory=$bulkCommand" + +$server = Start-Process -FilePath $serverExe ` + -ArgumentList @("--Endpoint=$Endpoint") ` + -PassThru ` + -NoNewWindow + +Start-Sleep -Milliseconds 500 + +$clientCreate = Start-Process -FilePath $clientExe ` + -ArgumentList @("--Endpoint=$Endpoint", + "`"$bulkArg`"") ` + -PassThru ` + -NoNewWindow ` + -Wait + +$bulkSuccess = 0x2004000F +if ($clientCreate.ExitCode -ne 0 -and $clientCreate.ExitCode -ne $bulkSuccess) { + throw ("Client bulk-create exited with code {0}" -f $clientCreate.ExitCode) +} + +$clientShutdown = Start-Process -FilePath $clientExe ` + -ArgumentList @("--Endpoint=$Endpoint", + "--Shutdown") ` + -PassThru ` + -NoNewWindow ` + -Wait + +if ($clientShutdown.ExitCode -ne 0) { + throw ("Client shutdown exited with code {0}" -f $clientShutdown.ExitCode) +} + +if (-not $server.WaitForExit($TimeoutSeconds * 1000)) { + $server.Kill() + throw ("Server did not exit within {0}s" -f $TimeoutSeconds) +} + +$server.WaitForExit() +$server.Refresh() + +if ($null -eq $server.ExitCode) { + $serverProcess = Get-Process -Id $server.Id -ErrorAction SilentlyContinue + if ($null -ne $serverProcess) { + throw "Server exit code unavailable after wait." + } + $serverExitCode = 0 +} else { + $serverExitCode = $server.ExitCode +} + +if ($serverExitCode -ne 0) { + throw ("Server exited with code {0}" -f $serverExitCode) +} + +Write-Host "IOCP stress test complete." diff --git a/scripts/stress-sys32.ps1 b/scripts/stress-sys32.ps1 new file mode 100644 index 00000000..2ab7fd84 --- /dev/null +++ b/scripts/stress-sys32.ps1 @@ -0,0 +1,51 @@ +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$KeysDir = "..\\perfecthash-keys\\sys32", + [string]$OutputDir = "build-win\\stress-sys32", + [string]$Algorithm = "Chm01", + [string]$HashFunction = "Mulshrolate4RX", + [string]$MaskFunction = "And", + [int]$MaximumConcurrency = 0, + [string[]]$ExtraArgs = @() +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$exe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashBulkCreate.exe" -f $Config) + +if (-not (Test-Path $exe)) { + throw "Bulk create exe not found: $exe" +} + +if (-not (Test-Path $KeysDir)) { + throw "Keys dir not found: $KeysDir" +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +$keysPath = (Resolve-Path $KeysDir).Path +$outputPath = (Resolve-Path $OutputDir).Path + +if ($MaximumConcurrency -le 0) { + $MaximumConcurrency = [Environment]::ProcessorCount +} + +$args = @($keysPath, + $outputPath, + $Algorithm, + $HashFunction, + $MaskFunction, + $MaximumConcurrency) + +if ($ExtraArgs -and $ExtraArgs.Count -gt 0) { + $args += $ExtraArgs +} + +& $exe @args +if ($LASTEXITCODE -ne 0) { + throw ("Bulk create exited with code {0}" -f $LASTEXITCODE) +} + +Write-Host "Stress test complete." diff --git a/scripts/stress-sys32.sh b/scripts/stress-sys32.sh new file mode 100644 index 00000000..109eb240 --- /dev/null +++ b/scripts/stress-sys32.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUILD_DIR="${BUILD_DIR:-build}" +CONFIG="${CONFIG:-Debug}" +KEYS_DIR="${KEYS_DIR:-../perfecthash-keys/sys32}" +OUTPUT_DIR="${OUTPUT_DIR:-${BUILD_DIR}/stress-sys32}" +ALGORITHM="${ALGORITHM:-Chm01}" +HASH_FUNCTION="${HASH_FUNCTION:-Mulshrolate4RX}" +MASK_FUNCTION="${MASK_FUNCTION:-And}" + +if command -v nproc >/dev/null 2>&1; then + MAX_CONCURRENCY="${MAX_CONCURRENCY:-$(nproc)}" +else + MAX_CONCURRENCY="${MAX_CONCURRENCY:-1}" +fi + +EXE="${BUILD_DIR}/bin/${CONFIG}/PerfectHashBulkCreate" +if [[ ! -x "${EXE}" ]]; then + EXE="${BUILD_DIR}/bin/PerfectHashBulkCreate" +fi + +if [[ ! -x "${EXE}" ]]; then + echo "Bulk create exe not found: ${EXE}" >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" + +"${EXE}" \ + "${KEYS_DIR}" \ + "${OUTPUT_DIR}" \ + "${ALGORITHM}" \ + "${HASH_FUNCTION}" \ + "${MASK_FUNCTION}" \ + "${MAX_CONCURRENCY}" \ + "$@" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5d56c1dd..c7c42669 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,6 +27,8 @@ endif() add_subdirectory(PerfectHash) add_subdirectory(PerfectHashCreateExe) add_subdirectory(PerfectHashBulkCreateExe) +add_subdirectory(PerfectHashServerExe) +add_subdirectory(PerfectHashClientExe) if(PERFECTHASH_HAS_CUDA) add_subdirectory(PerfectHashCuda) @@ -37,6 +39,8 @@ if(WIN32) add_dependencies(PerfectHash PerfectHashResources) add_dependencies(PerfectHashCreateExe PerfectHashResources) add_dependencies(PerfectHashBulkCreateExe PerfectHashResources) + add_dependencies(PerfectHashServerExe PerfectHashResources) + add_dependencies(PerfectHashClientExe PerfectHashResources) if(PERFECTHASH_HAS_CUDA) add_dependencies(PerfectHashCuda PerfectHashResources) endif() diff --git a/src/PerfectHash.sln b/src/PerfectHash.sln index 810ca625..a8da928c 100644 --- a/src/PerfectHash.sln +++ b/src/PerfectHash.sln @@ -28,6 +28,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerfectHashCreateExe", "Per {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} = {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerfectHashServerExe", "PerfectHashServerExe\PerfectHashServerExe.vcxproj", "{B269AB7F-EF35-490A-B66D-031E431C4481}" + ProjectSection(ProjectDependencies) = postProject + {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} = {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerfectHashClientExe", "PerfectHashClientExe\PerfectHashClientExe.vcxproj", "{EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}" + ProjectSection(ProjectDependencies) = postProject + {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} = {14D9F1FD-1EC4-47FF-BF73-3868ED05FEB8} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FunctionHook", "FunctionHook\FunctionHook.vcxproj", "{B9EF04DA-0AD9-45C4-97B0-857A13AA4850}" EndProject EndProject @@ -70,6 +80,26 @@ Global {244D25EC-A8AA-4AAD-A66C-D64AFA836CAE}.PGUpdate|x64.Build.0 = PGUpdate|x64 {244D25EC-A8AA-4AAD-A66C-D64AFA836CAE}.Release|x64.ActiveCfg = Release|x64 {244D25EC-A8AA-4AAD-A66C-D64AFA836CAE}.Release|x64.Build.0 = Release|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Debug|x64.ActiveCfg = Debug|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Debug|x64.Build.0 = Debug|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGOptimize|x64.ActiveCfg = PGOptimize|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGOptimize|x64.Build.0 = PGOptimize|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Release|x64.ActiveCfg = Release|x64 + {B269AB7F-EF35-490A-B66D-031E431C4481}.Release|x64.Build.0 = Release|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Debug|x64.ActiveCfg = Debug|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Debug|x64.Build.0 = Debug|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGOptimize|x64.ActiveCfg = PGOptimize|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGOptimize|x64.Build.0 = PGOptimize|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Release|x64.ActiveCfg = Release|x64 + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198}.Release|x64.Build.0 = Release|x64 {B9EF04DA-0AD9-45C4-97B0-857A13AA4850}.Debug|x64.ActiveCfg = Debug|x64 {B9EF04DA-0AD9-45C4-97B0-857A13AA4850}.Debug|x64.Build.0 = Debug|x64 {B9EF04DA-0AD9-45C4-97B0-857A13AA4850}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 diff --git a/src/PerfectHash/CMakeLists.txt b/src/PerfectHash/CMakeLists.txt index ddb75365..a65b6fc5 100644 --- a/src/PerfectHash/CMakeLists.txt +++ b/src/PerfectHash/CMakeLists.txt @@ -8,6 +8,7 @@ set(Private_Header_Files "BulkCreateBestCsv.h" "BulkCreateCsv.h" "Chm01.h" + "Chm01Async.h" "Chm01FileWork.h" "Chm01Private.h" "Chm02Private.h" @@ -25,7 +26,10 @@ set(Private_Header_Files "PerfectHashAllocator.h" "PerfectHashConstants.h" "PerfectHashContext.h" + "PerfectHashContextIocp.h" + "PerfectHashAsync.h" "PerfectHashCu.h" + "PerfectHashClient.h" "PerfectHashDirectory.h" "PerfectHashErrorHandling.h" "PerfectHashEventsPrivate.h" @@ -36,6 +40,7 @@ set(Private_Header_Files "PerfectHashPath.h" "PerfectHashPrimes.h" "PerfectHashPrivate.h" + "PerfectHashServer.h" "PerfectHashTable.h" "PerfectHashTimestamp.h" "PerfectHashTls.h" @@ -202,13 +207,16 @@ set(Source_Files "Math.c" "PerfectHashAllocator.c" "PerfectHashContext.c" + "PerfectHashContextIocp.c" "PerfectHashContextTableCreate.c" + "PerfectHashClient.c" "PerfectHashKeys.c" "PerfectHashKeysLoad.c" "PerfectHashKeysLoadTableSize.c" "PerfectHashNames.c" "PerfectHashPath.c" "PerfectHashPrimes.c" + "PerfectHashServer.c" "PerfectHashTable.c" "PerfectHashTableCreate.c" "PerfectHashTableDelete.c" @@ -263,7 +271,9 @@ endif() if(WIN32) list( APPEND Source_Files + "Chm01Async.c" "Chm01.c" + "PerfectHashAsync.c" "RtlUuid.c" "Security.c" "RtlRandom.c" diff --git a/src/PerfectHash/Chm01.c b/src/PerfectHash/Chm01.c index 10644e16..5ffad096 100644 --- a/src/PerfectHash/Chm01.c +++ b/src/PerfectHash/Chm01.c @@ -415,24 +415,21 @@ Return Value: TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); - ASSERT(!TlsContext->Flags.DisableGlobalAllocatorComponent); - ASSERT(!TlsContext->Flags.CustomAllocatorDetailsPresent); - ASSERT(!TlsContext->HeapCreateFlags); - ASSERT(!TlsContext->HeapMinimumSize); - TlsContextDisableGlobalAllocator(TlsContext); TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; TlsContext->HeapMinimumSize = (ULONG_PTR)Info.AllocSize; // - // Make sure we haven't overflowed MAX_ULONG. This should be caught - // earlier when preparing the graph info. + // Make sure we haven't overflowed the heap minimum size on 32-bit builds. + // This should be caught earlier when preparing the graph info. // +#if defined(_M_IX86) if ((ULONGLONG)TlsContext->HeapMinimumSize != Info.AllocSize) { PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); } +#endif // // Copy the table create flags into the TLS context as well; they are now @@ -646,7 +643,7 @@ Return Value: ZeroStructInline(Verb##Name); \ Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_PREPARE_FILE_WORK() \ PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) @@ -1057,7 +1054,7 @@ Return Value: // if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); } // @@ -1435,7 +1432,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->ConsoleWork, TRUE); WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); } // @@ -1485,14 +1482,14 @@ Return Value: Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ Verb##Name.EndOfFile = EndOfFile; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_CLOSE_FILE_WORK() \ CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_CLOSE_FILE_WORK) SUBMIT_CLOSE_FILE_WORK(); - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); #define EXPAND_AS_CHECK_CLOSE_ERRORS( \ Verb, VUpper, Name, Upper, \ diff --git a/src/PerfectHash/Chm01Async.c b/src/PerfectHash/Chm01Async.c new file mode 100644 index 00000000..a6e91e22 --- /dev/null +++ b/src/PerfectHash/Chm01Async.c @@ -0,0 +1,1833 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + Chm01Async.c + +Abstract: + + This module implements the CHM01 asynchronous table creation state + machine. Work is decomposed into fine-grained IOCP-driven steps that + overlap compute and file I/O. + +--*/ + +#include "stdafx.h" +#include "Chm01.h" +#include "Chm01Private.h" +#include "Chm01FileWork.h" +#include "Chm01Async.h" + +// +// Async job flags. +// + +#define CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES 0x00000001 +#define CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED 0x00000002 +#define CHM01_ASYNC_JOB_FLAG_TRY_LARGER_TABLE_SIZE 0x00000004 + +typedef struct _CHM01_ASYNC_GRAPH_WORK { + PERFECT_HASH_ASYNC_WORK Work; + PCHM01_ASYNC_JOB Job; + PGRAPH Graph; + BOOLEAN Started; + BYTE Padding[7]; +} CHM01_ASYNC_GRAPH_WORK; +typedef CHM01_ASYNC_GRAPH_WORK *PCHM01_ASYNC_GRAPH_WORK; + +FORCEINLINE +BOOLEAN +Chm01AsyncShouldSkipContextFileWork( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ FILE_WORK_ID FileWorkId + ) +{ + FILE_ID FileId; + + if (!SkipContextFileWork(Context)) { + return FALSE; + } + + FileId = FileWorkIdToFileId(FileWorkId); + + return IsValidContextFileId((CONTEXT_FILE_ID)FileId); +} + +static +HRESULT +Chm01AsyncGraphStep( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work + ); + +static +VOID +Chm01AsyncGraphComplete( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ); + +static +HRESULT +Chm01AsyncDispatchGraphWork( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncInitializeJob( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncPollSolveEvents( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeWaitPrepare( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeVerify( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeWaitSave( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncFinalizeClose( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncReleaseGraphs( + _In_ PCHM01_ASYNC_JOB Job + ); + +static +HRESULT +Chm01AsyncStep( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work + ); + +static +VOID +Chm01AsyncComplete( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ); + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncGraphStep( + PPERFECT_HASH_ASYNC_WORK Work + ) +{ + PGRAPH Graph; + PGRAPH LockedGraph; + PGRAPH NewGraph; + ULONG OldNumberOfVertices; + HRESULT Result; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); + Graph = GraphWork->Graph; + Context = Graph->Context; + + if (!GraphWork->Started) { + GraphWork->Started = TRUE; + InterlockedIncrement(&Context->ActiveSolvingLoops); + } + + LockedGraph = Graph; + AcquireGraphLockExclusive(LockedGraph); + + if (!IsGraphInfoLoaded(LockedGraph)) { + Result = LockedGraph->Vtbl->LoadInfo(LockedGraph); + if (FAILED(Result)) { + if (Result != E_OUTOFMEMORY) { + PH_ERROR(GraphLoadInfo, Result); + Result = PH_E_INVARIANT_CHECK_FAILED; + } else { + if (InterlockedDecrement(&Context->GraphMemoryFailures) == 0) { + Context->State.AllGraphsFailedMemoryAllocation = TRUE; + SetStopSolving(Context); + if (!SetEvent(Context->FailedEvent)) { + SYS_ERROR(SetEvent); + } + } + Result = S_OK; + } + goto End; + } + } + + if (!LockedGraph->Vtbl->ShouldWeContinueTryingToSolve(LockedGraph)) { + Result = S_OK; + goto End; + } + + Result = LockedGraph->Vtbl->Reset(LockedGraph); + if (FAILED(Result)) { + if (Result == PH_E_NO_MORE_SEEDS) { + Result = S_OK; + } + goto End; + } + + if (Result != PH_S_CONTINUE_GRAPH_SOLVING) { + Result = S_OK; + goto End; + } + + Result = LockedGraph->Vtbl->LoadNewSeeds(LockedGraph); + if (FAILED(Result)) { + if (Result == PH_E_NO_MORE_SEEDS) { + Result = S_OK; + } + goto End; + } + + NewGraph = NULL; + Result = LockedGraph->Vtbl->Solve(LockedGraph, &NewGraph); + if (FAILED(Result)) { + goto End; + } + + if (Result == PH_S_STOP_GRAPH_SOLVING || + Result == PH_S_GRAPH_SOLVING_STOPPED) { + Result = S_OK; + goto End; + } + + if (Result == PH_S_USE_NEW_GRAPH_FOR_SOLVING) { + if (!NewGraph) { + Result = PH_E_INVARIANT_CHECK_FAILED; + goto End; + } + + OldNumberOfVertices = LockedGraph->NumberOfVertices; + + AcquireGraphLockExclusive(NewGraph); + ReleaseGraphLockExclusive(LockedGraph); + LockedGraph = NewGraph; + GraphWork->Graph = NewGraph; + + if (!IsGraphInfoLoaded(LockedGraph) || + LockedGraph->LastLoadedNumberOfVertices < OldNumberOfVertices) { + Result = LockedGraph->Vtbl->LoadInfo(LockedGraph); + if (FAILED(Result)) { + PH_ERROR(GraphLoadInfo_NewGraph, Result); + goto End; + } + } + + Result = S_FALSE; + goto End; + } + + ASSERT(Result == PH_S_CONTINUE_GRAPH_SOLVING); + + Result = S_FALSE; + +End: + + if (LockedGraph) { + ReleaseGraphLockExclusive(LockedGraph); + } + + return Result; +} + +_Use_decl_annotations_ +static +VOID +Chm01AsyncGraphComplete( + PPERFECT_HASH_ASYNC_WORK Work, + HRESULT Result + ) +{ + PGRAPH Graph; + PHANDLE Event; + ULONG WaitResult; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_JOB Job; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + UNREFERENCED_PARAMETER(Result); + + GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); + Job = GraphWork->Job; + Graph = GraphWork->Graph; + Context = Graph->Context; + + if (GraphWork->Started) { + InterlockedDecrement(&Context->ActiveSolvingLoops); + } + + if (InterlockedDecrement(&Context->RemainingSolverLoops) == 0) { + WaitResult = WaitForSingleObject(Context->TryLargerTableSizeEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + if (Context->FinishedCount == 0) { + Event = &Context->FailedEvent; + } else { + Event = &Context->SucceededEvent; + } + if (!SetEvent(*Event)) { + SYS_ERROR(SetEvent); + } + SetStopSolving(Context); + } + } + + if (InterlockedDecrement(&Job->ActiveGraphs) == 0) { + if (Job->GraphsCompleteEvent) { + SetEvent(Job->GraphsCompleteEvent); + } + } + + if (Job->Allocator) { + Job->Allocator->Vtbl->FreePointer(Job->Allocator, (PVOID *)&GraphWork); + } +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncDispatchGraphWork( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONG Index; + ULONG ActiveGraphs; + PGRAPH Graph; + PGRAPH *Graphs; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + Context = Job->Context; + Graphs = Job->Graphs; + ActiveGraphs = 0; + + if (Job->GraphsCompleteEvent) { + ResetEvent(Job->GraphsCompleteEvent); + } + + ASSERT(Context->MainWorkList->Vtbl->IsEmpty(Context->MainWorkList)); + ASSERT(Context->FinishedWorkList->Vtbl->IsEmpty(Context->FinishedWorkList)); + + if (FirstSolvedGraphWins(Context)) { + ASSERT(Job->NumberOfGraphs == Job->Concurrency); + } else { + ASSERT(Job->NumberOfGraphs - 1 == Job->Concurrency); + } + + for (Index = 0; Index < Job->NumberOfGraphs; Index++) { + + Graph = Graphs[Index]; + + ResetSRWLock(&Graph->Lock); + InitializeListHead(&Graph->ListEntry); + + AcquireGraphLockExclusive(Graph); + Result = Graph->Vtbl->SetInfo(Graph, &Job->GraphInfo); + ReleaseGraphLockExclusive(Graph); + + if (FAILED(Result)) { + PH_ERROR(GraphSetInfo, Result); + return Result; + } + + Graph->Flags.IsInfoLoaded = FALSE; + + if (!FirstSolvedGraphWins(Context) && Index == 0) { + Graph->Flags.IsSpare = TRUE; + + _Benign_race_begin_ + Context->SpareGraph = Graph; + _Benign_race_end_ + + continue; + } + + Graph->Flags.IsSpare = FALSE; + + GraphWork = (PCHM01_ASYNC_GRAPH_WORK)( + Job->Allocator->Vtbl->Calloc( + Job->Allocator, + 1, + sizeof(*GraphWork) + ) + ); + + if (!GraphWork) { + return E_OUTOFMEMORY; + } + + GraphWork->Job = Job; + GraphWork->Graph = Graph; + GraphWork->Work.Step = Chm01AsyncGraphStep; + GraphWork->Work.Complete = Chm01AsyncGraphComplete; + GraphWork->Work.SliceBudget = Job->SliceBudget; + + ActiveGraphs++; + + Result = PerfectHashAsyncSubmit(&Job->Async, &GraphWork->Work); + if (FAILED(Result)) { + return Result; + } + } + + Job->ActiveGraphs = (LONG)ActiveGraphs; + + if (ActiveGraphs == 0 && Job->GraphsCompleteEvent) { + SetEvent(Job->GraphsCompleteEvent); + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncInitializeJob( + PCHM01_ASYNC_JOB Job + ) +{ + PRTL Rtl; + USHORT Index; + DOUBLE Limit; + DOUBLE Current; + HRESULT Result; + BOOLEAN LimitConcurrency; + ULONG Concurrency; + ULONG NumberOfGraphs; + ULONG NumberOfSeedsRequired; + ULONG NumberOfSeedsAvailable; + PGRAPH Graph; + PGRAPH *Graphs; + PALLOCATOR Allocator; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; + GRAPH_INFO_ON_DISK *GraphInfoOnDisk; + PTABLE_INFO_ON_DISK TableInfoOnDisk; + PHANDLE Event; + ULONG NumberOfEvents; + + Table = Job->Table; + Context = Table->Context; + Rtl = Table->Rtl; + Allocator = Table->Allocator; + TableCreateFlags.AsULongLong = Table->TableCreateFlags.AsULongLong; + + // + // Initialize event arrays. + // + + Job->Events[0] = Context->SucceededEvent; + Job->Events[1] = Context->CompletedEvent; + Job->Events[2] = Context->ShutdownEvent; + Job->Events[3] = Context->FailedEvent; + Job->Events[4] = Context->TryLargerTableSizeEvent; + Job->Events[5] = Context->LowMemoryEvent; + + { + PHANDLE SaveEvent = Job->SaveEvents; + PHANDLE PrepareEvent = Job->PrepareEvents; + +#define EXPAND_AS_ASSIGN_EVENT( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + *Verb##Event++ = Context->Verb##d##Name##Event; + + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASSIGN_EVENT); + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASSIGN_EVENT); + +#undef EXPAND_AS_ASSIGN_EVENT + } + + // + // Set up on-disk info buffers. + // + + GraphInfoOnDisk = &Job->GraphInfoOnDisk; + TableInfoOnDisk = &GraphInfoOnDisk->TableInfoOnDisk; + + Job->TableInfoOnDisk = TableInfoOnDisk; + Context->GraphInfoOnDisk = GraphInfoOnDisk; + Table->TableInfoOnDisk = TableInfoOnDisk; + + // + // Verify we have sufficient seeds available for the hash function. + // + + NumberOfSeedsRequired = HashRoutineNumberOfSeeds[Table->HashFunctionId]; + NumberOfSeedsAvailable = (( + FIELD_OFFSET(GRAPH, LastSeed) - + FIELD_OFFSET(GRAPH, FirstSeed) + ) / sizeof(ULONG)) + 1; + + if (NumberOfSeedsAvailable < NumberOfSeedsRequired) { + return PH_E_INVALID_NUMBER_OF_SEEDS; + } + + // + // Determine concurrency. + // + + Concurrency = Context->MaximumConcurrency; + + LimitConcurrency = ( + Table->PriorPredictedAttempts > 0 && + TableCreateFlags.TryUsePredictedAttemptsToLimitMaxConcurrency != FALSE + ); + + if (LimitConcurrency) { + Concurrency = min(Concurrency, Table->PriorPredictedAttempts); + } + + Job->Concurrency = Concurrency; + + if (FirstSolvedGraphWins(Context)) { + NumberOfGraphs = Concurrency; + } else { + NumberOfGraphs = Concurrency + 1; + if (NumberOfGraphs == 0) { + return E_INVALIDARG; + } + } + + Job->NumberOfGraphs = NumberOfGraphs; + + if (WantsAutoResizeIfKeysToEdgesRatioExceedsLimit(Table)) { + Limit = Table->AutoResizeWhenKeysToEdgesRatioExceeds; + Current = Table->Keys->KeysToEdgesRatio; + if (Current > Limit) { + Context->InitialResizes = 1; + } + } + + if (Context->InitialResizes > 0) { + ULONG InitialResizes; + ULARGE_INTEGER NumberOfEdges; + ULARGE_INTEGER NumberOfVertices; + + NumberOfEdges.QuadPart = Table->Keys->NumberOfKeys.QuadPart; + ASSERT(NumberOfEdges.HighPart == 0); + + NumberOfEdges.QuadPart = ( + Rtl->RoundUpPowerOfTwo32( + NumberOfEdges.LowPart + ) + ); + + if (NumberOfEdges.QuadPart < 8) { + NumberOfEdges.QuadPart = 8; + } + + if (NumberOfEdges.HighPart) { + Result = PH_E_TOO_MANY_EDGES; + goto Error; + } + + NumberOfVertices.QuadPart = ( + Rtl->RoundUpNextPowerOfTwo32(NumberOfEdges.LowPart) + ); + + Table->RequestedNumberOfTableElements.QuadPart = ( + NumberOfVertices.QuadPart + ); + + for (InitialResizes = Context->InitialResizes; + InitialResizes > 0; + InitialResizes--) { + + Table->RequestedNumberOfTableElements.QuadPart <<= 1ULL; + + if (Table->RequestedNumberOfTableElements.HighPart) { + Result = PH_I_REQUESTED_NUMBER_OF_TABLE_ELEMENTS_TOO_LARGE; + goto Error; + } + } + + Context->NumberOfTableResizeEvents = Context->InitialResizes; + } + + Result = PrepareGraphInfoChm01(Table, &Job->GraphInfo, NULL); + if (FAILED(Result)) { + PH_ERROR(CreatePerfectHashTableImplChm01_PrepareFirstGraphInfo, Result); + goto Error; + } + + // + // Allocate graph instances. + // + + Graphs = (PGRAPH *)( + Allocator->Vtbl->Calloc( + Allocator, + NumberOfGraphs, + sizeof(Graph) + ) + ); + + if (!Graphs) { + Result = PH_I_OUT_OF_MEMORY; + goto Error; + } + + Job->Graphs = Graphs; + + // + // Use per-graph allocator instances (TLS-toggled). + // + + TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + + TlsContextDisableGlobalAllocator(TlsContext); + TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; + TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; + TlsContext->HeapMinimumSize = (ULONG_PTR)Job->GraphInfo.AllocSize; + +#if defined(_M_IX86) + if ((ULONGLONG)TlsContext->HeapMinimumSize != Job->GraphInfo.AllocSize) { + PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); + } +#endif + + TlsContext->TableCreateFlags.AsULongLong = TableCreateFlags.AsULongLong; + TlsContext->Table = Table; + + for (Index = 0; Index < NumberOfGraphs; Index++) { + + Result = Table->Vtbl->CreateInstance(Table, + NULL, + &IID_PERFECT_HASH_GRAPH, + (PVOID *)&Graph); + + if (FAILED(Result)) { + if (Result != E_OUTOFMEMORY) { + PH_ERROR(CreatePerfectHashTableImplChm01_CreateGraph, Result); + } + break; + } + +#ifdef PH_WINDOWS + ASSERT(Graph->Allocator != Table->Allocator); + ASSERT(Graph->Allocator->HeapHandle != Table->Allocator->HeapHandle); +#endif + + ASSERT(Graph->Rtl == Table->Rtl); + + Graph->Flags.SkipVerification = ( + TableCreateFlags.SkipGraphVerification != FALSE + ); + + Graph->Flags.WantsWriteCombiningForVertexPairsArray = ( + TableCreateFlags.EnableWriteCombineForVertexPairs != FALSE + ); + + Graph->Flags.RemoveWriteCombineAfterSuccessfulHashKeys = ( + TableCreateFlags.RemoveWriteCombineAfterSuccessfulHashKeys != FALSE + ); + + Graph->Index = Index; + Graphs[Index] = Graph; + } + + TlsContextEnableGlobalAllocator(TlsContext); + TlsContext->Flags.CustomAllocatorDetailsPresent = FALSE; + TlsContext->HeapCreateFlags = 0; + TlsContext->HeapMinimumSize = 0; + TlsContext->Table = NULL; + + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + + if (FAILED(Result)) { + goto Error; + } + + // + // Reset all context events. + // + + Event = (PHANDLE)&Context->FirstEvent; + NumberOfEvents = GetNumberOfContextEvents(Context); + + for (Index = 0; Index < NumberOfEvents; Index++, Event++) { + if (!ResetEvent(*Event)) { + SYS_ERROR(ResetEvent); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + } + + Context->LowMemoryObserved = 0; + Context->State.AllGraphsFailedMemoryAllocation = FALSE; + Context->State.SolveTimeoutExpired = FALSE; + Context->State.FixedAttemptsReached = FALSE; + Context->State.MaxAttemptsReached = FALSE; + Context->State.UserRequestedResize = FALSE; + ClearStopSolving(Context); + + Job->Attempt = 1; + + // + // Set callbacks and algorithm context. + // + + Context->MainWorkCallback = ProcessGraphCallbackChm01; + Context->ConsoleWorkCallback = ProcessConsoleCallbackChm01; + Context->AlgorithmContext = &Job->GraphInfo; + Context->FileWorkCallback = FileWorkCallbackChm01; + + // + // Prepare output directory if required. + // + + if (!NoFileIo(Table)) { + Result = PrepareTableOutputDirectory(Table); + if (FAILED(Result)) { + PH_ERROR(PrepareTableOutputDirectory, Result); + goto Error; + } + } + + // + // Configure solve timeout if requested. + // + + if (Table->MaxSolveTimeInSeconds > 0) { + SetThreadpoolTimer(Context->SolveTimeout, + &Table->RelativeMaxSolveTimeInFiletime.AsFileTime, + 0, + 0); + } + + // + // Submit prepare file work. + // + + ASSERT(Context->FileWorkList->Vtbl->IsEmpty(Context->FileWorkList)); + + if (!NoFileIo(Table)) { + +#define EXPAND_AS_SUBMIT_FILE_WORK( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + ZeroStructInline(Job->Verb##Name); \ + Job->Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ + if (Chm01AsyncShouldSkipContextFileWork( \ + Context, \ + Job->Verb##Name.FileWorkId)) { \ + if (!SetEvent(Context->Verb##d##Name##Event)) { \ + SYS_ERROR(SetEvent); \ + Result = PH_E_SYSTEM_CALL_FAILED; \ + goto Error; \ + } \ + } else { \ + InsertTailFileWork(Context, &Job->Verb##Name.ListEntry); \ + PerfectHashContextSubmitFileWork(Context); \ + } + +#define SUBMIT_PREPARE_FILE_WORK() \ + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) + + SUBMIT_PREPARE_FILE_WORK(); + +#undef SUBMIT_PREPARE_FILE_WORK +#undef EXPAND_AS_SUBMIT_FILE_WORK + } + + // + // Capture initial timing. + // + + QueryPerformanceFrequency(&Context->Frequency); + CONTEXT_START_TIMERS(Solve); + Context->StartMilliseconds = GetTickCount64(); + + // + // Reset counters. + // + + Context->GraphMemoryFailures = Concurrency; + Context->RemainingSolverLoops = Concurrency; + Context->ActiveSolvingLoops = 0; + Context->Attempts = 0; + Context->FailedAttempts = 0; + Context->FinishedCount = 0; + Context->HighestDeletedEdgesCount = 0; + + // + // Dispatch graph work. + // + + Result = Chm01AsyncDispatchGraphWork(Job); + if (FAILED(Result)) { + goto Error; + } + + return S_OK; + +Error: + + if (Result == E_OUTOFMEMORY) { + Result = PH_I_OUT_OF_MEMORY; + } + + if (!NoFileIo(Table)) { + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); + } + + if (Result != S_OK) { + SetEvent(Context->ShutdownEvent); + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncPollSolveEvents( + PCHM01_ASYNC_JOB Job + ) +{ + ULONG WaitResult; + BOOLEAN TryLargerTableSize; + PPERFECT_HASH_CONTEXT Context; + + Context = Job->Context; + + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->Events), + Job->Events, + FALSE, + 0); + + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + + SetStopSolving(Context); + + if (CtrlCPressed) { + return PH_E_CTRL_C_PRESSED; + } + + if (WaitResult == WAIT_OBJECT_0 + 5) { + InterlockedIncrement(&Context->LowMemoryObserved); + return PH_I_LOW_MEMORY; + } + + TryLargerTableSize = ( + WaitResult == WAIT_OBJECT_0 + 4 || ( + WaitForSingleObject(Context->TryLargerTableSizeEvent, 0) == + WAIT_OBJECT_0 + ) + ); + + if (TryLargerTableSize) { + Job->Flags |= CHM01_ASYNC_JOB_FLAG_TRY_LARGER_TABLE_SIZE; + Job->LastResult = PH_E_NOT_IMPLEMENTED; + return S_OK; + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeWaitPrepare( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONG WaitResult; + + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->PrepareEvents), + Job->PrepareEvents, + TRUE, + 0); + + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + + Result = S_OK; + +#undef EXPAND_AS_CHECK_ERRORS +#define EXPAND_AS_CHECK_ERRORS( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Job->Verb##Name.NumberOfErrors > 0) { \ + Result = Job->Verb##Name.LastResult; \ + if (Result == S_OK || Result == E_UNEXPECTED) { \ + Result = PH_E_ERROR_DURING_##VUpper##_##Upper; \ + } \ + PH_ERROR( \ + CreatePerfectHashTableImplChm01_ErrorDuring##Verb##Name, \ + Result \ + ); \ + goto Error; \ + } + + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CHECK_ERRORS) + +#undef EXPAND_AS_CHECK_ERRORS + + return S_OK; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeVerify( + PCHM01_ASYNC_JOB Job + ) +{ + PRTL Rtl; + PRNG Rng; + HRESULT Result; + PGRAPH Graph; + ULONG WaitResult; + PLIST_ENTRY ListEntry; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; + BOOLEAN FailedEventSet; + BOOLEAN ShutdownEventSet; + BOOLEAN LowMemoryEventSet = FALSE; + + Context = Job->Context; + Table = Job->Table; + Rtl = Table->Rtl; + TableCreateFlags.AsULongLong = Table->TableCreateFlags.AsULongLong; + + if (Job->GraphsCompleteEvent) { + WaitResult = WaitForSingleObject(Job->GraphsCompleteEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + return S_FALSE; + } + } + + if (CtrlCPressed) { + return PH_E_CTRL_C_PRESSED; + } + + if (Context->FinishedCount == 0) { + + WaitResult = WaitForSingleObject(Context->FailedEvent, 0); + FailedEventSet = (WaitResult == WAIT_OBJECT_0); + + WaitResult = WaitForSingleObject(Context->ShutdownEvent, 0); + ShutdownEventSet = (WaitResult == WAIT_OBJECT_0); + + if ((!FailedEventSet && !ShutdownEventSet) || + Context->LowMemoryObserved > 0) { + LowMemoryEventSet = TRUE; + } + + if (LowMemoryEventSet) { + Result = PH_I_LOW_MEMORY; + } else if (FailedEventSet) { + if (Context->State.AllGraphsFailedMemoryAllocation != FALSE) { + Result = PH_I_FAILED_TO_ALLOCATE_MEMORY_FOR_ALL_GRAPHS; + } else if (Context->State.SolveTimeoutExpired != FALSE) { + Result = PH_I_SOLVE_TIMEOUT_EXPIRED; + } else { + Result = PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION; + } + } else if (ShutdownEventSet) { + Result = PH_I_CREATE_TABLE_ROUTINE_RECEIVED_SHUTDOWN_EVENT; + } else { + Result = PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION; + } + + return Result; + } + + ASSERT(Context->FinishedCount > 0); + + if (!TableCreateFlags.Quiet) { + Result = PrintCurrentContextStatsChm01(Context); + if (FAILED(Result)) { + PH_ERROR(PrintCurrentContextStatsChm01, Result); + return Result; + } + } + + if (FirstSolvedGraphWins(Context)) { + ListEntry = NULL; + if (!RemoveHeadFinishedWork(Context, &ListEntry)) { + Result = PH_E_GUARDED_LIST_EMPTY; + PH_ERROR(PerfectHashCreateChm01Callback_RemoveFinishedWork, Result); + return Result; + } + + Graph = CONTAINING_RECORD(ListEntry, GRAPH, ListEntry); + + } else { + + EnterCriticalSection(&Context->BestGraphCriticalSection); + Graph = Context->BestGraph; + Context->BestGraph = NULL; + LeaveCriticalSection(&Context->BestGraphCriticalSection); + + ASSERT(Graph != NULL); + } + + Table->MaximumGraphTraversalDepth = Graph->MaximumTraversalDepth; + Table->NumberOfEmptyVertices = Graph->NumberOfEmptyVertices; + Table->NumberOfCollisionsDuringAssignment = Graph->Collisions; + + Table->Flags.VertexPairsArrayUsesLargePages = ( + Graph->Flags.VertexPairsArrayUsesLargePages + ); + + Table->Flags.UsedAvx2HashFunction = Graph->Flags.UsedAvx2HashFunction; + Table->Flags.UsedAvx512HashFunction = Graph->Flags.UsedAvx512HashFunction; + Table->Flags.UsedAvx2MemoryCoverageFunction = + Graph->Flags.UsedAvx2MemoryCoverageFunction; + + COPY_GRAPH_COUNTERS_FROM_GRAPH_TO_TABLE(); + + if (Context->RngId != PerfectHashRngSystemId) { + Table->RngSeed = Graph->Rng->Seed; + Table->RngSubsequence = Graph->Rng->Subsequence; + Table->RngOffset = Graph->Rng->Offset; + + Rng = Graph->Rng; + Result = Rng->Vtbl->GetCurrentOffset(Rng, &Table->RngCurrentOffset); + if (FAILED(Result)) { + PH_ERROR(CreatePerfectHashTableImplChm01_RngGetCurrentOffset, + Result); + return Result; + } + } + + Table->SolveDurationInSeconds = (DOUBLE)( + ((DOUBLE)Context->SolveElapsedMicroseconds.QuadPart) / + ((DOUBLE)1e6) + ); + + Table->SolutionsFoundRatio = (DOUBLE)( + ((DOUBLE)Context->FinishedCount) / + ((DOUBLE)Context->Attempts) + ); + + Result = CalculatePredictedAttempts(Table->SolutionsFoundRatio, + &Table->PredictedAttempts); + if (FAILED(Result)) { + PH_ERROR(CreatePerfectHashTableImplChm01_CalculatePredictedAttempts, + Result); + return Result; + } + + Context->SolvedContext = Graph; + + if (!NoFileIo(Table)) { + +#define EXPAND_AS_SUBMIT_FILE_WORK( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + ZeroStructInline(Job->Verb##Name); \ + Job->Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ + if (Chm01AsyncShouldSkipContextFileWork( \ + Context, \ + Job->Verb##Name.FileWorkId)) { \ + if (!SetEvent(Context->Verb##d##Name##Event)) { \ + SYS_ERROR(SetEvent); \ + return PH_E_SYSTEM_CALL_FAILED; \ + } \ + } else { \ + InsertTailFileWork(Context, &Job->Verb##Name.ListEntry); \ + PerfectHashContextSubmitFileWork(Context); \ + } + +#define SUBMIT_SAVE_FILE_WORK() \ + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) + + SUBMIT_SAVE_FILE_WORK(); + +#undef SUBMIT_SAVE_FILE_WORK +#undef EXPAND_AS_SUBMIT_FILE_WORK + + } else { + + PGRAPH_INFO_ON_DISK NewGraphInfoOnDisk; + PGRAPH_INFO_ON_DISK ExistingGraphInfoOnDisk; + + ExistingGraphInfoOnDisk = Context->GraphInfoOnDisk; + + NewGraphInfoOnDisk = ( + Job->Allocator->Vtbl->Calloc( + Job->Allocator, + 1, + sizeof(*NewGraphInfoOnDisk) + ) + ); + + if (!NewGraphInfoOnDisk) { + return E_OUTOFMEMORY; + } + + CopyMemory(NewGraphInfoOnDisk, + ExistingGraphInfoOnDisk, + sizeof(*NewGraphInfoOnDisk)); + + if (Graph->FirstSeed == 0) { + Result = PH_E_INVARIANT_CHECK_FAILED; + PH_ERROR(CreatePerfectHashTableImplChm01_GraphFirstSeedIs0, Result); + PH_RAISE(Result); + } + + CopyMemory(&NewGraphInfoOnDisk->TableInfoOnDisk.FirstSeed, + &Graph->FirstSeed, + Graph->NumberOfSeeds * sizeof(Graph->FirstSeed)); + + Table->TableInfoOnDisk = &NewGraphInfoOnDisk->TableInfoOnDisk; + Table->State.TableInfoOnDiskWasHeapAllocated = TRUE; + + if (!IsTableCreateOnly(Table)) { + PVOID BaseAddress; + LONGLONG SizeInBytes; + BOOLEAN LargePagesForTableData; + PRTL_TRY_LARGE_PAGE_VIRTUAL_ALLOC TryLargePageVirtualAlloc; + + LargePagesForTableData = ( + Table->TableCreateFlags.TryLargePagesForTableData == TRUE + ); + + SizeInBytes = ( + Job->TableInfoOnDisk->NumberOfTableElements.QuadPart * + Job->TableInfoOnDisk->AssignedElementSizeInBytes + ); + + TryLargePageVirtualAlloc = Rtl->Vtbl->TryLargePageVirtualAlloc; + BaseAddress = TryLargePageVirtualAlloc(Rtl, + NULL, + SizeInBytes, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE, + &LargePagesForTableData); + + Table->TableDataBaseAddress = BaseAddress; + Table->TableDataSizeInBytes = SizeInBytes; + + if (!BaseAddress) { + return E_OUTOFMEMORY; + } + + Table->State.TableDataWasHeapAllocated = TRUE; + Table->Flags.TableDataUsesLargePages = LargePagesForTableData; + + CopyMemory(Table->TableDataBaseAddress, + Graph->Assigned, + SizeInBytes); + } + } + + CONTEXT_START_TIMERS(Verify); + + Result = Graph->Vtbl->Verify(Graph); + + CONTEXT_END_TIMERS(Verify); + + if (!SetEvent(Context->VerifiedTableEvent)) { + SYS_ERROR(SetEvent); + return PH_E_SYSTEM_CALL_FAILED; + } + + if (FAILED(Result)) { + return PH_E_TABLE_VERIFICATION_FAILED; + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeWaitSave( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONG WaitResult; + + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->SaveEvents), + Job->SaveEvents, + TRUE, + 0); + + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + + Result = S_OK; + +#undef EXPAND_AS_CHECK_ERRORS +#define EXPAND_AS_CHECK_ERRORS( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Job->Verb##Name.NumberOfErrors > 0) { \ + Result = Job->Verb##Name.LastResult; \ + if (Result == S_OK || Result == E_UNEXPECTED) { \ + Result = PH_E_ERROR_DURING_##VUpper##_##Upper; \ + } \ + PH_ERROR( \ + CreatePerfectHashTableImplChm01_ErrorDuring##Verb##Name, \ + Result \ + ); \ + goto Error; \ + } + + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CHECK_ERRORS) + +#undef EXPAND_AS_CHECK_ERRORS + + return S_OK; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncFinalizeClose( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + HRESULT CloseResult; + ULONG CloseFileErrorCount = 0; + ULONG WaitResult; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PLARGE_INTEGER EndOfFile = NULL; + LARGE_INTEGER EmptyEndOfFile = { 0 }; + + Context = Job->Context; + Table = Job->Table; + Result = S_OK; + CloseResult = S_OK; + + if (Job->Attempt == 0 || NoFileIo(Table)) { + return S_OK; + } + + if (Job->GraphsCompleteEvent) { + WaitResult = WaitForSingleObject(Job->GraphsCompleteEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + return S_FALSE; + } + } + + if (!NoFileIo(Table)) { + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->PrepareEvents), + Job->PrepareEvents, + TRUE, + 0); + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + } + + if (Job->Flags & CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES) { + EndOfFile = &EmptyEndOfFile; + } + + if ((Job->Flags & CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED) == 0) { + +#define EXPAND_AS_SUBMIT_CLOSE_FILE_WORK( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + ZeroStructInline(Job->Verb##Name); \ + Job->Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ + Job->Verb##Name.EndOfFile = EndOfFile; \ + if (!Chm01AsyncShouldSkipContextFileWork( \ + Context, \ + Job->Verb##Name.FileWorkId)) { \ + InsertTailFileWork(Context, &Job->Verb##Name.ListEntry); \ + PerfectHashContextSubmitFileWork(Context); \ + } + +#define SUBMIT_CLOSE_FILE_WORK() \ + CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_CLOSE_FILE_WORK) + + SUBMIT_CLOSE_FILE_WORK(); + +#undef SUBMIT_CLOSE_FILE_WORK +#undef EXPAND_AS_SUBMIT_CLOSE_FILE_WORK + + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED; + } + + if (Context->FileWorkOutstandingEvent) { + WaitResult = WaitForSingleObject(Context->FileWorkOutstandingEvent, 0); + if (WaitResult != WAIT_OBJECT_0) { + return S_FALSE; + } + } + +#define EXPAND_AS_CHECK_CLOSE_ERRORS( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Job->Verb##Name.NumberOfErrors > 0) { \ + CloseResult = Job->Verb##Name.LastResult; \ + if (CloseResult == S_OK || CloseResult == E_UNEXPECTED) { \ + CloseResult = PH_E_ERROR_DURING_##VUpper##_##Upper; \ + } \ + PH_ERROR( \ + CreatePerfectHashTableImplChm01_ErrorDuring##Verb##Name, \ + Result \ + ); \ + CloseFileErrorCount++; \ + } + + CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CHECK_CLOSE_ERRORS) + +#undef EXPAND_AS_CHECK_CLOSE_ERRORS + + if (CloseFileErrorCount > 0) { + Result = CloseResult; + } + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncReleaseGraphs( + PCHM01_ASYNC_JOB Job + ) +{ + USHORT Index; + ULONG ReferenceCount; + PGRAPH Graph; + PGRAPH *Graphs; + PALLOCATOR Allocator; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TABLE Table; + PHANDLE Event; + ULONG NumberOfEvents; + HRESULT Result; + + Result = Job->LastResult; + Context = Job->Context; + Table = Job->Table; + Allocator = Job->Allocator; + Graphs = Job->Graphs; + + if (Result == E_OUTOFMEMORY) { + Result = PH_I_OUT_OF_MEMORY; + } + + if (Graphs) { + for (Index = 0; Index < Job->NumberOfGraphs; Index++) { + + Graph = Graphs[Index]; + if (!Graph) { + continue; + } + + ReferenceCount = Graph->Vtbl->Release(Graph); + + if (ReferenceCount != 0) { + Result = PH_E_INVARIANT_CHECK_FAILED; + PH_ERROR(GraphReferenceCountNotZero, Result); + PH_RAISE(Result); + } + + Graphs[Index] = NULL; + } + + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Graphs); + Job->Graphs = NULL; + } + + Event = (PHANDLE)&Context->FirstEvent; + NumberOfEvents = GetNumberOfContextEvents(Context); + + for (Index = 0; Index < NumberOfEvents; Index++, Event++) { + if (!ResetEvent(*Event)) { + SYS_ERROR(ResetEvent); + if (Result == S_OK) { + Result = PH_E_SYSTEM_CALL_FAILED; + } + } + } + + RELEASE(Table->OutputPath); + + Job->LastResult = Result; + + return Result; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncStep( + PPERFECT_HASH_ASYNC_WORK Work + ) +{ + HRESULT Result; + PCHM01_ASYNC_JOB Job; + + Job = CONTAINING_RECORD(Work, CHM01_ASYNC_JOB, Work); + + switch (Job->State) { + + case Chm01AsyncStateInitialize: + Result = Chm01AsyncInitializeJob(Job); + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->State = Chm01AsyncStateSolveGraphs; + return S_FALSE; + + case Chm01AsyncStateSolveGraphs: + Result = Chm01AsyncPollSolveEvents(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + if (Job->Flags & CHM01_ASYNC_JOB_FLAG_TRY_LARGER_TABLE_SIZE) { + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeWaitPrepare; + } else { + Job->State = Chm01AsyncStateFinalizeVerify; + } + return S_FALSE; + + case Chm01AsyncStateFinalizeWaitPrepare: + Result = Chm01AsyncFinalizeWaitPrepare(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + + case Chm01AsyncStateFinalizeVerify: + Result = Chm01AsyncFinalizeVerify(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->LastResult = Result; + if (NoFileIo(Job->Table)) { + Job->State = Chm01AsyncStateReleaseGraphs; + } else { + Job->State = Chm01AsyncStateFinalizeWaitSave; + } + return S_FALSE; + + case Chm01AsyncStateFinalizeWaitSave: + Result = Chm01AsyncFinalizeWaitSave(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result)) { + Job->LastResult = Result; + Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + } + Job->LastResult = Result; + Job->State = Chm01AsyncStateFinalizeClose; + return S_FALSE; + + case Chm01AsyncStateFinalizeClose: + Result = Chm01AsyncFinalizeClose(Job); + if (Result == S_FALSE) { + return S_FALSE; + } + if (FAILED(Result) && Job->LastResult == S_OK) { + Job->LastResult = Result; + } + Job->State = Chm01AsyncStateReleaseGraphs; + return S_FALSE; + + case Chm01AsyncStateReleaseGraphs: + Result = Chm01AsyncReleaseGraphs(Job); + Job->State = Chm01AsyncStateComplete; + return Result; + + case Chm01AsyncStateComplete: + return Job->LastResult; + + case Chm01AsyncStateError: + default: + Job->LastResult = PH_E_INVARIANT_CHECK_FAILED; + return Job->LastResult; + } +} + +_Use_decl_annotations_ +static +VOID +Chm01AsyncComplete( + PPERFECT_HASH_ASYNC_WORK Work, + HRESULT Result + ) +{ + PCHM01_ASYNC_JOB Job; + + Job = CONTAINING_RECORD(Work, CHM01_ASYNC_JOB, Work); + Job->LastResult = Result; + + if (Job->CompletionEvent) { + SetEvent(Job->CompletionEvent); + } +} + +_Use_decl_annotations_ +HRESULT +Chm01AsyncCreateJob( + PPERFECT_HASH_TABLE Table, + HANDLE IoCompletionPort, + PCHM01_ASYNC_JOB *JobPointer + ) +{ + HRESULT Result; + PCHM01_ASYNC_JOB Job; + PPERFECT_HASH_CONTEXT Context; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(Table) || + !ARGUMENT_PRESENT(JobPointer)) { + return E_POINTER; + } + + Context = Table->Context; + Allocator = Table->Allocator; + + if (!Allocator) { + return E_UNEXPECTED; + } + + Job = (PCHM01_ASYNC_JOB)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*Job) + ); + + if (!Job) { + return E_OUTOFMEMORY; + } + + Job->Table = Table; + Job->Context = Context; + Job->Allocator = Allocator; + Job->Rtl = Table->Rtl; + Job->State = Chm01AsyncStateInitialize; + Job->SliceBudget = 1; + Job->LastResult = S_OK; + + Job->CompletionEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!Job->CompletionEvent) { + SYS_ERROR(CreateEventW_Chm01AsyncCompletionEvent); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Job->GraphsCompleteEvent = CreateEventW(NULL, TRUE, TRUE, NULL); + if (!Job->GraphsCompleteEvent) { + SYS_ERROR(CreateEventW_Chm01AsyncGraphsCompleteEvent); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Result = PerfectHashAsyncInitialize(&Job->Async, Context, IoCompletionPort); + if (FAILED(Result)) { + goto Error; + } + + Job->Work.Step = Chm01AsyncStep; + Job->Work.Complete = Chm01AsyncComplete; + Job->Work.SliceBudget = Job->SliceBudget; + + if (!Context->FileWorkIoCompletionPort) { + Context->FileWorkIoCompletionPort = IoCompletionPort; + } + + *JobPointer = Job; + return S_OK; + +Error: + + if (Job) { + if (Job->CompletionEvent) { + CloseHandle(Job->CompletionEvent); + Job->CompletionEvent = NULL; + } + if (Job->GraphsCompleteEvent) { + CloseHandle(Job->GraphsCompleteEvent); + Job->GraphsCompleteEvent = NULL; + } + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Job); + } + + return Result; +} + +_Use_decl_annotations_ +HRESULT +Chm01AsyncSubmitJob( + PCHM01_ASYNC_JOB Job + ) +{ + if (!ARGUMENT_PRESENT(Job)) { + return E_POINTER; + } + + return PerfectHashAsyncSubmit(&Job->Async, &Job->Work); +} + +#ifdef PH_WINDOWS +static +VOID +Chm01AsyncPumpIoCompletionPort( + _In_ PCHM01_ASYNC_JOB Job + ) +{ + BOOL Success; + DWORD NumberOfBytes; + ULONG_PTR CompletionKey; + LPOVERLAPPED Overlapped; + HANDLE IoCompletionPort; + PPERFECT_HASH_IOCP_WORK WorkItem; + const ULONG AllowedFlags = ( + PH_IOCP_WORK_FLAG_ASYNC | + PH_IOCP_WORK_FLAG_FILE_WORK | + PH_IOCP_WORK_FLAG_PIPE + ); + + IoCompletionPort = Job->Async.IoCompletionPort; + if (!IoCompletionPort || !Job->CompletionEvent) { + return; + } + + for (;;) { + if (WaitForSingleObject(Job->CompletionEvent, 0) == WAIT_OBJECT_0) { + break; + } + + Success = GetQueuedCompletionStatus(IoCompletionPort, + &NumberOfBytes, + &CompletionKey, + &Overlapped, + INFINITE); + + if (CompletionKey == PERFECT_HASH_IOCP_SHUTDOWN_KEY) { + PostQueuedCompletionStatus(IoCompletionPort, + NumberOfBytes, + CompletionKey, + Overlapped); + continue; + } + + if (!Overlapped) { + continue; + } + + WorkItem = (PPERFECT_HASH_IOCP_WORK)Overlapped; + + if (WorkItem->Signature != PH_IOCP_WORK_SIGNATURE || + !WorkItem->CompletionCallback) { + continue; + } + + if ((WorkItem->Flags & AllowedFlags) == 0) { + PostQueuedCompletionStatus(IoCompletionPort, + NumberOfBytes, + CompletionKey, + Overlapped); + continue; + } + + WorkItem->CompletionCallback(NULL, + CompletionKey, + Overlapped, + NumberOfBytes, + Success); + } +} +#endif + +_Use_decl_annotations_ +VOID +Chm01AsyncWaitJob( + PCHM01_ASYNC_JOB Job + ) +{ + if (!ARGUMENT_PRESENT(Job)) { + return; + } + + if (Job->CompletionEvent) { +#ifdef PH_WINDOWS + if (Job->Async.IoCompletionPort) { + Chm01AsyncPumpIoCompletionPort(Job); + return; + } +#endif + WaitForSingleObject(Job->CompletionEvent, INFINITE); + } +} + +_Use_decl_annotations_ +VOID +Chm01AsyncDestroyJob( + PCHM01_ASYNC_JOB *JobPointer + ) +{ + PCHM01_ASYNC_JOB Job; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(JobPointer)) { + return; + } + + Job = *JobPointer; + if (!Job) { + return; + } + + Allocator = Job->Allocator; + + if (Job->GraphsCompleteEvent) { + CloseHandle(Job->GraphsCompleteEvent); + Job->GraphsCompleteEvent = NULL; + } + + if (Job->CompletionEvent) { + CloseHandle(Job->CompletionEvent); + Job->CompletionEvent = NULL; + } + + PerfectHashAsyncRundown(&Job->Async); + + if (Job->Graphs) { + Chm01AsyncReleaseGraphs(Job); + } + + if (Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)JobPointer); + } else { + *JobPointer = NULL; + } +} + +_Use_decl_annotations_ +HRESULT +CreatePerfectHashTableImplChm01Async( + PPERFECT_HASH_TABLE Table, + HANDLE IoCompletionPort + ) +{ + HRESULT Result; + PCHM01_ASYNC_JOB Job; + + Result = Chm01AsyncCreateJob(Table, IoCompletionPort, &Job); + if (FAILED(Result)) { + return Result; + } + + Result = Chm01AsyncSubmitJob(Job); + if (FAILED(Result)) { + Chm01AsyncDestroyJob(&Job); + return Result; + } + + Chm01AsyncWaitJob(Job); + Result = Job->LastResult; + + Chm01AsyncDestroyJob(&Job); + return Result; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01Async.h b/src/PerfectHash/Chm01Async.h new file mode 100644 index 00000000..54b53912 --- /dev/null +++ b/src/PerfectHash/Chm01Async.h @@ -0,0 +1,137 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + Chm01Async.h + +Abstract: + + This module defines the CHM01 asynchronous table creation state machine. + It decomposes the CHM01 algorithm into small, IOCP-driven work steps. + +--*/ + +#pragma once + +#include "PerfectHashAsync.h" + +typedef enum _CHM01_ASYNC_STATE { + Chm01AsyncStateInitialize = 0, + Chm01AsyncStateSolveGraphs, + Chm01AsyncStateFinalizeWaitPrepare, + Chm01AsyncStateFinalizeVerify, + Chm01AsyncStateFinalizeWaitSave, + Chm01AsyncStateFinalizeClose, + Chm01AsyncStateReleaseGraphs, + Chm01AsyncStateComplete, + Chm01AsyncStateError +} CHM01_ASYNC_STATE; + +typedef struct _CHM01_ASYNC_GRAPH_WORK CHM01_ASYNC_GRAPH_WORK; +typedef CHM01_ASYNC_GRAPH_WORK *PCHM01_ASYNC_GRAPH_WORK; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4820) +#endif + +typedef struct _CHM01_ASYNC_JOB { + PERFECT_HASH_ASYNC_CONTEXT Async; + PERFECT_HASH_ASYNC_WORK Work; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_CONTEXT Context; + PALLOCATOR Allocator; + PRTL Rtl; + PGRAPH *Graphs; + ULONG NumberOfGraphs; + ULONG Concurrency; + ULONG Attempt; + ULONG Padding1; + GRAPH_INFO GraphInfo; + GRAPH_INFO PrevGraphInfo; + GRAPH_INFO_ON_DISK GraphInfoOnDisk; + PTABLE_INFO_ON_DISK TableInfoOnDisk; + HANDLE Events[6]; + HANDLE SaveEvents[NUMBER_OF_SAVE_FILE_EVENTS]; + HANDLE PrepareEvents[NUMBER_OF_PREPARE_FILE_EVENTS]; + PCHM01_ASYNC_GRAPH_WORK *GraphWorkItems; + volatile LONG ActiveGraphs; + ULONG Padding2; + HANDLE GraphsCompleteEvent; + +#define EXPAND_AS_ASYNC_FILE_WORK_ITEM( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + FILE_WORK_ITEM Verb##Name; + + PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASYNC_FILE_WORK_ITEM) + SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASYNC_FILE_WORK_ITEM) + CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_ASYNC_FILE_WORK_ITEM) + +#undef EXPAND_AS_ASYNC_FILE_WORK_ITEM + + CHM01_ASYNC_STATE State; + ULONG SliceBudget; + ULONG Flags; + HRESULT LastResult; + HANDLE CompletionEvent; +} CHM01_ASYNC_JOB; +typedef CHM01_ASYNC_JOB *PCHM01_ASYNC_JOB; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI CHM01_ASYNC_CREATE_JOB)( + _In_ PPERFECT_HASH_TABLE Table, + _In_ HANDLE IoCompletionPort, + _Outptr_ PCHM01_ASYNC_JOB *Job + ); +typedef CHM01_ASYNC_CREATE_JOB *PCHM01_ASYNC_CREATE_JOB; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI CHM01_ASYNC_SUBMIT_JOB)( + _In_ PCHM01_ASYNC_JOB Job + ); +typedef CHM01_ASYNC_SUBMIT_JOB *PCHM01_ASYNC_SUBMIT_JOB; + +typedef +VOID +(NTAPI CHM01_ASYNC_WAIT_JOB)( + _In_ PCHM01_ASYNC_JOB Job + ); +typedef CHM01_ASYNC_WAIT_JOB *PCHM01_ASYNC_WAIT_JOB; + +typedef +VOID +(NTAPI CHM01_ASYNC_DESTROY_JOB)( + _Inout_ PCHM01_ASYNC_JOB *Job + ); +typedef CHM01_ASYNC_DESTROY_JOB *PCHM01_ASYNC_DESTROY_JOB; + +extern CHM01_ASYNC_CREATE_JOB Chm01AsyncCreateJob; +extern CHM01_ASYNC_SUBMIT_JOB Chm01AsyncSubmitJob; +extern CHM01_ASYNC_WAIT_JOB Chm01AsyncWaitJob; +extern CHM01_ASYNC_DESTROY_JOB Chm01AsyncDestroyJob; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI CHM01_ASYNC_CREATE_TABLE)( + _In_ PPERFECT_HASH_TABLE Table, + _In_ HANDLE IoCompletionPort + ); +typedef CHM01_ASYNC_CREATE_TABLE *PCHM01_ASYNC_CREATE_TABLE; + +extern CHM01_ASYNC_CREATE_TABLE CreatePerfectHashTableImplChm01Async; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01Compat.c b/src/PerfectHash/Chm01Compat.c index 51910227..5926a893 100644 --- a/src/PerfectHash/Chm01Compat.c +++ b/src/PerfectHash/Chm01Compat.c @@ -428,11 +428,6 @@ Return Value: TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); - ASSERT(!TlsContext->Flags.DisableGlobalAllocatorComponent); - ASSERT(!TlsContext->Flags.CustomAllocatorDetailsPresent); - ASSERT(!TlsContext->HeapCreateFlags); - ASSERT(!TlsContext->HeapMinimumSize); - TlsContextDisableGlobalAllocator(TlsContext); TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; diff --git a/src/PerfectHash/Chm01FileWork.c b/src/PerfectHash/Chm01FileWork.c index 3df4d7b9..bf06ac66 100644 --- a/src/PerfectHash/Chm01FileWork.c +++ b/src/PerfectHash/Chm01FileWork.c @@ -400,17 +400,25 @@ Return Value: break; case EofInitTypeNumberOfKeysMultiplier: - EndOfFile.QuadPart = ( - (LONGLONG)Keys->NumberOfKeys.QuadPart * - Eof->Multiplier - ); + { + LONGLONG Computed = (LONGLONG)( + Keys->NumberOfKeys.QuadPart * Eof->Multiplier + ); + if (Computed > EndOfFile.QuadPart) { + EndOfFile.QuadPart = Computed; + } + } break; case EofInitTypeNumberOfTableElementsMultiplier: - EndOfFile.QuadPart = ( - NumberOfTableElements.QuadPart * - Eof->Multiplier - ); + { + LONGLONG Computed = (LONGLONG)( + NumberOfTableElements.QuadPart * Eof->Multiplier + ); + if (Computed > EndOfFile.QuadPart) { + EndOfFile.QuadPart = Computed; + } + } break; case EofInitTypeNumberOfPages: @@ -727,7 +735,11 @@ Return Value: if (Event) { #ifdef PH_WINDOWS - SetEventWhenCallbackReturns(Item->Instance, Event); + if (Item->Instance) { + SetEventWhenCallbackReturns(Item->Instance, Event); + } else { + SetEvent(Event); + } #else SetEvent(Event); #endif diff --git a/src/PerfectHash/Chm01FileWork.h b/src/PerfectHash/Chm01FileWork.h index 799a3df5..1899f660 100644 --- a/src/PerfectHash/Chm01FileWork.h +++ b/src/PerfectHash/Chm01FileWork.h @@ -13,6 +13,8 @@ Module Name: --*/ +#pragma once + #include "stdafx.h" #define EXPAND_AS_CALLBACK_DECL( \ @@ -37,7 +39,6 @@ SAVE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CALLBACK_DECL); #define PrepareCSourceTableDataFileChm01 NULL #define PreparePythonFileChm01 NULL #define PreparePythonTestFileChm01 NULL -#define PrepareCppHeaderOnlyFileChm01 NULL #define PrepareCppHeaderOnlyTestFileChm01 NULL #define PrepareRustCargoTomlFileChm01 NULL #define PrepareRustLibFileChm01 NULL diff --git a/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c b/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c index 994d5c02..b0550161 100644 --- a/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c +++ b/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c @@ -16,6 +16,76 @@ Module Name: #include "stdafx.h" +_Use_decl_annotations_ +HRESULT +PrepareCppHeaderOnlyFileChm01( + PPERFECT_HASH_CONTEXT Context, + PFILE_WORK_ITEM Item + ) +{ + HRESULT Result = S_OK; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + PPERFECT_HASH_FILE File; + PTABLE_INFO_ON_DISK TableInfo; + ULONGLONG NumberOfKeys; + ULONGLONG NumberOfTableElements; + ULONGLONG KeyElementBytes; + ULONGLONG TableElementBytes; + ULONGLONG BaseSize; + ULONGLONG RequiredSize; + ULONGLONG AlignedSize; + LARGE_INTEGER EndOfFile; + + // + // Initialize aliases. + // + + Table = Context->Table; + Keys = Table->Keys; + File = *Item->FilePointer; + TableInfo = Table->TableInfoOnDisk; + + if (!File) { + return E_UNEXPECTED; + } + + NumberOfKeys = Keys->NumberOfKeys.QuadPart; + NumberOfTableElements = TableInfo->NumberOfTableElements.QuadPart; + + // + // Estimate required output size. Use fixed-width hex output estimates + // plus a base allowance for headers and function bodies. + // + + BaseSize = Context->SystemAllocationGranularity; + TableElementBytes = 16; + KeyElementBytes = ( + Keys->OriginalKeySizeType == LongLongType ? 24 : 16 + ); + + RequiredSize = ( + BaseSize + + (NumberOfTableElements * TableElementBytes) + + (NumberOfKeys * KeyElementBytes) + ); + + AlignedSize = ALIGN_UP(RequiredSize, + Context->SystemAllocationGranularity); + + if (AlignedSize > (ULONGLONG)File->FileInfo.EndOfFile.QuadPart) { + EndOfFile.QuadPart = (LONGLONG)AlignedSize; + AcquirePerfectHashFileLockExclusive(File); + Result = File->Vtbl->Extend(File, &EndOfFile); + ReleasePerfectHashFileLockExclusive(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileExtend, Result); + } + } + + return Result; +} + _Use_decl_annotations_ HRESULT SaveCppHeaderOnlyFileChm01( diff --git a/src/PerfectHash/Chm02.c b/src/PerfectHash/Chm02.c index 53f3ff42..56f61022 100644 --- a/src/PerfectHash/Chm02.c +++ b/src/PerfectHash/Chm02.c @@ -357,7 +357,7 @@ Return Value: ZeroStructInline(Verb##Name); \ Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_PREPARE_FILE_WORK() \ PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_FILE_WORK) @@ -588,7 +588,7 @@ Return Value: // if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, CancelPending); + PerfectHashContextWaitForFileWorkCallbacks(Context, CancelPending); } // @@ -899,7 +899,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->MainWork, TRUE); WaitForThreadpoolWorkCallbacks(Context->ConsoleWork, TRUE); if (!NoFileIo(Table)) { - WaitForThreadpoolWorkCallbacks(Context->FileWork, TRUE); + PerfectHashContextWaitForFileWorkCallbacks(Context, TRUE); } // @@ -944,14 +944,14 @@ Return Value: Verb##Name.FileWorkId = FileWork##Verb##Name##Id; \ Verb##Name.EndOfFile = EndOfFile; \ InsertTailFileWork(Context, &Verb##Name.ListEntry); \ - SubmitThreadpoolWork(Context->FileWork); + PerfectHashContextSubmitFileWork(Context); #define SUBMIT_CLOSE_FILE_WORK() \ CLOSE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_SUBMIT_CLOSE_FILE_WORK) SUBMIT_CLOSE_FILE_WORK(); - WaitForThreadpoolWorkCallbacks(Context->FileWork, FALSE); + PerfectHashContextWaitForFileWorkCallbacks(Context, FALSE); #define EXPAND_AS_CHECK_CLOSE_ERRORS( \ Verb, VUpper, Name, Upper, \ diff --git a/src/PerfectHash/Chm02Shared.c b/src/PerfectHash/Chm02Shared.c index 02bfbf1e..56723cd7 100644 --- a/src/PerfectHash/Chm02Shared.c +++ b/src/PerfectHash/Chm02Shared.c @@ -725,11 +725,6 @@ Return Value: TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); - ASSERT(!TlsContext->Flags.DisableGlobalAllocatorComponent); - ASSERT(!TlsContext->Flags.CustomAllocatorDetailsPresent); - ASSERT(!TlsContext->HeapCreateFlags); - ASSERT(!TlsContext->HeapMinimumSize); - TlsContextDisableGlobalAllocator(TlsContext); TlsContext->Flags.CustomAllocatorDetailsPresent = TRUE; TlsContext->HeapCreateFlags = HEAP_NO_SERIALIZE; diff --git a/src/PerfectHash/PerfectHash.vcxproj b/src/PerfectHash/PerfectHash.vcxproj index d2cff653..1bd09b3f 100644 --- a/src/PerfectHash/PerfectHash.vcxproj +++ b/src/PerfectHash/PerfectHash.vcxproj @@ -69,6 +69,7 @@ + @@ -148,6 +149,9 @@ + + + @@ -158,6 +162,7 @@ + @@ -228,6 +233,7 @@ + @@ -247,6 +253,9 @@ + + + @@ -254,6 +263,7 @@ + diff --git a/src/PerfectHash/PerfectHash.vcxproj.filters b/src/PerfectHash/PerfectHash.vcxproj.filters index 2be77246..d0176d28 100644 --- a/src/PerfectHash/PerfectHash.vcxproj.filters +++ b/src/PerfectHash/PerfectHash.vcxproj.filters @@ -43,6 +43,9 @@ Source Files + + Source Files + Source Files @@ -58,6 +61,15 @@ Source Files + + Source Files + + + Source Files + + + Source Files + Source Files @@ -127,6 +139,9 @@ Source Files + + Source Files + Source Files @@ -393,6 +408,15 @@ Private Header Files + + Private Header Files + + + Private Header Files + + + Private Header Files + Private Header Files @@ -417,6 +441,9 @@ Private Header Files + + Private Header Files + Private Header Files @@ -432,6 +459,9 @@ Private Header Files + + Private Header Files + Private Header Files diff --git a/src/PerfectHash/PerfectHashAsync.c b/src/PerfectHash/PerfectHashAsync.c new file mode 100644 index 00000000..5db72d42 --- /dev/null +++ b/src/PerfectHash/PerfectHashAsync.c @@ -0,0 +1,296 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashAsync.c + +Abstract: + + This module implements the asynchronous work framework used by IOCP-based + execution paths. Work items are driven by per-item completion callbacks + and can cooperatively yield by re-posting to the completion port. + +--*/ + +#include "stdafx.h" + +static +HRESULT +PerfectHashAsyncRequeueWork( + _In_ PPERFECT_HASH_ASYNC_WORK Work + ) +{ + BOOL Success; + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext; + + AsyncContext = Work->AsyncContext; + if (!AsyncContext || !AsyncContext->IoCompletionPort) { + return E_UNEXPECTED; + } + + Success = PostQueuedCompletionStatus(AsyncContext->IoCompletionPort, + 0, + 0, + &Work->Iocp.Overlapped); + if (!Success) { + SYS_ERROR(PostQueuedCompletionStatus_AsyncRequeue); + return PH_E_SYSTEM_CALL_FAILED; + } + + return S_OK; +} + +static +HRESULT +PerfectHashAsyncCompleteWork( + _In_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ) +{ + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext; + + AsyncContext = Work->AsyncContext; + Work->LastResult = Result; + + if (Work->Complete) { + Work->Complete(Work, Result); + } + + if (AsyncContext) { + if (InterlockedDecrement(&AsyncContext->Outstanding) == 0) { + if (AsyncContext->OutstandingEvent) { + SetEvent(AsyncContext->OutstandingEvent); + } + } + } + + return Result; +} + +static +HRESULT +PerfectHashAsyncIocpCompletionCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ) +{ + HRESULT Result; + PPERFECT_HASH_ASYNC_WORK Work; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(CompletionKey); + UNREFERENCED_PARAMETER(NumberOfBytesTransferred); + UNREFERENCED_PARAMETER(Success); + + Work = (PPERFECT_HASH_ASYNC_WORK)Overlapped; + if (!Work || !Work->Step) { + return E_POINTER; + } + + Result = Work->Step(Work); + + if (Result == S_FALSE) { + return PerfectHashAsyncRequeueWork(Work); + } + + return PerfectHashAsyncCompleteWork(Work, Result); +} + +_Use_decl_annotations_ +HRESULT +PerfectHashAsyncInitialize( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + PPERFECT_HASH_CONTEXT Context, + HANDLE IoCompletionPort + ) +/*++ + +Routine Description: + + Initializes an async context for IOCP work submission. + +Arguments: + + AsyncContext - Supplies a pointer to the async context to initialize. + + Context - Supplies a pointer to the owning perfect hash context. + + IoCompletionPort - Supplies the IOCP handle to post work items to. + +Return Value: + + S_OK on success, or an appropriate error code on failure. + +--*/ +{ + if (!ARGUMENT_PRESENT(AsyncContext) || + !ARGUMENT_PRESENT(Context) || + !IoCompletionPort) { + return E_POINTER; + } + + ZeroStructPointerInline(AsyncContext); + AsyncContext->Context = Context; + AsyncContext->Allocator = Context->Allocator; + AsyncContext->IoCompletionPort = IoCompletionPort; + AsyncContext->Outstanding = 0; + AsyncContext->OutstandingEvent = NULL; + + return S_OK; +} + +_Use_decl_annotations_ +VOID +PerfectHashAsyncRundown( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ) +/*++ + +Routine Description: + + Releases resources associated with an async context. + +Arguments: + + AsyncContext - Supplies a pointer to the async context to rundown. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(AsyncContext)) { + return; + } + + if (AsyncContext->OutstandingEvent) { + if (!CloseEvent(AsyncContext->OutstandingEvent)) { + SYS_ERROR(CloseHandle); + } + AsyncContext->OutstandingEvent = NULL; + } + + AsyncContext->Outstanding = 0; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashAsyncSubmit( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + PPERFECT_HASH_ASYNC_WORK Work + ) +/*++ + +Routine Description: + + Submits an async work item to the IOCP dispatch queue. If submission + fails, the work is executed inline. + +Arguments: + + AsyncContext - Supplies a pointer to the async context. + + Work - Supplies a pointer to the async work item. + +Return Value: + + S_OK if the work was queued, otherwise the result of inline execution. + +--*/ +{ + BOOL Success; + LONG Outstanding; + HRESULT Result; + + if (!ARGUMENT_PRESENT(AsyncContext) || + !ARGUMENT_PRESENT(Work) || + !Work->Step) { + return E_POINTER; + } + + if (!AsyncContext->IoCompletionPort) { + return E_UNEXPECTED; + } + + Work->AsyncContext = AsyncContext; + Work->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + Work->Iocp.Flags = PH_IOCP_WORK_FLAG_ASYNC; + Work->Iocp.CompletionCallback = PerfectHashAsyncIocpCompletionCallback; + + if (!AsyncContext->OutstandingEvent) { + AsyncContext->OutstandingEvent = CreateEventW(NULL, TRUE, TRUE, NULL); + if (!AsyncContext->OutstandingEvent) { + SYS_ERROR(CreateEventW_AsyncOutstandingEvent); + goto InlineWork; + } + } + + Outstanding = InterlockedIncrement(&AsyncContext->Outstanding); + if (Outstanding == 1 && AsyncContext->OutstandingEvent) { + ResetEvent(AsyncContext->OutstandingEvent); + } + + Success = PostQueuedCompletionStatus(AsyncContext->IoCompletionPort, + 0, + 0, + &Work->Iocp.Overlapped); + if (!Success) { + SYS_ERROR(PostQueuedCompletionStatus_AsyncSubmit); + goto InlineWork; + } + + return S_OK; + +InlineWork: + + Work->Flags.InlineDispatch = TRUE; + Result = Work->Step(Work); + + if (Result == S_FALSE) { + Result = PerfectHashAsyncRequeueWork(Work); + if (FAILED(Result)) { + return PerfectHashAsyncCompleteWork(Work, Result); + } + return S_OK; + } + + return PerfectHashAsyncCompleteWork(Work, Result); +} + +_Use_decl_annotations_ +VOID +PerfectHashAsyncWait( + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ) +/*++ + +Routine Description: + + Waits for all outstanding async work to complete. + +Arguments: + + AsyncContext - Supplies a pointer to the async context. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(AsyncContext)) { + return; + } + + if (AsyncContext->OutstandingEvent) { + WaitForSingleObject(AsyncContext->OutstandingEvent, INFINITE); + } +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashAsync.h b/src/PerfectHash/PerfectHashAsync.h new file mode 100644 index 00000000..47cc62ce --- /dev/null +++ b/src/PerfectHash/PerfectHashAsync.h @@ -0,0 +1,122 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashAsync.h + +Abstract: + + This module defines the asynchronous work framework used by the IOCP + execution model. Work items are scheduled via PostQueuedCompletionStatus + and dispatched through per-item completion callbacks, providing a minimal + continuation-style engine for finer-grained algorithm execution. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef struct _PERFECT_HASH_ASYNC_CONTEXT PERFECT_HASH_ASYNC_CONTEXT; +typedef struct _PERFECT_HASH_ASYNC_WORK PERFECT_HASH_ASYNC_WORK; +typedef PERFECT_HASH_ASYNC_CONTEXT *PPERFECT_HASH_ASYNC_CONTEXT; +typedef PERFECT_HASH_ASYNC_WORK *PPERFECT_HASH_ASYNC_WORK; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_ASYNC_STEP_ROUTINE)( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work + ); +typedef PERFECT_HASH_ASYNC_STEP_ROUTINE *PPERFECT_HASH_ASYNC_STEP_ROUTINE; + +typedef +VOID +(NTAPI PERFECT_HASH_ASYNC_COMPLETE_ROUTINE)( + _Inout_ PPERFECT_HASH_ASYNC_WORK Work, + _In_ HRESULT Result + ); +typedef PERFECT_HASH_ASYNC_COMPLETE_ROUTINE + *PPERFECT_HASH_ASYNC_COMPLETE_ROUTINE; + +typedef union _PERFECT_HASH_ASYNC_WORK_FLAGS { + struct { + ULONG InlineDispatch:1; + ULONG Unused:31; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_ASYNC_WORK_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_ASYNC_WORK_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_ASYNC_WORK_FLAGS *PPERFECT_HASH_ASYNC_WORK_FLAGS; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4820) +#endif + +typedef struct _PERFECT_HASH_ASYNC_CONTEXT { + PPERFECT_HASH_CONTEXT Context; + PALLOCATOR Allocator; + HANDLE IoCompletionPort; + HANDLE OutstandingEvent; + volatile LONG Outstanding; + ULONG Padding; +} PERFECT_HASH_ASYNC_CONTEXT; + +typedef struct _PERFECT_HASH_ASYNC_WORK { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_ASYNC_CONTEXT AsyncContext; + PPERFECT_HASH_ASYNC_STEP_ROUTINE Step; + PPERFECT_HASH_ASYNC_COMPLETE_ROUTINE Complete; + PERFECT_HASH_ASYNC_WORK_FLAGS Flags; + ULONG SliceBudget; + HRESULT LastResult; + ULONG Padding; +} PERFECT_HASH_ASYNC_WORK; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_ASYNC_INITIALIZE)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ HANDLE IoCompletionPort + ); +typedef PERFECT_HASH_ASYNC_INITIALIZE *PPERFECT_HASH_ASYNC_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_ASYNC_RUNDOWN)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ); +typedef PERFECT_HASH_ASYNC_RUNDOWN *PPERFECT_HASH_ASYNC_RUNDOWN; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_ASYNC_SUBMIT)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext, + _In_ PPERFECT_HASH_ASYNC_WORK Work + ); +typedef PERFECT_HASH_ASYNC_SUBMIT *PPERFECT_HASH_ASYNC_SUBMIT; + +typedef +VOID +(NTAPI PERFECT_HASH_ASYNC_WAIT)( + _In_ PPERFECT_HASH_ASYNC_CONTEXT AsyncContext + ); +typedef PERFECT_HASH_ASYNC_WAIT *PPERFECT_HASH_ASYNC_WAIT; + +extern PERFECT_HASH_ASYNC_INITIALIZE PerfectHashAsyncInitialize; +extern PERFECT_HASH_ASYNC_RUNDOWN PerfectHashAsyncRundown; +extern PERFECT_HASH_ASYNC_SUBMIT PerfectHashAsyncSubmit; +extern PERFECT_HASH_ASYNC_WAIT PerfectHashAsyncWait; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashClient.c b/src/PerfectHash/PerfectHashClient.c new file mode 100644 index 00000000..8c919ea7 --- /dev/null +++ b/src/PerfectHash/PerfectHashClient.c @@ -0,0 +1,514 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashClient.c + +Abstract: + + This module implements the PERFECT_HASH_CLIENT component. The initial + implementation focuses on wiring the COM interface and reserving slots + for connection and request submission logic. + +--*/ + +#include "stdafx.h" +#include + +#ifdef PH_WINDOWS +static const UNICODE_STRING PerfectHashClientDefaultPipeName = + RTL_CONSTANT_STRING(L"\\\\.\\pipe\\PerfectHashServer"); +#endif + +static +HRESULT +PerfectHashClientWriteAll( + _In_ HANDLE Handle, + _In_reads_bytes_(Length) PVOID Buffer, + _In_ ULONG Length + ) +{ +#ifdef PH_WINDOWS + ULONG Offset; + + Offset = 0; + while (Offset < Length) { + DWORD Written = 0; + BOOL Success; + ULONG Remaining = Length - Offset; + + Success = WriteFile(Handle, + (PUCHAR)Buffer + Offset, + Remaining, + &Written, + NULL); + if (!Success) { + return HRESULT_FROM_WIN32(GetLastError()); + } + + if (Written == 0) { + return HRESULT_FROM_WIN32(ERROR_BROKEN_PIPE); + } + + Offset += Written; + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(Handle); + UNREFERENCED_PARAMETER(Buffer); + UNREFERENCED_PARAMETER(Length); + return E_NOTIMPL; +#endif +} + +static +HRESULT +PerfectHashClientReadAll( + _In_ HANDLE Handle, + _Out_writes_bytes_(Length) PVOID Buffer, + _In_ ULONG Length + ) +{ +#ifdef PH_WINDOWS + ULONG Offset; + + Offset = 0; + while (Offset < Length) { + DWORD Read = 0; + BOOL Success; + ULONG Remaining = Length - Offset; + + Success = ReadFile(Handle, + (PUCHAR)Buffer + Offset, + Remaining, + &Read, + NULL); + if (!Success) { + return HRESULT_FROM_WIN32(GetLastError()); + } + + if (Read == 0) { + return HRESULT_FROM_WIN32(ERROR_BROKEN_PIPE); + } + + Offset += Read; + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(Handle); + UNREFERENCED_PARAMETER(Buffer); + UNREFERENCED_PARAMETER(Length); + return E_NOTIMPL; +#endif +} + +PERFECT_HASH_CLIENT_INITIALIZE PerfectHashClientInitialize; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientInitialize( + PPERFECT_HASH_CLIENT Client + ) +/*++ + +Routine Description: + + Initializes a PERFECT_HASH_CLIENT instance. + +Arguments: + + Client - Supplies a pointer to a PERFECT_HASH_CLIENT structure for which + initialization is to be performed. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ + HRESULT Result; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + Result = Client->Vtbl->CreateInstance(Client, + NULL, + &IID_PERFECT_HASH_RTL, + &Client->Rtl); + if (FAILED(Result)) { + return Result; + } + + Result = Client->Vtbl->CreateInstance(Client, + NULL, + &IID_PERFECT_HASH_ALLOCATOR, + &Client->Allocator); + if (FAILED(Result)) { + return Result; + } + + Client->ResponsePayload.Buffer = NULL; + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + Client->State.Initialized = TRUE; + + return S_OK; +} + +PERFECT_HASH_CLIENT_RUNDOWN PerfectHashClientRundown; + +_Use_decl_annotations_ +VOID +PerfectHashClientRundown( + PPERFECT_HASH_CLIENT Client + ) +/*++ + +Routine Description: + + Releases resources associated with a PERFECT_HASH_CLIENT instance. + +Arguments: + + Client - Supplies a pointer to a PERFECT_HASH_CLIENT structure for which + rundown is to be performed. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(Client)) { + return; + } + +#ifdef PH_WINDOWS + if (Client->ConnectionHandle) { + CloseHandle(Client->ConnectionHandle); + Client->ConnectionHandle = NULL; + } +#endif + + if (Client->ResponsePayload.Buffer && Client->Allocator) { + Client->Allocator->Vtbl->FreePointer( + Client->Allocator, + (PVOID *)&Client->ResponsePayload.Buffer + ); + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + } + + RELEASE(Client->Allocator); + RELEASE(Client->Rtl); +} + +PERFECT_HASH_CLIENT_CONNECT PerfectHashClientConnect; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientConnect( + PPERFECT_HASH_CLIENT Client, + PCUNICODE_STRING Endpoint + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Client); + UNREFERENCED_PARAMETER(Endpoint); + return E_NOTIMPL; +#else + BOOL Success; + HANDLE PipeHandle; + DWORD Mode; + DWORD LastError; + PCUNICODE_STRING TargetEndpoint; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (Client->ConnectionHandle) { + CloseHandle(Client->ConnectionHandle); + Client->ConnectionHandle = NULL; + } + + TargetEndpoint = Endpoint ? Endpoint : &PerfectHashClientDefaultPipeName; + + while (TRUE) { + PipeHandle = CreateFileW(TargetEndpoint->Buffer, + GENERIC_READ | GENERIC_WRITE, + 0, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (PipeHandle != INVALID_HANDLE_VALUE) { + break; + } + + LastError = GetLastError(); + if (LastError != ERROR_PIPE_BUSY) { + return HRESULT_FROM_WIN32(LastError); + } + + Success = WaitNamedPipeW(TargetEndpoint->Buffer, + NMPWAIT_WAIT_FOREVER); + if (!Success) { + return HRESULT_FROM_WIN32(GetLastError()); + } + } + + Mode = PIPE_READMODE_BYTE; + Success = SetNamedPipeHandleState(PipeHandle, + &Mode, + NULL, + NULL); + if (!Success) { + CloseHandle(PipeHandle); + return HRESULT_FROM_WIN32(GetLastError()); + } + + Client->ConnectionHandle = PipeHandle; + Client->Endpoint = *TargetEndpoint; + Client->State.Connected = TRUE; + Client->State.Disconnected = FALSE; + + return S_OK; +#endif +} + +PERFECT_HASH_CLIENT_DISCONNECT PerfectHashClientDisconnect; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientDisconnect( + PPERFECT_HASH_CLIENT Client + ) +{ + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + +#ifdef PH_WINDOWS + if (Client->ConnectionHandle) { + CloseHandle(Client->ConnectionHandle); + Client->ConnectionHandle = NULL; + } +#endif + + if (Client->ResponsePayload.Buffer && Client->Allocator) { + Client->Allocator->Vtbl->FreePointer( + Client->Allocator, + (PVOID *)&Client->ResponsePayload.Buffer + ); + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + } + + Client->State.Disconnected = TRUE; + + return S_OK; +} + +PERFECT_HASH_CLIENT_SUBMIT_REQUEST PerfectHashClientSubmitRequest; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientSubmitRequest( + PPERFECT_HASH_CLIENT Client, + PPERFECT_HASH_SERVER_REQUEST Request + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Client); + UNREFERENCED_PARAMETER(Request); + return E_NOTIMPL; +#else + HRESULT Result; + ULONG PayloadLength; + HANDLE Handle; + WCHAR Terminator; + PRTL Rtl; + PALLOCATOR Allocator; + PERFECT_HASH_SERVER_REQUEST_HEADER RequestHeader; + PERFECT_HASH_SERVER_RESPONSE_HEADER ResponseHeader; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Request)) { + return E_POINTER; + } + + if (!IsValidPerfectHashServerRequestType(Request->RequestType)) { + return E_INVALIDARG; + } + + Handle = Client->ConnectionHandle; + if (!Handle) { + return E_UNEXPECTED; + } + + Rtl = Client->Rtl; + if (!Rtl) { + return E_UNEXPECTED; + } + + Allocator = Client->Allocator; + if (!Allocator) { + return E_UNEXPECTED; + } + + if (Client->ResponsePayload.Buffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Client->ResponsePayload.Buffer + ); + Client->ResponsePayload.Length = 0; + Client->ResponsePayload.MaximumLength = 0; + Client->ResponsePayloadBufferSize = 0; + Client->ResponseFlags = 0; + } + + PayloadLength = 0; + if (Request->CommandLine.Length > 0) { + if (!Request->CommandLine.Buffer) { + return E_INVALIDARG; + } + if (Request->CommandLine.Length > + (PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE - sizeof(WCHAR))) { + return E_INVALIDARG; + } + PayloadLength = Request->CommandLine.Length + sizeof(WCHAR); + } + + ZeroMemory(&RequestHeader, sizeof(RequestHeader)); + RequestHeader.SizeOfStruct = sizeof(RequestHeader); + RequestHeader.Version = PERFECT_HASH_SERVER_MESSAGE_VERSION; + RequestHeader.RequestType = Request->RequestType; + RequestHeader.RequestId = Request->RequestId; + RequestHeader.PayloadLength = PayloadLength; + + Result = PerfectHashClientWriteAll(Handle, + &RequestHeader, + sizeof(RequestHeader)); + if (FAILED(Result)) { + return Result; + } + + if (PayloadLength > 0) { + Result = PerfectHashClientWriteAll(Handle, + Request->CommandLine.Buffer, + Request->CommandLine.Length); + if (FAILED(Result)) { + return Result; + } + + Terminator = L'\0'; + Result = PerfectHashClientWriteAll(Handle, + &Terminator, + sizeof(Terminator)); + if (FAILED(Result)) { + return Result; + } + } + + Result = PerfectHashClientReadAll(Handle, + &ResponseHeader, + sizeof(ResponseHeader)); + if (FAILED(Result)) { + return Result; + } + + if (ResponseHeader.SizeOfStruct != sizeof(ResponseHeader) || + ResponseHeader.Version != PERFECT_HASH_SERVER_MESSAGE_VERSION) { + return E_UNEXPECTED; + } + + Client->ResponseFlags = ResponseHeader.Flags; + + if (ResponseHeader.PayloadLength > 0) { + PVOID PayloadBuffer; + + PayloadLength = ResponseHeader.PayloadLength; + + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + return E_INVALIDARG; + } + + if (PayloadLength > (ULONG)USHRT_MAX) { + return E_INVALIDARG; + } + + PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + PayloadLength); + if (!PayloadBuffer) { + return E_OUTOFMEMORY; + } + + Result = PerfectHashClientReadAll(Handle, + PayloadBuffer, + PayloadLength); + if (FAILED(Result)) { + Allocator->Vtbl->FreePointer(Allocator, &PayloadBuffer); + return Result; + } + + Client->ResponsePayload.Buffer = (PWSTR)PayloadBuffer; + Client->ResponsePayload.Length = (USHORT)( + PayloadLength >= sizeof(WCHAR) ? + PayloadLength - sizeof(WCHAR) : + 0 + ); + Client->ResponsePayload.MaximumLength = (USHORT)PayloadLength; + Client->ResponsePayloadBufferSize = PayloadLength; + } + + return ResponseHeader.Result; +#endif +} + +PERFECT_HASH_CLIENT_GET_LAST_RESPONSE PerfectHashClientGetLastResponse; + +_Use_decl_annotations_ +HRESULT +PerfectHashClientGetLastResponse( + PPERFECT_HASH_CLIENT Client, + PUNICODE_STRING ResponsePayload, + PULONG ResponseFlags + ) +{ + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ResponsePayload)) { + return E_POINTER; + } + + ResponsePayload->Buffer = Client->ResponsePayload.Buffer; + ResponsePayload->Length = Client->ResponsePayload.Length; + ResponsePayload->MaximumLength = Client->ResponsePayload.MaximumLength; + + if (ARGUMENT_PRESENT(ResponseFlags)) { + *ResponseFlags = Client->ResponseFlags; + } + + return S_OK; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashClient.h b/src/PerfectHash/PerfectHashClient.h new file mode 100644 index 00000000..958a7067 --- /dev/null +++ b/src/PerfectHash/PerfectHashClient.h @@ -0,0 +1,89 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashClient.h + +Abstract: + + This is the private header file for the PERFECT_HASH_CLIENT component of + the perfect hash library. It defines the structure, and function pointer + typedefs for private non-vtbl members. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef union _PERFECT_HASH_CLIENT_STATE { + struct { + + ULONG Initialized:1; + ULONG Connected:1; + ULONG Disconnected:1; + + ULONG Unused:29; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CLIENT_STATE; +C_ASSERT(sizeof(PERFECT_HASH_CLIENT_STATE) == sizeof(ULONG)); +typedef PERFECT_HASH_CLIENT_STATE *PPERFECT_HASH_CLIENT_STATE; + +typedef union _PERFECT_HASH_CLIENT_FLAGS { + struct { + + ULONG Unused:32; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CLIENT_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_CLIENT_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_CLIENT_FLAGS *PPERFECT_HASH_CLIENT_FLAGS; + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CLIENT { + + COMMON_COMPONENT_HEADER(PERFECT_HASH_CLIENT); + + HANDLE ConnectionHandle; + UNICODE_STRING Endpoint; + UNICODE_STRING ResponsePayload; + ULONG ResponsePayloadBufferSize; + ULONG ResponseFlags; + + // + // Backing vtbl. + // + + PERFECT_HASH_CLIENT_VTBL Interface; + +} PERFECT_HASH_CLIENT; +typedef PERFECT_HASH_CLIENT *PPERFECT_HASH_CLIENT; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CLIENT_INITIALIZE)( + _In_ PPERFECT_HASH_CLIENT Client + ); +typedef PERFECT_HASH_CLIENT_INITIALIZE *PPERFECT_HASH_CLIENT_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_CLIENT_RUNDOWN)( + _In_ _Post_ptr_invalid_ PPERFECT_HASH_CLIENT Client + ); +typedef PERFECT_HASH_CLIENT_RUNDOWN *PPERFECT_HASH_CLIENT_RUNDOWN; + +extern PERFECT_HASH_CLIENT_INITIALIZE PerfectHashClientInitialize; +extern PERFECT_HASH_CLIENT_RUNDOWN PerfectHashClientRundown; + +extern PERFECT_HASH_CLIENT_CONNECT PerfectHashClientConnect; +extern PERFECT_HASH_CLIENT_DISCONNECT PerfectHashClientDisconnect; +extern PERFECT_HASH_CLIENT_SUBMIT_REQUEST PerfectHashClientSubmitRequest; +extern PERFECT_HASH_CLIENT_GET_LAST_RESPONSE PerfectHashClientGetLastResponse; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashConstants.c b/src/PerfectHash/PerfectHashConstants.c index 97f2169a..d2c9178e 100644 --- a/src/PerfectHash/PerfectHashConstants.c +++ b/src/PerfectHash/PerfectHashConstants.c @@ -900,7 +900,7 @@ const ULONG IndexMaskPlaceholder = 0xbbbbbbbb; // -#define NUMBER_OF_INTERFACES 15 +#define NUMBER_OF_INTERFACES 18 #define EXPECTED_ARRAY_SIZE NUMBER_OF_INTERFACES+2 #define VERIFY_ARRAY_SIZE(Name) C_ASSERT(ARRAYSIZE(Name) == EXPECTED_ARRAY_SIZE) @@ -988,6 +988,9 @@ const SHORT ComponentInterfaceTlsContextOffsets[] = { -1, // IClassFactory (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Keys), (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Context), + -1, // ContextIocp + -1, // Server + -1, // Client (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Table), (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Rtl), (SHORT)FIELD_OFFSET(PERFECT_HASH_TLS_CONTEXT, Allocator), @@ -1011,6 +1014,9 @@ const SHORT GlobalComponentsInterfaceOffsets[] = { -1, // IClassFactory -1, // Keys -1, // Context + -1, // ContextIocp + -1, // Server + -1, // Client -1, // Table (SHORT)FIELD_OFFSET(GLOBAL_COMPONENTS, Rtl), (SHORT)FIELD_OFFSET(GLOBAL_COMPONENTS, Allocator), @@ -1118,6 +1124,80 @@ const PERFECT_HASH_CONTEXT_VTBL PerfectHashContextInterface = { }; VERIFY_VTBL_SIZE(PERFECT_HASH_CONTEXT, 12); +// +// PerfectHashContextIocp +// + +const PERFECT_HASH_CONTEXT_IOCP_VTBL PerfectHashContextIocpInterface = { + (PPERFECT_HASH_CONTEXT_IOCP_QUERY_INTERFACE)&ComponentQueryInterface, + (PPERFECT_HASH_CONTEXT_IOCP_ADD_REF)&ComponentAddRef, + (PPERFECT_HASH_CONTEXT_IOCP_RELEASE)&ComponentRelease, + (PPERFECT_HASH_CONTEXT_IOCP_CREATE_INSTANCE)&ComponentCreateInstance, + (PPERFECT_HASH_CONTEXT_IOCP_LOCK_SERVER)&ComponentLockServer, + &PerfectHashContextIocpSetMaximumConcurrency, + &PerfectHashContextIocpGetMaximumConcurrency, + &PerfectHashContextIocpSetNumaNodeMask, + &PerfectHashContextIocpGetNumaNodeMask, + &PerfectHashContextIocpSetBaseOutputDirectory, + &PerfectHashContextIocpGetBaseOutputDirectory, + &PerfectHashContextIocpBulkCreate, + &PerfectHashContextIocpBulkCreateArgvW, + &PerfectHashContextIocpExtractBulkCreateArgsFromArgvW, + &PerfectHashContextIocpTableCreate, + &PerfectHashContextIocpTableCreateArgvW, + &PerfectHashContextIocpExtractTableCreateArgsFromArgvW, +#ifdef PH_COMPAT + &PerfectHashContextIocpTableCreateArgvA, + &PerfectHashContextIocpBulkCreateArgvA, +#else + NULL, + NULL, +#endif +}; +VERIFY_VTBL_SIZE(PERFECT_HASH_CONTEXT_IOCP, 14); + +// +// PerfectHashServer +// + +const PERFECT_HASH_SERVER_VTBL PerfectHashServerInterface = { + (PPERFECT_HASH_SERVER_QUERY_INTERFACE)&ComponentQueryInterface, + (PPERFECT_HASH_SERVER_ADD_REF)&ComponentAddRef, + (PPERFECT_HASH_SERVER_RELEASE)&ComponentRelease, + (PPERFECT_HASH_SERVER_CREATE_INSTANCE)&ComponentCreateInstance, + (PPERFECT_HASH_SERVER_LOCK_SERVER)&ComponentLockServer, + &PerfectHashServerSetMaximumConcurrency, + &PerfectHashServerGetMaximumConcurrency, + &PerfectHashServerSetNumaNodeMask, + &PerfectHashServerGetNumaNodeMask, + &PerfectHashServerSetEndpoint, + &PerfectHashServerGetEndpoint, + &PerfectHashServerSetLocalOnly, + &PerfectHashServerGetLocalOnly, + &PerfectHashServerStart, + &PerfectHashServerStop, + &PerfectHashServerWait, + &PerfectHashServerSubmitRequest, +}; +VERIFY_VTBL_SIZE(PERFECT_HASH_SERVER, 12); + +// +// PerfectHashClient +// + +const PERFECT_HASH_CLIENT_VTBL PerfectHashClientInterface = { + (PPERFECT_HASH_CLIENT_QUERY_INTERFACE)&ComponentQueryInterface, + (PPERFECT_HASH_CLIENT_ADD_REF)&ComponentAddRef, + (PPERFECT_HASH_CLIENT_RELEASE)&ComponentRelease, + (PPERFECT_HASH_CLIENT_CREATE_INSTANCE)&ComponentCreateInstance, + (PPERFECT_HASH_CLIENT_LOCK_SERVER)&ComponentLockServer, + &PerfectHashClientConnect, + &PerfectHashClientDisconnect, + &PerfectHashClientSubmitRequest, + &PerfectHashClientGetLastResponse, +}; +VERIFY_VTBL_SIZE(PERFECT_HASH_CLIENT, 4); + // // PerfectHashTable // @@ -1472,6 +1552,9 @@ const VOID *ComponentInterfaces[] = { &IClassFactoryInterface, &PerfectHashKeysInterface, &PerfectHashContextInterface, + &PerfectHashContextIocpInterface, + &PerfectHashServerInterface, + &PerfectHashClientInterface, &PerfectHashTableInterface, &RtlInterface, &AllocatorInterface, @@ -1500,6 +1583,9 @@ const PCOMPONENT_INITIALIZE ComponentInitializeRoutines[] = { (PCOMPONENT_INITIALIZE)&PerfectHashKeysInitialize, (PCOMPONENT_INITIALIZE)&PerfectHashContextInitialize, + (PCOMPONENT_INITIALIZE)&PerfectHashContextIocpInitialize, + (PCOMPONENT_INITIALIZE)&PerfectHashServerInitialize, + (PCOMPONENT_INITIALIZE)&PerfectHashClientInitialize, (PCOMPONENT_INITIALIZE)&PerfectHashTableInitialize, (PCOMPONENT_INITIALIZE)&RtlInitialize, (PCOMPONENT_INITIALIZE)&AllocatorInitialize, @@ -1529,6 +1615,9 @@ const PCOMPONENT_RUNDOWN ComponentRundownRoutines[] = { (PCOMPONENT_RUNDOWN)&PerfectHashKeysRundown, (PCOMPONENT_RUNDOWN)&PerfectHashContextRundown, + (PCOMPONENT_RUNDOWN)&PerfectHashContextIocpRundown, + (PCOMPONENT_RUNDOWN)&PerfectHashServerRundown, + (PCOMPONENT_RUNDOWN)&PerfectHashClientRundown, (PCOMPONENT_RUNDOWN)&PerfectHashTableRundown, (PCOMPONENT_RUNDOWN)&RtlRundown, (PCOMPONENT_RUNDOWN)&AllocatorRundown, diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index 165fe470..c748b9b0 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -492,9 +492,13 @@ Return Value: } if (!SetThreadpoolThreadMinimum(Threadpool, ThreadpoolConcurrency)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Threadpool, ThreadpoolConcurrency); @@ -593,9 +597,13 @@ Return Value: SetThreadpoolThreadMaximum(Threadpool, MaximumConcurrency); if (!SetThreadpoolThreadMinimum(Threadpool, MaximumConcurrency)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Threadpool, MaximumConcurrency); @@ -656,9 +664,13 @@ Return Value: Context->FinishedThreadpool); if (!SetThreadpoolThreadMinimum(Context->FinishedThreadpool, 1)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Context->FinishedThreadpool, 1); @@ -689,9 +701,13 @@ Return Value: if (!SetThreadpoolThreadMinimum(Context->ErrorThreadpool, 1)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SYSTEM_CALL_FAILED; - goto Error; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } } SetThreadpoolThreadMaximum(Context->ErrorThreadpool, 1); @@ -951,6 +967,13 @@ Return Value: #ifdef PH_WINDOWS + if (Context->FileWorkOutstandingEvent) { + if (!CloseEvent(Context->FileWorkOutstandingEvent)) { + SYS_ERROR(CloseHandle); + } + Context->FileWorkOutstandingEvent = NULL; + } + if (Context->MainCleanupGroup) { CloseThreadpoolCleanupGroupMembers(Context->MainCleanupGroup, @@ -1150,6 +1173,13 @@ Return Value: Context->FileWorkList->Vtbl->Reset(Context->FileWorkList); Context->FinishedWorkList->Vtbl->Reset(Context->FinishedWorkList); +#ifdef PH_WINDOWS + Context->FileWorkOutstanding = 0; + if (Context->FileWorkOutstandingEvent) { + SetEvent(Context->FileWorkOutstandingEvent); + } +#endif + Context->KeysSubset = NULL; Context->UserSeeds = NULL; Context->SeedMasks = NULL; @@ -1421,6 +1451,252 @@ Return Value: return; } +/*++ + +Structure Description: + + This structure wraps an IOCP work header and the target context pointer + for IOCP-dispatched file work callbacks. + +--*/ +typedef struct _PERFECT_HASH_FILE_WORK_IOCP { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_CONTEXT Context; +} PERFECT_HASH_FILE_WORK_IOCP; +typedef PERFECT_HASH_FILE_WORK_IOCP *PPERFECT_HASH_FILE_WORK_IOCP; + +static +HRESULT +PerfectHashContextFileWorkIocpCompletionCallback( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG_PTR CompletionKey, + LPOVERLAPPED Overlapped, + DWORD NumberOfBytesTransferred, + BOOL Success + ) +/*++ + +Routine Description: + + Completion callback for IOCP-dispatched file work. This routine removes + one file work item from the context list, invokes the registered file work + callback, decrements the outstanding count, and signals the outstanding + event when all work items have completed. + +Arguments: + + ContextIocp - Supplies the owning IOCP context. + + CompletionKey - Supplies the IOCP completion key. + + Overlapped - Supplies a pointer to the IOCP work header for this request. + + NumberOfBytesTransferred - Supplies the number of bytes transferred. + + Success - Supplies the success status of the IOCP completion. + +Return Value: + + S_OK on success, or E_POINTER for invalid arguments. + +--*/ +{ + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_FILE_WORK_IOCP WorkItem; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(CompletionKey); + UNREFERENCED_PARAMETER(NumberOfBytesTransferred); + UNREFERENCED_PARAMETER(Success); + + WorkItem = (PPERFECT_HASH_FILE_WORK_IOCP)Overlapped; + if (!WorkItem) { + return E_POINTER; + } + + Context = WorkItem->Context; + if (!Context) { + return E_POINTER; + } + + FileWorkCallback(NULL, Context, NULL); + + if (InterlockedDecrement(&Context->FileWorkOutstanding) == 0) { + if (Context->FileWorkOutstandingEvent) { + SetEvent(Context->FileWorkOutstandingEvent); + } + } + + if (Context->Allocator) { + Context->Allocator->Vtbl->FreePointer(Context->Allocator, + (PVOID *)&WorkItem); + } + + return S_OK; +} + +PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK PerfectHashContextSubmitFileWork; + +_Use_decl_annotations_ +VOID +PerfectHashContextSubmitFileWork( + PPERFECT_HASH_CONTEXT Context + ) +/*++ + +Routine Description: + + Submits a file work item. If an IOCP port has been configured, this + routine posts an IOCP completion and tracks outstanding file work so that + callers can wait on a single event. Otherwise, the file work is submitted + to the file threadpool. + +Arguments: + + Context - Supplies a pointer to the PERFECT_HASH_CONTEXT. + +Return Value: + + None. + +--*/ +{ + BOOL Success; + BOOLEAN Incremented = FALSE; + LONG Outstanding; + PALLOCATOR Allocator; + PPERFECT_HASH_FILE_WORK_IOCP WorkItem = NULL; + + if (!ARGUMENT_PRESENT(Context)) { + return; + } + +#ifdef PH_WINDOWS + if (Context->FileWorkIoCompletionPort) { + + if (!Context->FileWorkOutstandingEvent) { + Context->FileWorkOutstandingEvent = + CreateEventW(NULL, TRUE, TRUE, NULL); + if (!Context->FileWorkOutstandingEvent) { + SYS_ERROR(CreateEventW_FileWorkOutstandingEvent); + goto InlineWork; + } + } + + Allocator = Context->Allocator; + if (!Allocator) { + goto InlineWork; + } + + WorkItem = (PPERFECT_HASH_FILE_WORK_IOCP)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*WorkItem) + ); + if (!WorkItem) { + goto InlineWork; + } + + WorkItem->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + WorkItem->Iocp.Flags = PH_IOCP_WORK_FLAG_FILE_WORK; + WorkItem->Iocp.CompletionCallback = + PerfectHashContextFileWorkIocpCompletionCallback; + WorkItem->Context = Context; + + Outstanding = InterlockedIncrement(&Context->FileWorkOutstanding); + Incremented = TRUE; + if (Outstanding == 1 && Context->FileWorkOutstandingEvent) { + ResetEvent(Context->FileWorkOutstandingEvent); + } + + Success = PostQueuedCompletionStatus( + Context->FileWorkIoCompletionPort, + 0, + 0, + &WorkItem->Iocp.Overlapped + ); + + if (!Success) { + SYS_ERROR(PostQueuedCompletionStatus_FileWork); + goto InlineWork; + } + + return; + } +#endif + + SubmitThreadpoolWork(Context->FileWork); + return; + +InlineWork: + + // + // Fall back to synchronous execution if IOCP submission fails. + // + + FileWorkCallback(NULL, Context, NULL); + + if (Context->FileWorkIoCompletionPort && Incremented) { + if (InterlockedDecrement(&Context->FileWorkOutstanding) == 0) { + if (Context->FileWorkOutstandingEvent) { + SetEvent(Context->FileWorkOutstandingEvent); + } + } + } + + Allocator = Context->Allocator; + if (Allocator && WorkItem) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + } + + return; +} + +PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS + PerfectHashContextWaitForFileWorkCallbacks; + +_Use_decl_annotations_ +VOID +PerfectHashContextWaitForFileWorkCallbacks( + PPERFECT_HASH_CONTEXT Context, + BOOL CancelPending + ) +/*++ + +Routine Description: + + Waits for all file work callbacks to complete. When IOCP dispatch is in + effect, this waits on the outstanding file work event; otherwise, it + waits on the file threadpool work callbacks. + +Arguments: + + Context - Supplies a pointer to the PERFECT_HASH_CONTEXT. + + CancelPending - Supplies the cancellation behavior for threadpool work. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(Context)) { + return; + } + +#ifdef PH_WINDOWS + if (Context->FileWorkIoCompletionPort) { + if (Context->FileWorkOutstandingEvent) { + WaitForSingleObject(Context->FileWorkOutstandingEvent, INFINITE); + } + return; + } +#endif + + WaitForThreadpoolWorkCallbacks(Context->FileWork, CancelPending); +} + _Use_decl_annotations_ VOID FinishedWorkCallback( @@ -1688,6 +1964,7 @@ Return Value: PRTL Rtl; PTP_POOL Threadpool; HRESULT Result = S_OK; + ULONG LastError; ULONG ThreadpoolConcurrency; if (!ARGUMENT_PRESENT(Context)) { @@ -1714,8 +1991,12 @@ Return Value: SetThreadpoolThreadMaximum(Threadpool, ThreadpoolConcurrency); if (!SetThreadpoolThreadMinimum(Threadpool, ThreadpoolConcurrency)) { - SYS_ERROR(SetThreadpoolThreadMinimum); - Result = PH_E_SET_MAXIMUM_CONCURRENCY_FAILED; + LastError = GetLastError(); + if (LastError != ERROR_ACCESS_DENIED) { + SetLastError(LastError); + SYS_ERROR(SetThreadpoolThreadMinimum); + Result = PH_E_SET_MAXIMUM_CONCURRENCY_FAILED; + } } #endif diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index e264525f..b473e3b8 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -126,6 +126,12 @@ typedef union _PERFECT_HASH_CONTEXT_STATE { ULONG IsBulkCreate:1; + // + // When set, indicates context-level file work should be skipped. + // + + ULONG SkipContextFileWork:1; + // // When set, indicates the graph solving failed because every worker // thread failed to allocate sufficient memory to even attempt solving @@ -200,7 +206,7 @@ typedef union _PERFECT_HASH_CONTEXT_STATE { // Unused bits. // - ULONG Unused:16; + ULONG Unused:15; }; LONG AsLong; ULONG AsULong; @@ -245,6 +251,13 @@ typedef PERFECT_HASH_CONTEXT_STATE *PPERFECT_HASH_CONTEXT_STATE; #define SetContextBulkCreate(Context) ((Context)->State.IsBulkCreate = TRUE) #define ClearContextBulkCreate(Context) ((Context)->State.IsBulkCreate = FALSE) +#define SkipContextFileWork(Context) \ + ((Context)->State.SkipContextFileWork == TRUE) +#define SetContextSkipContextFileWork(Context) \ + ((Context)->State.SkipContextFileWork = TRUE) +#define ClearContextSkipContextFileWork(Context) \ + ((Context)->State.SkipContextFileWork = FALSE) + #define IsContextTableCreate(Context) ((Context)->State.IsTableCreate == TRUE) #define SetContextTableCreate(Context) ((Context)->State.IsTableCreate = TRUE) #define ClearContextTableCreate(Context) \ @@ -1071,6 +1084,15 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { #endif PTP_WORK FileWork; + // + // Optional IOCP integration for file work dispatch. + // + + HANDLE FileWorkIoCompletionPort; + HANDLE FileWorkOutstandingEvent; + volatile LONG FileWorkOutstanding; + ULONG Padding7; + volatile LONG GraphRegisterSolvedTsxSuccess; ULONG Padding4; @@ -1393,6 +1415,23 @@ HRESULT ); typedef PERFECT_HASH_CONTEXT_RESET *PPERFECT_HASH_CONTEXT_RESET; +typedef +VOID +(NTAPI PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK)( + _In_ PPERFECT_HASH_CONTEXT Context + ); +typedef PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK + *PPERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK; + +typedef +VOID +(NTAPI PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS)( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ BOOL CancelPending + ); +typedef PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS + *PPERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS; + typedef VOID (NTAPI PERFECT_HASH_CONTEXT_APPLY_THREADPOOL_PRIORITIES)( @@ -1477,6 +1516,9 @@ PerfectHashContextInitializeLowMemoryMonitor( extern PERFECT_HASH_CONTEXT_INITIALIZE PerfectHashContextInitialize; extern PERFECT_HASH_CONTEXT_RUNDOWN PerfectHashContextRundown; extern PERFECT_HASH_CONTEXT_RESET PerfectHashContextReset; +extern PERFECT_HASH_CONTEXT_SUBMIT_FILE_WORK PerfectHashContextSubmitFileWork; +extern PERFECT_HASH_CONTEXT_WAIT_FOR_FILE_WORK_CALLBACKS + PerfectHashContextWaitForFileWorkCallbacks; extern PERFECT_HASH_CONTEXT_INITIALIZE_KEY_SIZE PerfectHashContextInitializeKeySize; extern PERFECT_HASH_CONTEXT_SET_MAXIMUM_CONCURRENCY diff --git a/src/PerfectHash/PerfectHashContextIocp.c b/src/PerfectHash/PerfectHashContextIocp.c new file mode 100644 index 00000000..564ba1e7 --- /dev/null +++ b/src/PerfectHash/PerfectHashContextIocp.c @@ -0,0 +1,1250 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashContextIocp.c + +Abstract: + + This module implements the IOCP-backed perfect hash context component. + The implementation is currently scaffolded to allow incremental bring-up + of the IOCP runtime, NUMA-aware worker pools, and bulk/table create + workflows. + +--*/ + +#include "stdafx.h" + +// +// Forward decls for internal helpers. +// + +static +HRESULT +PerfectHashContextIocpEnumerateNumaNodes( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); + +static +HRESULT +PerfectHashContextIocpCreateIoCompletionPorts( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); + +static +HRESULT +PerfectHashContextIocpCreateWorkerThreads( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); + +static +VOID +PerfectHashIocpApplyNodeAffinity( + _In_ PPERFECT_HASH_IOCP_NODE Node + ); + +static +#ifdef PH_WINDOWS +DWORD +WINAPI +PerfectHashIocpWorkerThreadProc( + _In_ PVOID Parameter + ); +#else +VOID +PerfectHashIocpWorkerThreadProc( + _In_ PVOID Parameter + ); +#endif + +PERFECT_HASH_CONTEXT_IOCP_INITIALIZE PerfectHashContextIocpInitialize; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpInitialize( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Initializes an IOCP-backed perfect hash table context. This routine + creates the Rtl and Allocator components and seeds default configuration + values. IOCP resources are brought online in later steps. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + for which initialization is to be performed. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ + HRESULT Result; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_RTL, + &ContextIocp->Rtl); + + if (FAILED(Result)) { + return Result; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_ALLOCATOR, + &ContextIocp->Allocator); + + if (FAILED(Result)) { + return Result; + } + + Rtl = ContextIocp->Rtl; + + ContextIocp->MaximumConcurrency = Rtl->CpuFeatures.LogicalProcessorCount; + if (ContextIocp->MaximumConcurrency == 0) { + ContextIocp->MaximumConcurrency = 1; + } + + ContextIocp->NumaNodeCount = Rtl->CpuFeatures.NumaNodeCount; + if (ContextIocp->NumaNodeCount == 0) { + ContextIocp->NumaNodeCount = 1; + } + + ContextIocp->NumaNodeMask = PERFECT_HASH_NUMA_NODE_MASK_ALL; + ContextIocp->Flags.UseNumaNodeMask = FALSE; + ContextIocp->State.Initialized = TRUE; + ContextIocp->CompletionCallback = NULL; + ContextIocp->CompletionContext = NULL; + + ContextIocp->StartedEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!ContextIocp->StartedEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + ContextIocp->ShutdownEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!ContextIocp->ShutdownEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_RUNDOWN PerfectHashContextIocpRundown; + +_Use_decl_annotations_ +VOID +PerfectHashContextIocpRundown( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Rundowns an IOCP-backed perfect hash table context. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + to rundown. + +Return Value: + + None. + +--*/ +{ + ULONG Index; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return; + } + + if (ContextIocp->State.Running) { + PerfectHashContextIocpStop(ContextIocp); + } + + if (ContextIocp->Nodes) { + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; + ULONG ThreadIndex; + + if (Node->WorkerThreads) { + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + if (Node->WorkerThreads[ThreadIndex]) { + CloseHandle(Node->WorkerThreads[ThreadIndex]); + } + } + if (ContextIocp->Allocator) { + ContextIocp->Allocator->Vtbl->FreePointer( + ContextIocp->Allocator, + (PVOID *)&Node->WorkerThreads + ); + } + } + + if (Node->IoCompletionPort) { + CloseHandle(Node->IoCompletionPort); + } + } + if (ContextIocp->Allocator) { + ContextIocp->Allocator->Vtbl->FreePointer( + ContextIocp->Allocator, + (PVOID *)&ContextIocp->Nodes + ); + } + } + + if (ContextIocp->ShutdownEvent) { + CloseHandle(ContextIocp->ShutdownEvent); + ContextIocp->ShutdownEvent = NULL; + } + + if (ContextIocp->StartedEvent) { + CloseHandle(ContextIocp->StartedEvent); + ContextIocp->StartedEvent = NULL; + } + + RELEASE(ContextIocp->BaseOutputDirectory); + RELEASE(ContextIocp->Allocator); + RELEASE(ContextIocp->Rtl); +} + +PERFECT_HASH_CONTEXT_IOCP_START PerfectHashContextIocpStart; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpStart( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Starts the IOCP runtime for a PERFECT_HASH_CONTEXT_IOCP instance. This + enumerates NUMA nodes, creates one IOCP per node, and spawns worker threads + with appropriate affinity. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + for which the runtime is to be started. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#else + HRESULT Result; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + if (ContextIocp->State.Running) { + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + return S_FALSE; + } + + Result = PerfectHashContextIocpEnumerateNumaNodes(ContextIocp); + if (FAILED(Result)) { + goto Error; + } + + Result = PerfectHashContextIocpCreateIoCompletionPorts(ContextIocp); + if (FAILED(Result)) { + goto Error; + } + + Result = PerfectHashContextIocpCreateWorkerThreads(ContextIocp); + if (FAILED(Result)) { + goto Error; + } + + ContextIocp->State.Running = TRUE; + if (ContextIocp->StartedEvent) { + SetEvent(ContextIocp->StartedEvent); + } + + Result = S_OK; + goto End; + +Error: + + PerfectHashContextIocpStop(ContextIocp); + +End: + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return Result; +#endif +} + +PERFECT_HASH_CONTEXT_IOCP_STOP PerfectHashContextIocpStop; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpStop( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +/*++ + +Routine Description: + + Stops the IOCP runtime for a PERFECT_HASH_CONTEXT_IOCP instance by posting + shutdown completions to each worker thread and releasing resources. + +Arguments: + + ContextIocp - Supplies a pointer to a PERFECT_HASH_CONTEXT_IOCP structure + for which the runtime is to be stopped. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#else + ULONG NodeIndex; + ULONG ThreadIndex; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ContextIocp->Nodes) { + return S_FALSE; + } + + ContextIocp->State.Stopping = TRUE; + if (ContextIocp->ShutdownEvent) { + SetEvent(ContextIocp->ShutdownEvent); + } + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + PostQueuedCompletionStatus(Node->IoCompletionPort, + 0, + PERFECT_HASH_IOCP_SHUTDOWN_KEY, + NULL); + } + } + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + if (Node->WorkerThreads[ThreadIndex]) { + WaitForSingleObject(Node->WorkerThreads[ThreadIndex], + INFINITE); + } + } + } + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + + if (Node->WorkerThreads) { + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + if (Node->WorkerThreads[ThreadIndex]) { + CloseHandle(Node->WorkerThreads[ThreadIndex]); + } + } + if (ContextIocp->Allocator) { + ContextIocp->Allocator->Vtbl->FreePointer( + ContextIocp->Allocator, + (PVOID *)&Node->WorkerThreads + ); + } + } + + if (Node->IoCompletionPort) { + CloseHandle(Node->IoCompletionPort); + Node->IoCompletionPort = NULL; + } + } + + if (ContextIocp->Allocator) { + ContextIocp->Allocator->Vtbl->FreePointer( + ContextIocp->Allocator, + (PVOID *)&ContextIocp->Nodes + ); + } + + ContextIocp->Nodes = NULL; + ContextIocp->NodeCount = 0; + ContextIocp->IoCompletionPortCount = 0; + ContextIocp->TotalWorkerThreadCount = 0; + + ContextIocp->State.Stopped = TRUE; + ContextIocp->State.Running = FALSE; + + return S_OK; +#endif +} + +static +VOID +PerfectHashIocpApplyNodeAffinity( + PPERFECT_HASH_IOCP_NODE Node + ) +{ +#ifdef PH_WINDOWS + GROUP_AFFINITY Previous; + + if (!SetThreadGroupAffinity(GetCurrentThread(), + &Node->GroupAffinity, + &Previous)) { + if (Node->GroupAffinity.Group == 0) { + SetThreadAffinityMask(GetCurrentThread(), + Node->GroupAffinity.Mask); + } + } +#else + UNREFERENCED_PARAMETER(Node); +#endif +} + +#ifdef PH_WINDOWS +static +DWORD +WINAPI +PerfectHashIocpWorkerThreadProc( + PVOID Parameter + ) +{ + BOOL Success; + ULONG_PTR Key; + DWORD NumberOfBytes; + LPOVERLAPPED Overlapped; + PPERFECT_HASH_IOCP_NODE Node = (PPERFECT_HASH_IOCP_NODE)Parameter; + + PerfectHashIocpApplyNodeAffinity(Node); + + while (TRUE) { + Success = GetQueuedCompletionStatus(Node->IoCompletionPort, + &NumberOfBytes, + &Key, + &Overlapped, + INFINITE); + + if (Key == PERFECT_HASH_IOCP_SHUTDOWN_KEY) { + break; + } + + if (Overlapped) { + PPERFECT_HASH_IOCP_WORK WorkItem = + (PPERFECT_HASH_IOCP_WORK)Overlapped; + + if (WorkItem->Signature == PH_IOCP_WORK_SIGNATURE && + WorkItem->CompletionCallback) { + WorkItem->CompletionCallback(Node->ContextIocp, + Key, + Overlapped, + NumberOfBytes, + Success); + continue; + } + } + + if (Node->ContextIocp->CompletionCallback && Overlapped) { + Node->ContextIocp->CompletionCallback(Node->ContextIocp, + Key, + Overlapped, + NumberOfBytes, + Success); + } + } + + return 0; +} +#else +static +VOID +PerfectHashIocpWorkerThreadProc( + PVOID Parameter + ) +{ + UNREFERENCED_PARAMETER(Parameter); +} +#endif + +static +HRESULT +PerfectHashContextIocpEnumerateNumaNodes( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +{ +#ifdef PH_WINDOWS + BOOL Success; + USHORT NodeId; + ULONG HighestNode; + USHORT HighestNodeShort; + ULONG SelectedCount; + ULONG TotalProcessors; + ULONGLONG SelectedMask; + PRTL Rtl; + PALLOCATOR Allocator; + PPERFECT_HASH_IOCP_NODE Nodes; + + if (ContextIocp->Nodes) { + return S_OK; + } + + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + if (!GetNumaHighestNodeNumber(&HighestNode)) { + SYS_ERROR(GetNumaHighestNodeNumber); + return PH_E_SYSTEM_CALL_FAILED; + } + + if (HighestNode >= 64) { + return E_NOTIMPL; + } + + HighestNodeShort = (USHORT)HighestNode; + + SelectedMask = ContextIocp->Flags.UseNumaNodeMask ? + ContextIocp->NumaNodeMask : + PERFECT_HASH_NUMA_NODE_MASK_ALL; + + SelectedCount = 0; + TotalProcessors = 0; + + for (NodeId = 0; NodeId <= HighestNodeShort; NodeId++) { + GROUP_AFFINITY Affinity = { 0 }; + ULONG ProcessorCount; + + Success = GetNumaNodeProcessorMaskEx(NodeId, &Affinity); + if (!Success) { + continue; + } + + if (SelectedMask != PERFECT_HASH_NUMA_NODE_MASK_ALL && + !(SelectedMask & (1ULL << NodeId))) { + continue; + } + + ProcessorCount = (ULONG)( + Rtl->PopulationCountPointer((ULONG_PTR)Affinity.Mask) + ); + + if (ProcessorCount == 0) { + continue; + } + + SelectedCount++; + TotalProcessors += ProcessorCount; + } + + if (SelectedCount == 0 || TotalProcessors == 0) { + return E_INVALIDARG; + } + + Nodes = (PPERFECT_HASH_IOCP_NODE)( + Allocator->Vtbl->Calloc(Allocator, SelectedCount, sizeof(*Nodes)) + ); + if (!Nodes) { + return E_OUTOFMEMORY; + } + + ContextIocp->Nodes = Nodes; + ContextIocp->NodeCount = SelectedCount; + ContextIocp->IoCompletionPortCount = SelectedCount; + + SelectedCount = 0; + + for (NodeId = 0; NodeId <= HighestNodeShort; NodeId++) { + GROUP_AFFINITY Affinity = { 0 }; + ULONG ProcessorCount; + PPERFECT_HASH_IOCP_NODE Node; + + Success = GetNumaNodeProcessorMaskEx(NodeId, &Affinity); + if (!Success) { + continue; + } + + if (SelectedMask != PERFECT_HASH_NUMA_NODE_MASK_ALL && + !(SelectedMask & (1ULL << NodeId))) { + continue; + } + + ProcessorCount = (ULONG)( + Rtl->PopulationCountPointer((ULONG_PTR)Affinity.Mask) + ); + + if (ProcessorCount == 0) { + continue; + } + + Node = &Nodes[SelectedCount++]; + Node->ContextIocp = ContextIocp; + Node->NodeId = NodeId; + Node->ProcessorCount = ProcessorCount; + Node->GroupAffinity = Affinity; + } + + // + // Distribute worker thread counts across nodes. + // + + { + ULONG Index; + ULONG TargetConcurrency; + ULONG Remaining; + + TargetConcurrency = ContextIocp->MaximumConcurrency; + if (TargetConcurrency == 0 || TargetConcurrency > TotalProcessors) { + TargetConcurrency = TotalProcessors; + } + + Remaining = TargetConcurrency; + + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &Nodes[Index]; + ULONG Share = (Node->ProcessorCount * TargetConcurrency) / + TotalProcessors; + Node->WorkerThreadCount = Share; + Remaining -= Share; + } + + Index = 0; + while (Remaining) { + Nodes[Index].WorkerThreadCount++; + Remaining--; + Index++; + if (Index == ContextIocp->NodeCount) { + Index = 0; + } + } + + ContextIocp->TotalWorkerThreadCount = TargetConcurrency; + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#endif +} + +static +HRESULT +PerfectHashContextIocpCreateIoCompletionPorts( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +{ +#ifdef PH_WINDOWS + ULONG Index; + + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; + + if (Node->IoCompletionPort) { + continue; + } + + Node->IoCompletionPort = CreateIoCompletionPort( + INVALID_HANDLE_VALUE, + NULL, + 0, + Node->ProcessorCount + ); + + if (!Node->IoCompletionPort) { + SYS_ERROR(CreateIoCompletionPort); + return PH_E_SYSTEM_CALL_FAILED; + } + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#endif +} + +static +HRESULT +PerfectHashContextIocpCreateWorkerThreads( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ) +{ +#ifdef PH_WINDOWS + ULONG Index; + PALLOCATOR Allocator; + + Allocator = ContextIocp->Allocator; + + for (Index = 0; Index < ContextIocp->NodeCount; Index++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; + ULONG ThreadIndex; + + if (Node->WorkerThreadCount == 0) { + continue; + } + + if (Node->WorkerThreads) { + continue; + } + + Node->WorkerThreads = (PHANDLE)( + Allocator->Vtbl->Calloc(Allocator, + Node->WorkerThreadCount, + sizeof(HANDLE)) + ); + if (!Node->WorkerThreads) { + return E_OUTOFMEMORY; + } + + for (ThreadIndex = 0; + ThreadIndex < Node->WorkerThreadCount; + ThreadIndex++) { + HANDLE ThreadHandle; + + ThreadHandle = CreateThread(NULL, + 0, + PerfectHashIocpWorkerThreadProc, + Node, + 0, + NULL); + if (!ThreadHandle) { + SYS_ERROR(CreateThread); + return PH_E_SYSTEM_CALL_FAILED; + } + + Node->WorkerThreads[ThreadIndex] = ThreadHandle; + } + } + + return S_OK; +#else + UNREFERENCED_PARAMETER(ContextIocp); + return E_NOTIMPL; +#endif +} + +PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY + PerfectHashContextIocpSetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetMaximumConcurrency( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG MaximumConcurrency + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (MaximumConcurrency == 0) { + return E_INVALIDARG; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + ContextIocp->MaximumConcurrency = MaximumConcurrency; + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY + PerfectHashContextIocpGetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetMaximumConcurrency( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PULONG MaximumConcurrency + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + *MaximumConcurrency = ContextIocp->MaximumConcurrency; + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK + PerfectHashContextIocpSetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetNumaNodeMask( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + ContextIocp->NumaNodeMask = NumaNodeMask; + ContextIocp->Flags.UseNumaNodeMask = TRUE; + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK + PerfectHashContextIocpGetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetNumaNodeMask( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(NumaNodeMask)) { + return E_POINTER; + } + + *NumaNodeMask = ContextIocp->NumaNodeMask; + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpSetBaseOutputDirectory; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetBaseOutputDirectory( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PCUNICODE_STRING BaseOutputDirectory + ) +{ + HRESULT Result = S_OK; + PPERFECT_HASH_PATH Path = NULL; + PPERFECT_HASH_PATH_PARTS Parts = NULL; + PPERFECT_HASH_DIRECTORY Directory; + PERFECT_HASH_DIRECTORY_CREATE_FLAGS DirectoryCreateFlags = { 0 }; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectory)) { + return E_POINTER; + } + + if (!IsValidMinimumDirectoryNullTerminatedUnicodeString( + BaseOutputDirectory)) { + return E_INVALIDARG; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + if (ContextIocp->BaseOutputDirectory) { + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + return PH_E_CONTEXT_BASE_OUTPUT_DIRECTORY_ALREADY_SET; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_PATH, + &Path); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashPathCreateInstance, Result); + goto Error; + } + + Result = Path->Vtbl->Copy(Path, BaseOutputDirectory, &Parts, NULL); + if (FAILED(Result)) { + PH_ERROR(PerfectHashPathCopy, Result); + goto Error; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_DIRECTORY, + &ContextIocp->BaseOutputDirectory); + + if (FAILED(Result)) { + PH_ERROR(CreateInstancePerfectHashDirectory, Result); + goto Error; + } + + Directory = ContextIocp->BaseOutputDirectory; + + Result = Directory->Vtbl->Create(Directory, + Path, + &DirectoryCreateFlags); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashDirectoryCreate, Result); + goto Error; + } + + goto End; + +Error: + + if (Result == S_OK) { + Result = PH_E_CONTEXT_SET_BASE_OUTPUT_DIRECTORY_FAILED; + } + +End: + + RELEASE(Path); + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + return Result; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpGetBaseOutputDirectory; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetBaseOutputDirectory( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PPERFECT_HASH_DIRECTORY *BaseOutputDirectoryPointer + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectoryPointer)) { + return E_POINTER; + } + + *BaseOutputDirectoryPointer = ContextIocp->BaseOutputDirectory; + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE PerfectHashContextIocpBulkCreate; +PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW PerfectHashContextIocpBulkCreateArgvW; +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractBulkCreateArgsFromArgvW; +PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE PerfectHashContextIocpTableCreate; +PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW + PerfectHashContextIocpTableCreateArgvW; +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractTableCreateArgsFromArgvW; +PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA + PerfectHashContextIocpTableCreateArgvA; +PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA + PerfectHashContextIocpBulkCreateArgvA; + +static +HRESULT +PerfectHashContextIocpInvokeLegacyContextArgvW( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLineW, + _In_ BOOLEAN IsBulkCreate + ) +{ + HRESULT Result; + PPERFECT_HASH_CONTEXT Context; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_CONTEXT, + &Context); + if (FAILED(Result)) { + return Result; + } + + if (ContextIocp->MaximumConcurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency( + Context, + ContextIocp->MaximumConcurrency + ); + if (FAILED(Result)) { + RELEASE(Context); + return Result; + } + } + + if (IsBulkCreate) { + Result = Context->Vtbl->BulkCreateArgvW(Context, + NumberOfArguments, + ArgvW, + CommandLineW); + } else { + Result = Context->Vtbl->TableCreateArgvW(Context, + NumberOfArguments, + ArgvW, + CommandLineW); + } + + RELEASE(Context); + return Result; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpBulkCreate( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PCUNICODE_STRING KeysDirectory, + PCUNICODE_STRING BaseOutputDirectory, + PERFECT_HASH_ALGORITHM_ID AlgorithmId, + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(KeysDirectory); + UNREFERENCED_PARAMETER(BaseOutputDirectory); + UNREFERENCED_PARAMETER(AlgorithmId); + UNREFERENCED_PARAMETER(HashFunctionId); + UNREFERENCED_PARAMETER(MaskFunctionId); + UNREFERENCED_PARAMETER(ContextBulkCreateFlags); + UNREFERENCED_PARAMETER(KeysLoadFlags); + UNREFERENCED_PARAMETER(TableCreateFlags); + UNREFERENCED_PARAMETER(TableCompileFlags); + UNREFERENCED_PARAMETER(TableCreateParameters); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpBulkCreateArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW + ) +{ + return PerfectHashContextIocpInvokeLegacyContextArgvW( + ContextIocp, + NumberOfArguments, + ArgvW, + CommandLineW, + TRUE + ); +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpExtractBulkCreateArgsFromArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW, + PUNICODE_STRING KeysDirectory, + PUNICODE_STRING BaseOutputDirectory, + PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PULONG MaximumConcurrency, + PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvW); + UNREFERENCED_PARAMETER(CommandLineW); + UNREFERENCED_PARAMETER(KeysDirectory); + UNREFERENCED_PARAMETER(BaseOutputDirectory); + UNREFERENCED_PARAMETER(AlgorithmId); + UNREFERENCED_PARAMETER(HashFunctionId); + UNREFERENCED_PARAMETER(MaskFunctionId); + UNREFERENCED_PARAMETER(MaximumConcurrency); + UNREFERENCED_PARAMETER(ContextBulkCreateFlags); + UNREFERENCED_PARAMETER(KeysLoadFlags); + UNREFERENCED_PARAMETER(TableCreateFlags); + UNREFERENCED_PARAMETER(TableCompileFlags); + UNREFERENCED_PARAMETER(TableCreateParameters); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpTableCreate( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PCUNICODE_STRING KeysPath, + PCUNICODE_STRING BaseOutputDirectory, + PERFECT_HASH_ALGORITHM_ID AlgorithmId, + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(KeysPath); + UNREFERENCED_PARAMETER(BaseOutputDirectory); + UNREFERENCED_PARAMETER(AlgorithmId); + UNREFERENCED_PARAMETER(HashFunctionId); + UNREFERENCED_PARAMETER(MaskFunctionId); + UNREFERENCED_PARAMETER(ContextTableCreateFlags); + UNREFERENCED_PARAMETER(KeysLoadFlags); + UNREFERENCED_PARAMETER(TableCreateFlags); + UNREFERENCED_PARAMETER(TableCompileFlags); + UNREFERENCED_PARAMETER(TableCreateParameters); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpTableCreateArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW + ) +{ + return PerfectHashContextIocpInvokeLegacyContextArgvW( + ContextIocp, + NumberOfArguments, + ArgvW, + CommandLineW, + FALSE + ); +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpExtractTableCreateArgsFromArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW, + PUNICODE_STRING KeysPath, + PUNICODE_STRING BaseOutputDirectory, + PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PULONG MaximumConcurrency, + PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvW); + UNREFERENCED_PARAMETER(CommandLineW); + UNREFERENCED_PARAMETER(KeysPath); + UNREFERENCED_PARAMETER(BaseOutputDirectory); + UNREFERENCED_PARAMETER(AlgorithmId); + UNREFERENCED_PARAMETER(HashFunctionId); + UNREFERENCED_PARAMETER(MaskFunctionId); + UNREFERENCED_PARAMETER(MaximumConcurrency); + UNREFERENCED_PARAMETER(ContextTableCreateFlags); + UNREFERENCED_PARAMETER(KeysLoadFlags); + UNREFERENCED_PARAMETER(TableCreateFlags); + UNREFERENCED_PARAMETER(TableCompileFlags); + UNREFERENCED_PARAMETER(TableCreateParameters); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpTableCreateArgvA( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPSTR *ArgvA + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + + return E_NOTIMPL; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpBulkCreateArgvA( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPSTR *ArgvA + ) +{ + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + + return E_NOTIMPL; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashContextIocp.h b/src/PerfectHash/PerfectHashContextIocp.h new file mode 100644 index 00000000..8ebe6ae8 --- /dev/null +++ b/src/PerfectHash/PerfectHashContextIocp.h @@ -0,0 +1,239 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashContextIocp.h + +Abstract: + + This is the private header file for the PERFECT_HASH_CONTEXT_IOCP + component of the perfect hash library. It defines the structure, and + function pointer typedefs for private non-vtbl members. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef union _PERFECT_HASH_CONTEXT_IOCP_STATE { + struct { + + ULONG Initialized:1; + ULONG Running:1; + ULONG Stopping:1; + ULONG Stopped:1; + + ULONG Unused:28; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CONTEXT_IOCP_STATE; +C_ASSERT(sizeof(PERFECT_HASH_CONTEXT_IOCP_STATE) == sizeof(ULONG)); +typedef PERFECT_HASH_CONTEXT_IOCP_STATE *PPERFECT_HASH_CONTEXT_IOCP_STATE; + +typedef union _PERFECT_HASH_CONTEXT_IOCP_FLAGS { + struct { + + // + // When set, indicates an explicit NUMA node mask is in effect. + // + + ULONG UseNumaNodeMask:1; + + ULONG Unused:31; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CONTEXT_IOCP_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_CONTEXT_IOCP_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_CONTEXT_IOCP_FLAGS *PPERFECT_HASH_CONTEXT_IOCP_FLAGS; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_IOCP_COMPLETION_CALLBACK)( + _In_ struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ); +typedef PERFECT_HASH_IOCP_COMPLETION_CALLBACK + *PPERFECT_HASH_IOCP_COMPLETION_CALLBACK; + +// +// IOCP work item header used to route completions to a per-item callback. +// + +#define PERFECT_HASH_IOCP_SHUTDOWN_KEY ((ULONG_PTR)1) + +#define PH_IOCP_WORK_SIGNATURE 0x50434F49u // 'IOCP' + +#define PH_IOCP_WORK_FLAG_PIPE 0x00000001 +#define PH_IOCP_WORK_FLAG_BULK 0x00000002 +#define PH_IOCP_WORK_FLAG_ASYNC 0x00000004 +#define PH_IOCP_WORK_FLAG_FILE_WORK 0x00000008 + +typedef struct _PERFECT_HASH_IOCP_WORK { + OVERLAPPED Overlapped; + ULONG Signature; + ULONG Flags; + PPERFECT_HASH_IOCP_COMPLETION_CALLBACK CompletionCallback; + PVOID CompletionContext; +} PERFECT_HASH_IOCP_WORK; +typedef PERFECT_HASH_IOCP_WORK *PPERFECT_HASH_IOCP_WORK; + +C_ASSERT(FIELD_OFFSET(PERFECT_HASH_IOCP_WORK, Overlapped) == 0); + +typedef struct _PERFECT_HASH_IOCP_NODE { + struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp; + ULONG NodeId; + ULONG ProcessorCount; +#ifdef PH_WINDOWS + GROUP_AFFINITY GroupAffinity; +#else + ULONGLONG ProcessorMask; +#endif + HANDLE IoCompletionPort; + HANDLE *WorkerThreads; + ULONG WorkerThreadCount; + ULONG Padding1; +} PERFECT_HASH_IOCP_NODE; +typedef PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT_IOCP { + + COMMON_COMPONENT_HEADER(PERFECT_HASH_CONTEXT_IOCP); + + // + // Configuration. + // + + ULONG MaximumConcurrency; + ULONG NumaNodeCount; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; + ULONG Padding1; + ULONG Padding2; + + // + // Pointer to the base output directory, if set. + // + + PPERFECT_HASH_DIRECTORY BaseOutputDirectory; + + // + // Threading resources (per NUMA node IOCPs, worker threads, etc.). + // + + PPERFECT_HASH_IOCP_NODE Nodes; + ULONG NodeCount; + ULONG TotalWorkerThreadCount; + ULONG IoCompletionPortCount; + ULONG Padding3; + + // + // Completion dispatch callback (owned by higher-level components). + // + + PPERFECT_HASH_IOCP_COMPLETION_CALLBACK CompletionCallback; + PVOID CompletionContext; + + // + // Shutdown and coordination. + // + + HANDLE ShutdownEvent; + HANDLE StartedEvent; + + // + // Backing vtbl. + // + + PERFECT_HASH_CONTEXT_IOCP_VTBL Interface; + +} PERFECT_HASH_CONTEXT_IOCP; +typedef PERFECT_HASH_CONTEXT_IOCP *PPERFECT_HASH_CONTEXT_IOCP; + +#define TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp) \ + TryAcquireSRWLockExclusive(&ContextIocp->Lock) + +#define ReleasePerfectHashContextIocpLockExclusive(ContextIocp) \ + ReleaseSRWLockExclusive(&ContextIocp->Lock) + +// +// Function pointer typedefs for non-vtbl members (if applicable). +// + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_INITIALIZE)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_INITIALIZE + *PPERFECT_HASH_CONTEXT_IOCP_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_CONTEXT_IOCP_RUNDOWN)( + _In_ _Post_ptr_invalid_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_RUNDOWN + *PPERFECT_HASH_CONTEXT_IOCP_RUNDOWN; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_START)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_START + *PPERFECT_HASH_CONTEXT_IOCP_START; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_STOP)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp + ); +typedef PERFECT_HASH_CONTEXT_IOCP_STOP + *PPERFECT_HASH_CONTEXT_IOCP_STOP; + +extern PERFECT_HASH_CONTEXT_IOCP_INITIALIZE PerfectHashContextIocpInitialize; +extern PERFECT_HASH_CONTEXT_IOCP_RUNDOWN PerfectHashContextIocpRundown; +extern PERFECT_HASH_CONTEXT_IOCP_START PerfectHashContextIocpStart; +extern PERFECT_HASH_CONTEXT_IOCP_STOP PerfectHashContextIocpStop; + +extern PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY + PerfectHashContextIocpSetMaximumConcurrency; +extern PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY + PerfectHashContextIocpGetMaximumConcurrency; +extern PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK + PerfectHashContextIocpSetNumaNodeMask; +extern PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK + PerfectHashContextIocpGetNumaNodeMask; +extern PERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpSetBaseOutputDirectory; +extern PERFECT_HASH_CONTEXT_IOCP_GET_BASE_OUTPUT_DIRECTORY + PerfectHashContextIocpGetBaseOutputDirectory; +extern PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE + PerfectHashContextIocpBulkCreate; +extern PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW + PerfectHashContextIocpBulkCreateArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractBulkCreateArgsFromArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE + PerfectHashContextIocpTableCreate; +extern PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVW + PerfectHashContextIocpTableCreateArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractTableCreateArgsFromArgvW; +extern PERFECT_HASH_CONTEXT_IOCP_TABLE_CREATE_ARGVA + PerfectHashContextIocpTableCreateArgvA; +extern PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVA + PerfectHashContextIocpBulkCreateArgvA; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashDirectory.c b/src/PerfectHash/PerfectHashDirectory.c index fef5faab..495ed9b7 100644 --- a/src/PerfectHash/PerfectHashDirectory.c +++ b/src/PerfectHash/PerfectHashDirectory.c @@ -1725,10 +1725,10 @@ Return Value: goto Error; } - if (IsDirectoryClosed(Directory)) { - Result = PH_E_DIRECTORY_CLOSED; - goto Error; - } + // + // Allow removals even if the directory has been closed. This is required + // to break directory/file reference cycles during rundown. + // // // Argument validation complete. Add the file to the files list and diff --git a/src/PerfectHash/PerfectHashErrors.dbg b/src/PerfectHash/PerfectHashErrors.dbg index 16c3078d..f6a1a93a 100644 --- a/src/PerfectHash/PerfectHashErrors.dbg +++ b/src/PerfectHash/PerfectHashErrors.dbg @@ -22,6 +22,7 @@ struct { (HRESULT) PH_S_FUNCTION_HOOK_CALLBACK_DLL_INITIALIZED, (char*) "PH_S_FUNCTION_HOOK_CALLBACK_DLL_INITIALIZED", (HRESULT) PH_S_TABLE_CREATE_PARAMETER_NOT_FOUND, (char*) "PH_S_TABLE_CREATE_PARAMETER_NOT_FOUND", (HRESULT) PH_S_CU_KERNEL_RUNTIME_TARGET_REACHED, (char*) "PH_S_CU_KERNEL_RUNTIME_TARGET_REACHED", + (HRESULT) PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED, (char*) "PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED", (HRESULT) PH_I_CREATE_TABLE_ROUTINE_RECEIVED_SHUTDOWN_EVENT, (char*) "PH_I_CREATE_TABLE_ROUTINE_RECEIVED_SHUTDOWN_EVENT", (HRESULT) PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION, (char*) "PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION", (HRESULT) PH_I_MAXIMUM_NUMBER_OF_TABLE_RESIZE_EVENTS_REACHED, (char*) "PH_I_MAXIMUM_NUMBER_OF_TABLE_RESIZE_EVENTS_REACHED", @@ -177,6 +178,7 @@ struct { (HRESULT) PH_E_ERROR_DURING_PREPARE_BUILD_SOLUTION_BATCH_FILE, (char*) "PH_E_ERROR_DURING_PREPARE_BUILD_SOLUTION_BATCH_FILE", (HRESULT) PH_E_ERROR_DURING_PREPARE_C_HEADER_COMPILED_PERFECT_HASH_FILE, (char*) "PH_E_ERROR_DURING_PREPARE_C_HEADER_COMPILED_PERFECT_HASH_FILE", (HRESULT) PH_E_ERROR_DURING_PREPARE_VCPROPS_COMPILED_PERFECT_HASH_FILE, (char*) "PH_E_ERROR_DURING_PREPARE_VCPROPS_COMPILED_PERFECT_HASH_FILE", + (HRESULT) PH_E_SERVER_BULK_CREATE_FAILED, (char*) "PH_E_SERVER_BULK_CREATE_FAILED", (HRESULT) PH_E_ERROR_DURING_SAVE_VCPROJECT_DLL_FILE, (char*) "PH_E_ERROR_DURING_SAVE_VCPROJECT_DLL_FILE", (HRESULT) PH_E_ERROR_DURING_SAVE_C_SOURCE_SUPPORT_FILE, (char*) "PH_E_ERROR_DURING_SAVE_C_SOURCE_SUPPORT_FILE", (HRESULT) PH_E_ERROR_DURING_SAVE_C_SOURCE_TEST_FILE, (char*) "PH_E_ERROR_DURING_SAVE_C_SOURCE_TEST_FILE", diff --git a/src/PerfectHash/PerfectHashErrors.mc b/src/PerfectHash/PerfectHashErrors.mc index 56b8c19a..8a0d7ddd 100644 --- a/src/PerfectHash/PerfectHashErrors.mc +++ b/src/PerfectHash/PerfectHashErrors.mc @@ -136,6 +136,14 @@ Language=English CUDA kernel runtime target reached. . +MessageId=0x00f +Severity=Success +Facility=ITF +SymbolicName=PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED +Language=English +Bulk create request completed successfully. +. + ; ;//////////////////////////////////////////////////////////////////////////////// ;// PH_SEVERITY_INFORMATIONAL @@ -2338,8 +2346,16 @@ Language=English Error preparing CompiledPerfectHash.props file. . +MessageId=0x290 +Severity=Fail +Facility=ITF +SymbolicName=PH_E_SERVER_BULK_CREATE_FAILED +Language=English +Bulk create request failed. +. + ;// -;// Spare IDs: 0x290, 0x291. +;// Spare ID: 0x291. ;// MessageId=0x292 diff --git a/src/PerfectHash/PerfectHashErrors_English.bin b/src/PerfectHash/PerfectHashErrors_English.bin index 96762a9f4697fe834837dfe94fe523b5fffb85de..a319a2e7b8df6c652982657e31fe860355ae4689 100644 GIT binary patch delta 385 zcmY+AEO|DkOR7FrRQrqHCQA>wzf;u=lbk-%Hfg(zc)~%3&bkIrsb#5+# zxGrC(AoU^wSI6Tx;(Ijp delta 287 zcmWm6F-t;G7{>AEIrk+sNRSL7!Acs03K|-Oz`4=EEr+wTL{uLjFwntTgrFSTr14V2 zZE$EvaOyLh!7*{UV}s8)Ud`ldtu}toW4>ghYnD|x~^7T$E)rwj8n|!YG VDe9$`q{jP;QD@3(^6NRiS^>24M4JEr diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c new file mode 100644 index 00000000..a2ea2ffb --- /dev/null +++ b/src/PerfectHash/PerfectHashServer.c @@ -0,0 +1,2863 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashServer.c + +Abstract: + + This module implements the PERFECT_HASH_SERVER component. The initial + implementation provides configuration plumbing and scaffolding for the + IOCP runtime and request processing. + +--*/ + +#include "stdafx.h" +#include "PerfectHashTls.h" +#include +#include + +#ifdef PH_WINDOWS + +static const UNICODE_STRING PerfectHashServerDefaultPipeName = + RTL_CONSTANT_STRING(L"\\\\.\\pipe\\PerfectHashServer"); + +typedef enum _PERFECT_HASH_SERVER_PIPE_STATE { + PerfectHashServerPipeStateInvalid = 0, + PerfectHashServerPipeStateAccepting, + PerfectHashServerPipeStateReadingHeader, + PerfectHashServerPipeStateReadingPayload, + PerfectHashServerPipeStateWritingResponseHeader, + PerfectHashServerPipeStateWritingResponsePayload +} PERFECT_HASH_SERVER_PIPE_STATE; + +typedef struct _PERFECT_HASH_SERVER_PIPE { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_IOCP_NODE Node; + HANDLE Pipe; + PERFECT_HASH_SERVER_PIPE_STATE State; + ULONG Flags; + ULONG BytesTransferred; + ULONG PayloadLength; + BOOLEAN ShutdownAfterSend; + UCHAR Padding1[7]; + PERFECT_HASH_SERVER_REQUEST_HEADER RequestHeader; + PERFECT_HASH_SERVER_RESPONSE_HEADER ResponseHeader; + PVOID PayloadBuffer; + ULONG PayloadBufferSize; + ULONG Padding2; +} PERFECT_HASH_SERVER_PIPE; +typedef PERFECT_HASH_SERVER_PIPE *PPERFECT_HASH_SERVER_PIPE; + +typedef struct _PERFECT_HASH_SERVER_BULK_REQUEST { + ULONG SizeOfStruct; + volatile LONG OutstandingWorkItems; + volatile LONG FailedWorkItems; + volatile LONG PendingNodes; + volatile LONG CompletionSignaled; + ULONG TotalWorkItems; + ULONG PerFileMaximumConcurrency; + ULONG Padding1; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_CONTEXT ParseContext; + LPWSTR *ArgvW; + PWSTR CommandLineBuffer; + PERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags; + PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags; + ULONG Padding2; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags; + ULONG Padding3; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters; + PERFECT_HASH_ALGORITHM_ID AlgorithmId; + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId; + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId; + ULONG Padding4; + UNICODE_STRING BaseOutputDirectory; + PWSTR BaseOutputDirectoryBuffer; + HANDLE CompletionEvent; + HANDLE ResultMappingHandle; + PPERFECT_HASH_SERVER_BULK_RESULT ResultMapping; + volatile LONG FirstFailure; + volatile LONG DispatchComplete; + PPERFECT_HASH_IOCP_NODE Nodes; + ULONG NodeCount; + ULONG NextNodeIndex; + PLONG NodeOutstandingCounts; +} PERFECT_HASH_SERVER_BULK_REQUEST; +typedef PERFECT_HASH_SERVER_BULK_REQUEST *PPERFECT_HASH_SERVER_BULK_REQUEST; + +typedef struct _PERFECT_HASH_SERVER_BULK_WORK_ITEM { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_SERVER_BULK_REQUEST Request; + PPERFECT_HASH_IOCP_NODE Node; + ULONG NodeIndex; + ULONG Padding1; + UNICODE_STRING KeysPath; + PWSTR KeysPathBuffer; +} PERFECT_HASH_SERVER_BULK_WORK_ITEM; +typedef PERFECT_HASH_SERVER_BULK_WORK_ITEM *PPERFECT_HASH_SERVER_BULK_WORK_ITEM; + +static +HRESULT +PerfectHashServerCreatePipes( + _In_ PPERFECT_HASH_SERVER Server + ); + +static +VOID +PerfectHashServerDestroyPipes( + _In_ PPERFECT_HASH_SERVER Server + ); + +static +HRESULT +PerfectHashServerIssueConnect( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueReadHeader( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueReadPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueWriteResponseHeader( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerIssueWriteResponsePayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +VOID +PerfectHashServerResetPipe( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ BOOLEAN Reconnect + ); + +static +HRESULT +PerfectHashServerDispatchRequest( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + +static +HRESULT +PerfectHashServerDispatchBulkCreateDirectoryRequest( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLine, + _Out_ PBOOLEAN ArgvWOwned + ); + +static +HRESULT +PerfectHashServerPrepareErrorPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ HRESULT Error + ); + +static +HRESULT +PerfectHashServerPrepareBulkCreateDirectoryPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ HANDLE EventHandle, + _In_ HANDLE ResultHandle + ); + +static +HRESULT +PerfectHashServerEnqueueBulkRequest( + _In_ PPERFECT_HASH_SERVER_BULK_REQUEST Request, + _In_ PUNICODE_STRING KeysDirectory + ); + +static +VOID +PerfectHashServerCompleteBulkRequest( + _In_ PPERFECT_HASH_SERVER_BULK_REQUEST Request + ); + +static +HRESULT +PerfectHashServerBulkCreateWorkItemCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ); + +static +VOID +PerfectHashServerLogBulkCreateException( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_ struct _EXCEPTION_POINTERS *ExceptionPointers + ); + +static +LONG +PerfectHashServerBulkCreateExceptionFilter( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_ struct _EXCEPTION_POINTERS *ExceptionPointers + ); + +static +HRESULT +PerfectHashServerIocpCompletionCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ); + +#endif + +PERFECT_HASH_SERVER_INITIALIZE PerfectHashServerInitialize; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerInitialize( + PPERFECT_HASH_SERVER Server + ) +/*++ + +Routine Description: + + Initializes a PERFECT_HASH_SERVER instance. This routine creates the + Rtl, Allocator, and IOCP context components and initializes default + configuration values. + +Arguments: + + Server - Supplies a pointer to a PERFECT_HASH_SERVER structure for which + initialization is to be performed. + +Return Value: + + S_OK on success, an appropriate error code on failure. + +--*/ +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + HRESULT Result; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_RTL, + &Server->Rtl); + if (FAILED(Result)) { + return Result; + } + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_ALLOCATOR, + &Server->Allocator); + if (FAILED(Result)) { + return Result; + } + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_CONTEXT_IOCP, + &Server->ContextIocp); + if (FAILED(Result)) { + return Result; + } + + ContextIocp = Server->ContextIocp; + + Server->MaximumConcurrency = ContextIocp->MaximumConcurrency; + Server->NumaNodeMask = ContextIocp->NumaNodeMask; + Server->NumaNodeCount = ContextIocp->NumaNodeCount; + + Server->StartedEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!Server->StartedEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + Server->ShutdownEvent = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!Server->ShutdownEvent) { + return PH_E_SYSTEM_CALL_FAILED; + } + + Server->Endpoint = PerfectHashServerDefaultPipeName; + Server->Flags.LocalOnly = TRUE; + Server->Flags.EndpointAllocated = FALSE; + Server->State.Initialized = TRUE; + + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_RUNDOWN PerfectHashServerRundown; + +_Use_decl_annotations_ +VOID +PerfectHashServerRundown( + PPERFECT_HASH_SERVER Server + ) +/*++ + +Routine Description: + + Releases resources associated with a PERFECT_HASH_SERVER instance. + +Arguments: + + Server - Supplies a pointer to a PERFECT_HASH_SERVER structure for which + rundown is to be performed. + +Return Value: + + None. + +--*/ +{ + if (!ARGUMENT_PRESENT(Server)) { + return; + } + + Server->State.Stopping = TRUE; + + if (Server->ContextIocp) { +#ifdef PH_WINDOWS + PerfectHashServerDestroyPipes(Server); +#endif + PerfectHashContextIocpStop(Server->ContextIocp); + } + +#ifdef PH_WINDOWS + if (Server->ShutdownEvent) { + CloseHandle(Server->ShutdownEvent); + Server->ShutdownEvent = NULL; + } + if (Server->StartedEvent) { + CloseHandle(Server->StartedEvent); + Server->StartedEvent = NULL; + } +#endif + + if (Server->Flags.EndpointAllocated && Server->Endpoint.Buffer) { + if (Server->Allocator) { + Server->Allocator->Vtbl->FreePointer( + Server->Allocator, + (PVOID *)&Server->Endpoint.Buffer + ); + } + Server->Endpoint.Length = 0; + Server->Endpoint.MaximumLength = 0; + Server->Flags.EndpointAllocated = FALSE; + } + + RELEASE(Server->ContextIocp); + RELEASE(Server->Allocator); + RELEASE(Server->Rtl); +} + +PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY + PerfectHashServerSetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetMaximumConcurrency( + PPERFECT_HASH_SERVER Server, + ULONG MaximumConcurrency + ) +{ + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (MaximumConcurrency == 0) { + return E_INVALIDARG; + } + + if (Server->ContextIocp) { + Result = Server->ContextIocp->Vtbl->SetMaximumConcurrency( + Server->ContextIocp, + MaximumConcurrency + ); + if (FAILED(Result)) { + return Result; + } + } + + Server->MaximumConcurrency = MaximumConcurrency; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY + PerfectHashServerGetMaximumConcurrency; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetMaximumConcurrency( + PPERFECT_HASH_SERVER Server, + PULONG MaximumConcurrency + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + *MaximumConcurrency = Server->MaximumConcurrency; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK PerfectHashServerSetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetNumaNodeMask( + PPERFECT_HASH_SERVER Server, + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->ContextIocp) { + Result = Server->ContextIocp->Vtbl->SetNumaNodeMask( + Server->ContextIocp, + NumaNodeMask + ); + if (FAILED(Result)) { + return Result; + } + } + + Server->NumaNodeMask = NumaNodeMask; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK PerfectHashServerGetNumaNodeMask; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetNumaNodeMask( + PPERFECT_HASH_SERVER Server, + PPERFECT_HASH_NUMA_NODE_MASK NumaNodeMask + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(NumaNodeMask)) { + return E_POINTER; + } + + *NumaNodeMask = Server->NumaNodeMask; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_ENDPOINT PerfectHashServerSetEndpoint; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetEndpoint( + PPERFECT_HASH_SERVER Server, + PCUNICODE_STRING Endpoint + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + PWSTR Buffer; + ULONG BytesToAllocate; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Endpoint)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + if (!IsValidNullTerminatedUnicodeString(Endpoint)) { + return E_INVALIDARG; + } + + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + BytesToAllocate = Endpoint->Length + sizeof(WCHAR); + if (BytesToAllocate < Endpoint->Length) { + return E_INVALIDARG; + } + if (BytesToAllocate > (ULONG)USHRT_MAX) { + return E_INVALIDARG; + } + + Buffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, 1, BytesToAllocate); + if (!Buffer) { + return E_OUTOFMEMORY; + } + + CopyMemory(Buffer, Endpoint->Buffer, Endpoint->Length); + + if (Server->Flags.EndpointAllocated && Server->Endpoint.Buffer) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Server->Endpoint.Buffer); + } + + Server->Endpoint.Buffer = Buffer; + Server->Endpoint.Length = Endpoint->Length; + Server->Endpoint.MaximumLength = (USHORT)BytesToAllocate; + Server->Flags.EndpointAllocated = TRUE; + + return S_OK; +} + +PERFECT_HASH_SERVER_GET_ENDPOINT PerfectHashServerGetEndpoint; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetEndpoint( + PPERFECT_HASH_SERVER Server, + PUNICODE_STRING Endpoint + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Endpoint)) { + return E_POINTER; + } + + *Endpoint = Server->Endpoint; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_LOCAL_ONLY PerfectHashServerSetLocalOnly; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetLocalOnly( + PPERFECT_HASH_SERVER Server, + BOOLEAN LocalOnly + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.LocalOnly = LocalOnly ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_LOCAL_ONLY PerfectHashServerGetLocalOnly; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetLocalOnly( + PPERFECT_HASH_SERVER Server, + PBOOLEAN LocalOnly + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(LocalOnly)) { + return E_POINTER; + } + + *LocalOnly = (Server->Flags.LocalOnly != 0); + return S_OK; +} + +PERFECT_HASH_SERVER_START PerfectHashServerStart; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerStart( + PPERFECT_HASH_SERVER Server + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->ContextIocp) { + Result = PerfectHashContextIocpStart(Server->ContextIocp); + if (FAILED(Result)) { + return Result; + } + } + + Result = PerfectHashServerCreatePipes(Server); + if (FAILED(Result)) { + PerfectHashContextIocpStop(Server->ContextIocp); + return Result; + } + + Server->State.Running = TRUE; + if (Server->StartedEvent) { + SetEvent(Server->StartedEvent); + } + + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_STOP PerfectHashServerStop; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerStop( + PPERFECT_HASH_SERVER Server + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + HRESULT Result; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + Server->State.Stopping = TRUE; + + if (Server->ContextIocp) { + PerfectHashServerDestroyPipes(Server); + Result = PerfectHashContextIocpStop(Server->ContextIocp); + if (FAILED(Result)) { + return Result; + } + } + + if (Server->ShutdownEvent) { + SetEvent(Server->ShutdownEvent); + } + + Server->State.Stopped = TRUE; + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_WAIT PerfectHashServerWait; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerWait( + PPERFECT_HASH_SERVER Server + ) +{ +#ifndef PH_WINDOWS + UNREFERENCED_PARAMETER(Server); + return E_NOTIMPL; +#else + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->ShutdownEvent) { + WaitForSingleObject(Server->ShutdownEvent, INFINITE); + } + + return S_OK; +#endif +} + +PERFECT_HASH_SERVER_SUBMIT_REQUEST PerfectHashServerSubmitRequest; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSubmitRequest( + PPERFECT_HASH_SERVER Server, + PPERFECT_HASH_SERVER_REQUEST Request + ) +{ + UNREFERENCED_PARAMETER(Server); + UNREFERENCED_PARAMETER(Request); + + return E_NOTIMPL; +} + +#ifdef PH_WINDOWS + +static +HRESULT +PerfectHashServerCreatePipes( + PPERFECT_HASH_SERVER Server + ) +{ + ULONG NodeIndex; + ULONG PipeIndex; + ULONG PipeCount; + HRESULT Result; + PALLOCATOR Allocator; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->Pipes) { + return S_OK; + } + + ContextIocp = Server->ContextIocp; + Allocator = Server->Allocator; + + if (!ContextIocp || !Allocator) { + return E_UNEXPECTED; + } + + PipeCount = ContextIocp->TotalWorkerThreadCount; + if (PipeCount == 0) { + PipeCount = 1; + } + + Server->Pipes = (PPERFECT_HASH_SERVER_PIPE)( + Allocator->Vtbl->Calloc(Allocator, + PipeCount, + sizeof(*Server->Pipes)) + ); + if (!Server->Pipes) { + return E_OUTOFMEMORY; + } + + Server->PipeCount = PipeCount; + PipeIndex = 0; + + for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { + PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + ULONG InstanceIndex; + + for (InstanceIndex = 0; + InstanceIndex < Node->WorkerThreadCount; + InstanceIndex++) { + DWORD OpenMode; + DWORD PipeMode; + HANDLE PipeHandle; + HANDLE PortHandle; + PPERFECT_HASH_SERVER_PIPE Pipe; + + if (PipeIndex >= Server->PipeCount) { + break; + } + + Pipe = &Server->Pipes[PipeIndex++]; + Pipe->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + Pipe->Iocp.Flags = PH_IOCP_WORK_FLAG_PIPE; + Pipe->Iocp.CompletionCallback = + PerfectHashServerIocpCompletionCallback; + Pipe->Iocp.CompletionContext = Pipe; + Pipe->Server = Server; + Pipe->Node = Node; + + OpenMode = PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED; + PipeMode = PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT; + +#ifdef PIPE_REJECT_REMOTE_CLIENTS + if (Server->Flags.LocalOnly) { + PipeMode |= PIPE_REJECT_REMOTE_CLIENTS; + } +#endif + + PipeHandle = CreateNamedPipeW(Server->Endpoint.Buffer, + OpenMode, + PipeMode, + PIPE_UNLIMITED_INSTANCES, + 0, + 0, + 0, + NULL); + if (!PipeHandle || PipeHandle == INVALID_HANDLE_VALUE) { + SYS_ERROR(CreateNamedPipeW); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Pipe->Pipe = PipeHandle; + + PortHandle = CreateIoCompletionPort(PipeHandle, + Node->IoCompletionPort, + (ULONG_PTR)Pipe, + 0); + if (!PortHandle) { + SYS_ERROR(CreateIoCompletionPort); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Result = PerfectHashServerIssueConnect(Pipe); + if (FAILED(Result)) { + goto Error; + } + } + } + + return S_OK; + +Error: + + PerfectHashServerDestroyPipes(Server); + return Result; +} + +static +VOID +PerfectHashServerDestroyPipes( + PPERFECT_HASH_SERVER Server + ) +{ + ULONG Index; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(Server)) { + return; + } + + if (!Server->Pipes) { + return; + } + + Allocator = Server->Allocator; + + for (Index = 0; Index < Server->PipeCount; Index++) { + PPERFECT_HASH_SERVER_PIPE Pipe = &Server->Pipes[Index]; + + if (Pipe->Pipe) { + CancelIoEx(Pipe->Pipe, NULL); + CloseHandle(Pipe->Pipe); + Pipe->Pipe = NULL; + } + + if (Pipe->PayloadBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + Pipe->PayloadBufferSize = 0; + } + } + + if (Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Server->Pipes); + } + + Server->Pipes = NULL; + Server->PipeCount = 0; +} + +static +HRESULT +PerfectHashServerIssueConnect( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateAccepting; + Pipe->BytesTransferred = 0; + Pipe->PayloadLength = 0; + Pipe->ShutdownAfterSend = FALSE; + + Success = ConnectNamedPipe(Pipe->Pipe, &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + if (LastError == ERROR_PIPE_CONNECTED) { + if (!PostQueuedCompletionStatus(Pipe->Node->IoCompletionPort, + 0, + (ULONG_PTR)Pipe, + &Pipe->Iocp.Overlapped)) { + SYS_ERROR(PostQueuedCompletionStatus); + return PH_E_SYSTEM_CALL_FAILED; + } + return S_OK; + } + + SYS_ERROR(ConnectNamedPipe); + return PH_E_SYSTEM_CALL_FAILED; +} + +static +HRESULT +PerfectHashServerIssueReadHeader( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = sizeof(PERFECT_HASH_SERVER_REQUEST_HEADER) - + Pipe->BytesTransferred; + Buffer = (PUCHAR)&Pipe->RequestHeader + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateReadingHeader; + + Success = ReadFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +HRESULT +PerfectHashServerIssueReadPayload( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = Pipe->PayloadLength - Pipe->BytesTransferred; + Buffer = (PUCHAR)Pipe->PayloadBuffer + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateReadingPayload; + + Success = ReadFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +HRESULT +PerfectHashServerIssueWriteResponseHeader( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER) - + Pipe->BytesTransferred; + Buffer = (PUCHAR)&Pipe->ResponseHeader + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateWritingResponseHeader; + + Success = WriteFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +HRESULT +PerfectHashServerIssueWriteResponsePayload( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + BOOL Success; + DWORD LastError; + ULONG Remaining; + PUCHAR Buffer; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + Remaining = Pipe->ResponseHeader.PayloadLength - Pipe->BytesTransferred; + Buffer = (PUCHAR)Pipe->PayloadBuffer + Pipe->BytesTransferred; + + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + Pipe->State = PerfectHashServerPipeStateWritingResponsePayload; + + Success = WriteFile(Pipe->Pipe, + Buffer, + Remaining, + NULL, + &Pipe->Iocp.Overlapped); + if (Success) { + return S_OK; + } + + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + return S_OK; + } + + return HRESULT_FROM_WIN32(LastError); +} + +static +VOID +PerfectHashServerResetPipe( + PPERFECT_HASH_SERVER_PIPE Pipe, + BOOLEAN Reconnect + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + PPERFECT_HASH_SERVER Server; + + if (!ARGUMENT_PRESENT(Pipe)) { + return; + } + + Server = Pipe->Server; + Allocator = Server ? Server->Allocator : NULL; + Rtl = Server ? Server->Rtl : NULL; + + Pipe->BytesTransferred = 0; + Pipe->PayloadLength = 0; + Pipe->ShutdownAfterSend = FALSE; + + if (Pipe->PayloadBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + Pipe->PayloadBufferSize = 0; + } + + if (Rtl) { + ZeroMemory(&Pipe->RequestHeader, sizeof(Pipe->RequestHeader)); + ZeroMemory(&Pipe->ResponseHeader, sizeof(Pipe->ResponseHeader)); + ZeroMemory(&Pipe->Iocp.Overlapped, sizeof(Pipe->Iocp.Overlapped)); + } + + if (Pipe->Pipe) { + DisconnectNamedPipe(Pipe->Pipe); + } + + if (Reconnect && Server && + !Server->State.Stopping && + !Server->State.Stopped) { + PerfectHashServerIssueConnect(Pipe); + } +} + +static +HRESULT +PerfectHashServerPrepareErrorPayload( + PPERFECT_HASH_SERVER_PIPE Pipe, + HRESULT Error + ) +{ + BOOL FreeMessage; + DWORD Flags; + DWORD Count; + ULONG PayloadLength; + ULONG BytesToAllocate; + PRTL Rtl; + PALLOCATOR Allocator; + PWSTR Buffer; + PWSTR Message; + WCHAR Fallback[64]; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + FreeMessage = FALSE; + Message = NULL; + Fallback[0] = L'\0'; + + Flags = ( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_HMODULE | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS + ); + + Count = FormatMessageW(Flags, + PerfectHashModule, + (DWORD)Error, + 0, + (LPWSTR)&Message, + 0, + NULL); + + if (Count == 0 || !Message) { + swprintf_s(Fallback, + ARRAYSIZE(Fallback), + L"Error 0x%08lX", + (ULONG)Error); + Message = Fallback; + Count = (DWORD)wcslen(Message); + } else { + FreeMessage = TRUE; + } + + while (Count > 0) { + WCHAR Ch = Message[Count - 1]; + if (Ch == L'\r' || Ch == L'\n' || Ch == L' ' || Ch == L'\t') { + Message[Count - 1] = L'\0'; + Count--; + continue; + } + break; + } + + PayloadLength = (Count + 1) * sizeof(WCHAR); + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + ULONG MaxChars; + + MaxChars = (PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE / sizeof(WCHAR)); + if (MaxChars == 0) { + PayloadLength = 0; + } else { + MaxChars -= 1; + PayloadLength = (MaxChars + 1) * sizeof(WCHAR); + Message[MaxChars] = L'\0'; + Count = MaxChars; + } + } + + if (PayloadLength == 0) { + if (FreeMessage) { + LocalFree(Message); + } + return E_UNEXPECTED; + } + + BytesToAllocate = PayloadLength; + + if (!Pipe->PayloadBuffer || Pipe->PayloadBufferSize < BytesToAllocate) { + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + BytesToAllocate); + if (!Pipe->PayloadBuffer) { + if (FreeMessage) { + LocalFree(Message); + } + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = BytesToAllocate; + } else { + ZeroMemory(Pipe->PayloadBuffer, BytesToAllocate); + } + + Buffer = (PWSTR)Pipe->PayloadBuffer; + CopyMemory(Buffer, Message, Count * sizeof(WCHAR)); + Buffer[Count] = L'\0'; + + Pipe->ResponseHeader.PayloadLength = PayloadLength; + Pipe->ResponseHeader.Flags |= PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE; + + if (FreeMessage) { + LocalFree(Message); + } + + return S_OK; +} + +static +HRESULT +PerfectHashServerPrepareBulkCreateDirectoryPayload( + PPERFECT_HASH_SERVER_PIPE Pipe, + HANDLE EventHandle, + HANDLE ResultHandle + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + WCHAR Payload[128]; + int Count; + ULONG PayloadLength; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + Count = swprintf_s(Payload, + ARRAYSIZE(Payload), + PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT, + (ULONGLONG)(ULONG_PTR)EventHandle, + (ULONGLONG)(ULONG_PTR)ResultHandle); + if (Count <= 0) { + return E_UNEXPECTED; + } + + PayloadLength = (ULONG)((Count + 1) * sizeof(WCHAR)); + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + return E_INVALIDARG; + } + + if (!Pipe->PayloadBuffer || Pipe->PayloadBufferSize < PayloadLength) { + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + PayloadLength); + if (!Pipe->PayloadBuffer) { + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = PayloadLength; + } else { + ZeroMemory(Pipe->PayloadBuffer, PayloadLength); + } + + CopyMemory(Pipe->PayloadBuffer, Payload, PayloadLength); + + Pipe->ResponseHeader.PayloadLength = PayloadLength; + Pipe->ResponseHeader.Flags |= + PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN; + + return S_OK; +} + +static +VOID +PerfectHashServerCompleteBulkRequest( + PPERFECT_HASH_SERVER_BULK_REQUEST Request + ) +{ + ULONG Failed; + ULONG Total; + ULONG Succeeded; + HRESULT Result; + HRESULT FirstFailure; + PPERFECT_HASH_SERVER_BULK_RESULT BulkResult; + PALLOCATOR Allocator; + + if (!ARGUMENT_PRESENT(Request)) { + return; + } + + if (InterlockedCompareExchange(&Request->CompletionSignaled, 1, 0) != 0) { + return; + } + + Failed = (ULONG)Request->FailedWorkItems; + Total = Request->TotalWorkItems; + Succeeded = (Total >= Failed) ? (Total - Failed) : 0; + FirstFailure = (HRESULT)Request->FirstFailure; + + if (Total == 0) { + Result = (FirstFailure != S_OK) ? + FirstFailure : + PH_E_NO_KEYS_FOUND_IN_DIRECTORY; + } else if (Failed == 0) { + Result = PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED; + } else { + Result = PH_E_SERVER_BULK_CREATE_FAILED; + } + + BulkResult = Request->ResultMapping; + if (BulkResult) { + BulkResult->SizeOfStruct = sizeof(*BulkResult); + BulkResult->Version = PERFECT_HASH_SERVER_BULK_RESULT_VERSION; + BulkResult->Result = Result; + BulkResult->Flags = 0; + BulkResult->TotalFiles = Total; + BulkResult->SucceededFiles = Succeeded; + BulkResult->FailedFiles = Failed; + BulkResult->FirstFailure = FirstFailure; + } + + if (Request->CompletionEvent) { + SetEvent(Request->CompletionEvent); + } + + if (Request->ResultMapping) { + UnmapViewOfFile(Request->ResultMapping); + } + + if (Request->ResultMappingHandle) { + CloseHandle(Request->ResultMappingHandle); + } + + if (Request->CompletionEvent) { + CloseHandle(Request->CompletionEvent); + } + + if (Request->ParseContext) { + HRESULT CleanupResult; + + CleanupResult = CleanupTableCreateParameters( + &Request->TableCreateParameters + ); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + } + + Request->ParseContext->Vtbl->Release(Request->ParseContext); + } + + if (Request->ArgvW) { + LocalFree(Request->ArgvW); + Request->ArgvW = NULL; + } + + Allocator = Request->Server ? Request->Server->Allocator : NULL; + if (Allocator) { + if (Request->NodeOutstandingCounts) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->NodeOutstandingCounts + ); + } + + if (Request->BaseOutputDirectoryBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->BaseOutputDirectoryBuffer + ); + } + + if (Request->CommandLineBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->CommandLineBuffer + ); + } + + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Request); + } +} + +static +VOID +PerfectHashServerLogBulkCreateException( + ULONG Stage, + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + struct _EXCEPTION_POINTERS *ExceptionPointers + ) +{ +#ifdef PH_WINDOWS + HANDLE LogHandle; + DWORD BytesWritten; + ULONG NodeIndex = 0; + ULONG_PTR Address = 0; + DWORD Code = 0; + DWORD ThreadId; + CHAR Buffer[256]; + + if (!ARGUMENT_PRESENT(ExceptionPointers)) { + return; + } + + if (GetEnvironmentVariableW(L"PH_LOG_SERVER_CRASH", NULL, 0) == 0) { + return; + } + + if (WorkItem) { + NodeIndex = WorkItem->NodeIndex; + } + + LogHandle = CreateFileW(L"PerfectHashServerBulkCreateCrash.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + ThreadId = GetCurrentThreadId(); + Code = ExceptionPointers->ExceptionRecord->ExceptionCode; + Address = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress; + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Code=0x%08lX Address=0x%p ThreadId=%lu Node=%lu\r\n", + Stage, + Code, + (PVOID)Address, + ThreadId, + NodeIndex); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(WorkItem); + UNREFERENCED_PARAMETER(ExceptionPointers); +#endif +} + +static +LONG +PerfectHashServerBulkCreateExceptionFilter( + ULONG Stage, + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + struct _EXCEPTION_POINTERS *ExceptionPointers + ) +{ + PerfectHashServerLogBulkCreateException(Stage, + WorkItem, + ExceptionPointers); + return EXCEPTION_EXECUTE_HANDLER; +} + +static +HRESULT +PerfectHashServerBulkCreateWorkItemCallback( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG_PTR CompletionKey, + LPOVERLAPPED Overlapped, + DWORD NumberOfBytesTransferred, + BOOL Success + ) +{ + HRESULT Result; + HRESULT Failure; + LONG NodeOutstanding; + PALLOCATOR Allocator = NULL; + PPERFECT_HASH_CONTEXT Context = NULL; + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem; + PPERFECT_HASH_SERVER_BULK_REQUEST Request; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext = { 0 }; + ULONG Stage = 0; + BOOLEAN ExceptionRaised = FALSE; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(NumberOfBytesTransferred); + + if (!Success) { + Failure = HRESULT_FROM_WIN32(GetLastError()); + } else { + Failure = S_OK; + } + + WorkItem = CONTAINING_RECORD(Overlapped, + PERFECT_HASH_SERVER_BULK_WORK_ITEM, + Iocp.Overlapped); + Request = WorkItem->Request; + Allocator = (Request && Request->Server) ? Request->Server->Allocator : NULL; + + __try { + Stage = 1; + + if ((ULONG_PTR)WorkItem != CompletionKey) { + Failure = E_UNEXPECTED; + } + + if (FAILED(Failure)) { + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + Stage = 2; + Result = Request->Server->Vtbl->CreateInstance( + Request->Server, + NULL, + &IID_PERFECT_HASH_CONTEXT, + &Context + ); + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + if (Request->CommandLineBuffer) { + Context->CommandLineW = Request->CommandLineBuffer; + } + + if (WorkItem->Node && WorkItem->Node->IoCompletionPort) { + Context->FileWorkIoCompletionPort = WorkItem->Node->IoCompletionPort; + } + + SetContextSkipContextFileWork(Context); + + if (Request->PerFileMaximumConcurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency( + Context, + Request->PerFileMaximumConcurrency + ); + if (FAILED(Result)) { + if (Result == PH_E_SET_MAXIMUM_CONCURRENCY_FAILED && + GetLastError() == ERROR_ACCESS_DENIED) { + // + // Ignore access-denied threadpool sizing failures; proceed + // with the default concurrency configured by initialization. + // + Result = S_OK; + } + + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + } + } + + Stage = 3; + Result = PerfectHashContextInitializeFunctionHookCallbackDll( + Context, + &Request->TableCreateFlags, + &Request->TableCreateParameters + ); + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + PerfectHashContextApplyThreadpoolPriorities( + Context, + &Request->TableCreateParameters + ); + + Stage = 4; + Result = PerfectHashContextInitializeRng( + Context, + &Request->TableCreateFlags, + &Request->TableCreateParameters + ); + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + goto Complete; + } + + ContextTableCreateFlags = Request->ContextTableCreateFlags; + + Stage = 5; + TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + TlsContext->Context = Context; + + Stage = 6; + Result = Context->Vtbl->TableCreate( + Context, + &WorkItem->KeysPath, + &Request->BaseOutputDirectory, + Request->AlgorithmId, + Request->HashFunctionId, + Request->MaskFunctionId, + &ContextTableCreateFlags, + &Request->KeysLoadFlags, + &Request->TableCreateFlags, + &Request->TableCompileFlags, + &Request->TableCreateParameters + ); + + Stage = 7; + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + + if (FAILED(Result)) { + Failure = Result; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + } + + } __except (PerfectHashServerBulkCreateExceptionFilter( + Stage, + WorkItem, + GetExceptionInformation())) { + ExceptionRaised = TRUE; + } + + if (ExceptionRaised) { + Failure = E_UNEXPECTED; + if (Stage >= 6) { + __try { + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + } __except (EXCEPTION_EXECUTE_HANDLER) { + NOTHING; + } + } + if (Request) { + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Failure, + (LONG)S_OK); + } + } + +Complete: + + if (Context) { + Context->Vtbl->Release(Context); + } + + if (WorkItem->KeysPathBuffer && Request->Server && + Request->Server->Allocator) { + Request->Server->Allocator->Vtbl->FreePointer( + Request->Server->Allocator, + (PVOID *)&WorkItem->KeysPathBuffer + ); + } + + if (Request->NodeOutstandingCounts) { + NodeOutstanding = InterlockedDecrement( + &Request->NodeOutstandingCounts[WorkItem->NodeIndex] + ); + if (NodeOutstanding == 0) { + InterlockedDecrement(&Request->PendingNodes); + } + } + + if (InterlockedDecrement(&Request->OutstandingWorkItems) == 0 && + Request->DispatchComplete != 0) { + PerfectHashServerCompleteBulkRequest(Request); + Request = NULL; + } + + if (Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + } + + return S_OK; +} + +static +HRESULT +PerfectHashServerEnqueueBulkRequest( + PPERFECT_HASH_SERVER_BULK_REQUEST Request, + PUNICODE_STRING KeysDirectory + ) +{ + BOOL Success; + HRESULT Result = S_OK; + DWORD LastError; + ULONG NodeIndex; + ULONG BytesToAllocate; + ULONG WildcardBytes; + ULONG KeysPathBytes; + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + PWSTR WildcardBuffer = NULL; + PWSTR KeysPathBuffer = NULL; + BOOLEAN DispatchInitialized = FALSE; + PWSTR Dest; + UNICODE_STRING WildcardPath; + WIN32_FIND_DATAW FindData; + HANDLE FindHandle = NULL; + + if (!ARGUMENT_PRESENT(Request)) { + return E_POINTER; + } + + Request->OutstandingWorkItems = 1; + Request->TotalWorkItems = 0; + Request->DispatchComplete = 0; + DispatchInitialized = TRUE; + + if (!ARGUMENT_PRESENT(KeysDirectory)) { + Result = E_POINTER; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + if (!IsValidMinimumDirectoryUnicodeString(KeysDirectory)) { + Result = E_INVALIDARG; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + Allocator = Request->Server ? Request->Server->Allocator : NULL; + if (!Allocator) { + Result = E_UNEXPECTED; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + Rtl = Request->Server ? Request->Server->Rtl : NULL; + if (!Rtl) { + Result = E_UNEXPECTED; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + BytesToAllocate = KeysDirectory->Length + + sizeof(WCHAR) + + KeysWildcardSuffix.Length + + sizeof(WCHAR); + if (BytesToAllocate < KeysDirectory->Length) { + Result = E_INVALIDARG; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + WildcardBuffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, + 1, + BytesToAllocate); + if (!WildcardBuffer) { + Result = E_OUTOFMEMORY; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + CopyMemory(WildcardBuffer, + KeysDirectory->Buffer, + KeysDirectory->Length); + + Dest = (PWSTR)RtlOffsetToPointer(WildcardBuffer, + KeysDirectory->Length); + *Dest++ = PATHSEP; + CopyMemory(Dest, KeysWildcardSuffix.Buffer, KeysWildcardSuffix.Length); + + WildcardBytes = BytesToAllocate - sizeof(WCHAR); + WildcardPath.Buffer = WildcardBuffer; + WildcardPath.Length = (USHORT)WildcardBytes; + WildcardPath.MaximumLength = (USHORT)BytesToAllocate; + + FindHandle = FindFirstFileW(WildcardPath.Buffer, &FindData); + if (!IsValidHandle(FindHandle)) { + LastError = GetLastError(); + if (LastError == ERROR_FILE_NOT_FOUND) { + Result = PH_E_NO_KEYS_FOUND_IN_DIRECTORY; + } else { + Result = HRESULT_FROM_WIN32(LastError); + } + if (Result != PH_E_NO_KEYS_FOUND_IN_DIRECTORY) { + InterlockedIncrement(&Request->FailedWorkItems); + } + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + goto End; + } + + NodeIndex = Request->NextNodeIndex; + + do { + ULONG FileNameLength; + PPERFECT_HASH_IOCP_NODE Node; + PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem; + LONG NodeOutstanding; + + if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + + FileNameLength = (ULONG)wcslen(FindData.cFileName) * sizeof(WCHAR); + + KeysPathBytes = KeysDirectory->Length + + sizeof(WCHAR) + + FileNameLength + + sizeof(WCHAR); + if (KeysPathBytes < KeysDirectory->Length || + KeysPathBytes > (ULONG)USHRT_MAX) { + Result = E_INVALIDARG; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + break; + } + + WorkItem = (PPERFECT_HASH_SERVER_BULK_WORK_ITEM)( + Allocator->Vtbl->Calloc(Allocator, 1, sizeof(*WorkItem)) + ); + if (!WorkItem) { + Result = E_OUTOFMEMORY; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + break; + } + + KeysPathBuffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, + 1, + KeysPathBytes); + if (!KeysPathBuffer) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + Result = E_OUTOFMEMORY; + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + break; + } + + CopyMemory(KeysPathBuffer, + KeysDirectory->Buffer, + KeysDirectory->Length); + + Dest = (PWSTR)RtlOffsetToPointer(KeysPathBuffer, + KeysDirectory->Length); + *Dest++ = PATHSEP; + CopyMemory(Dest, FindData.cFileName, FileNameLength); + + WorkItem->Request = Request; + WorkItem->NodeIndex = NodeIndex; + WorkItem->Node = &Request->Nodes[NodeIndex]; + WorkItem->KeysPathBuffer = KeysPathBuffer; + WorkItem->KeysPath.Buffer = KeysPathBuffer; + WorkItem->KeysPath.Length = (USHORT)(KeysPathBytes - sizeof(WCHAR)); + WorkItem->KeysPath.MaximumLength = (USHORT)KeysPathBytes; + WorkItem->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + WorkItem->Iocp.Flags = PH_IOCP_WORK_FLAG_BULK; + WorkItem->Iocp.CompletionCallback = + PerfectHashServerBulkCreateWorkItemCallback; + WorkItem->Iocp.CompletionContext = WorkItem; + + Node = WorkItem->Node; + InterlockedIncrement(&Request->OutstandingWorkItems); + Request->TotalWorkItems++; + NodeOutstanding = InterlockedIncrement( + &Request->NodeOutstandingCounts[NodeIndex] + ); + if (NodeOutstanding == 1) { + InterlockedIncrement(&Request->PendingNodes); + } + + Success = PostQueuedCompletionStatus(Node->IoCompletionPort, + 0, + (ULONG_PTR)WorkItem, + &WorkItem->Iocp.Overlapped); + if (!Success) { + Result = HRESULT_FROM_WIN32(GetLastError()); + + InterlockedDecrement(&Request->OutstandingWorkItems); + Request->TotalWorkItems--; + NodeOutstanding = InterlockedDecrement( + &Request->NodeOutstandingCounts[NodeIndex] + ); + if (NodeOutstanding == 0) { + InterlockedDecrement(&Request->PendingNodes); + } + + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + + if (WorkItem->KeysPathBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&WorkItem->KeysPathBuffer); + } + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + } + + NodeIndex++; + if (NodeIndex >= Request->NodeCount) { + NodeIndex = 0; + } + + } while (FindNextFileW(FindHandle, &FindData)); + + if (Result == S_OK) { + LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) { + Result = HRESULT_FROM_WIN32(LastError); + InterlockedIncrement(&Request->FailedWorkItems); + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + } + } + + Request->NextNodeIndex = NodeIndex; + + if (Request->TotalWorkItems == 0 && Result == S_OK) { + Result = PH_E_NO_KEYS_FOUND_IN_DIRECTORY; + InterlockedCompareExchange(&Request->FirstFailure, + (LONG)Result, + (LONG)S_OK); + } + +End: + + if (FindHandle && FindHandle != INVALID_HANDLE_VALUE) { + FindClose(FindHandle); + } + + if (DispatchInitialized) { + Request->DispatchComplete = 1; + if (InterlockedDecrement(&Request->OutstandingWorkItems) == 0) { + PerfectHashServerCompleteBulkRequest(Request); + } + } + + if (WildcardBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WildcardBuffer); + } + + return Result; +} + +static +HRESULT +PerfectHashServerDispatchBulkCreateDirectoryRequest( + PPERFECT_HASH_SERVER_PIPE Pipe, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLine, + PBOOLEAN ArgvWOwned + ) +{ + BOOL Success; + HRESULT Result; + HRESULT CleanupResult; + ULONG BytesToAllocate; + ULONG CommandLineChars; + ULONG CommandLineBytes; + DWORD ClientProcessId = 0; + HANDLE ClientProcess = NULL; + HANDLE EventHandle = NULL; + HANDLE ResultHandle = NULL; + HANDLE ClientEventHandle = NULL; + HANDLE ClientResultHandle = NULL; + ULONG AdjustedArguments; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_CONTEXT ParseContext = NULL; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + PPERFECT_HASH_SERVER_BULK_REQUEST Request = NULL; + PPERFECT_HASH_SERVER_BULK_RESULT BulkResult = NULL; + UNICODE_STRING KeysDirectory = { 0 }; + UNICODE_STRING BaseOutputDirectory = { 0 }; + PERFECT_HASH_ALGORITHM_ID AlgorithmId = 0; + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId = 0; + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId = 0; + ULONG MaximumConcurrency = 0; + PERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags = { 0 }; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags = { 0 }; + PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags = { 0 }; + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters = { 0 }; + LPWSTR *AdjustedArgvW = NULL; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Server = Pipe->Server; + if (!Server) { + return E_UNEXPECTED; + } + + ContextIocp = Server->ContextIocp; + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!ContextIocp || !Allocator || !Rtl) { + return E_UNEXPECTED; + } + + if (!ARGUMENT_PRESENT(ArgvWOwned)) { + return E_POINTER; + } + + *ArgvWOwned = FALSE; + + Result = Server->Vtbl->CreateInstance(Server, + NULL, + &IID_PERFECT_HASH_CONTEXT, + &ParseContext); + if (FAILED(Result)) { + return Result; + } + + Result = LoadDefaultTableCreateFlags(&TableCreateFlags); + if (FAILED(Result)) { + goto Error; + } + + TableCreateParameters.SizeOfStruct = sizeof(TableCreateParameters); + TableCreateParameters.Allocator = ParseContext->Allocator; + + AdjustedArguments = NumberOfArguments + 1; + AdjustedArgvW = Allocator->Vtbl->Calloc(Allocator, + AdjustedArguments, + sizeof(*AdjustedArgvW)); + if (!AdjustedArgvW) { + Result = E_OUTOFMEMORY; + goto Error; + } + + AdjustedArgvW[0] = L"PerfectHashBulkCreate"; + CopyMemory(&AdjustedArgvW[1], + ArgvW, + NumberOfArguments * sizeof(*AdjustedArgvW)); + + Result = ParseContext->Vtbl->ExtractBulkCreateArgsFromArgvW( + ParseContext, + AdjustedArguments, + AdjustedArgvW, + CommandLine, + &KeysDirectory, + &BaseOutputDirectory, + &AlgorithmId, + &HashFunctionId, + &MaskFunctionId, + &MaximumConcurrency, + &ContextBulkCreateFlags, + &KeysLoadFlags, + &TableCreateFlags, + &TableCompileFlags, + &TableCreateParameters + ); + if (FAILED(Result)) { + goto Error; + } + + ContextTableCreateFlags.AsULong = 0; + ContextTableCreateFlags.SkipTestAfterCreate = + ContextBulkCreateFlags.SkipTestAfterCreate; + ContextTableCreateFlags.Compile = ContextBulkCreateFlags.Compile; + ContextTableCreateFlags.MonitorLowMemory = + ContextBulkCreateFlags.MonitorLowMemory; + + // + // Suppress per-file CSV output for async bulk operations. + // + + TableCreateFlags.DisableCsvOutputFile = TRUE; + + Request = (PPERFECT_HASH_SERVER_BULK_REQUEST)( + Allocator->Vtbl->Calloc(Allocator, 1, sizeof(*Request)) + ); + if (!Request) { + Result = E_OUTOFMEMORY; + goto Error; + } + + Request->SizeOfStruct = sizeof(*Request); + Request->Server = Server; + Request->ParseContext = ParseContext; + Request->ContextBulkCreateFlags = ContextBulkCreateFlags; + Request->ContextTableCreateFlags = ContextTableCreateFlags; + Request->KeysLoadFlags = KeysLoadFlags; + Request->TableCreateFlags = TableCreateFlags; + Request->TableCompileFlags = TableCompileFlags; + Request->TableCreateParameters = TableCreateParameters; + Request->AlgorithmId = AlgorithmId; + Request->HashFunctionId = HashFunctionId; + Request->MaskFunctionId = MaskFunctionId; + Request->PerFileMaximumConcurrency = MaximumConcurrency; + Request->FirstFailure = (LONG)S_OK; + Request->ArgvW = ArgvW; + *ArgvWOwned = TRUE; + + CommandLineChars = (ULONG)wcslen(CommandLine); + if (CommandLineChars > ((ULONG_MAX / sizeof(WCHAR)) - 1)) { + Result = E_INVALIDARG; + goto Error; + } + + CommandLineBytes = (CommandLineChars + 1) * sizeof(WCHAR); + Request->CommandLineBuffer = (PWSTR)( + Allocator->Vtbl->Calloc(Allocator, 1, CommandLineBytes) + ); + if (!Request->CommandLineBuffer) { + Result = E_OUTOFMEMORY; + goto Error; + } + + CopyMemory(Request->CommandLineBuffer, + CommandLine, + CommandLineBytes); + + Request->Nodes = ContextIocp->Nodes; + Request->NodeCount = ContextIocp->NodeCount; + + if (!Request->Nodes || Request->NodeCount == 0) { + Result = E_UNEXPECTED; + goto Error; + } + + Request->NodeOutstandingCounts = (PLONG)( + Allocator->Vtbl->Calloc(Allocator, + Request->NodeCount, + sizeof(*Request->NodeOutstandingCounts)) + ); + if (!Request->NodeOutstandingCounts) { + Result = E_OUTOFMEMORY; + goto Error; + } + + BytesToAllocate = BaseOutputDirectory.Length + sizeof(WCHAR); + if (BytesToAllocate < BaseOutputDirectory.Length || + BytesToAllocate > (ULONG)USHRT_MAX) { + Result = E_INVALIDARG; + goto Error; + } + + Request->BaseOutputDirectoryBuffer = (PWSTR)( + Allocator->Vtbl->Calloc(Allocator, 1, BytesToAllocate) + ); + if (!Request->BaseOutputDirectoryBuffer) { + Result = E_OUTOFMEMORY; + goto Error; + } + + CopyMemory(Request->BaseOutputDirectoryBuffer, + BaseOutputDirectory.Buffer, + BaseOutputDirectory.Length); + + Request->BaseOutputDirectory.Buffer = Request->BaseOutputDirectoryBuffer; + Request->BaseOutputDirectory.Length = BaseOutputDirectory.Length; + Request->BaseOutputDirectory.MaximumLength = (USHORT)BytesToAllocate; + + EventHandle = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!EventHandle) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + Request->CompletionEvent = EventHandle; + + ResultHandle = CreateFileMappingW(INVALID_HANDLE_VALUE, + NULL, + PAGE_READWRITE, + 0, + sizeof(PERFECT_HASH_SERVER_BULK_RESULT), + NULL); + if (!ResultHandle) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + Request->ResultMappingHandle = ResultHandle; + + BulkResult = (PPERFECT_HASH_SERVER_BULK_RESULT)( + MapViewOfFile(ResultHandle, FILE_MAP_WRITE, 0, 0, 0) + ); + if (!BulkResult) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + ZeroMemory(BulkResult, sizeof(*BulkResult)); + Request->ResultMapping = BulkResult; + + Success = GetNamedPipeClientProcessId(Pipe->Pipe, &ClientProcessId); + if (!Success) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + ClientProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, ClientProcessId); + if (!ClientProcess) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Success = DuplicateHandle(GetCurrentProcess(), + EventHandle, + ClientProcess, + &ClientEventHandle, + 0, + FALSE, + DUPLICATE_SAME_ACCESS); + if (!Success) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + Success = DuplicateHandle(GetCurrentProcess(), + ResultHandle, + ClientProcess, + &ClientResultHandle, + 0, + FALSE, + DUPLICATE_SAME_ACCESS); + if (!Success) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + + CloseHandle(ClientProcess); + ClientProcess = NULL; + + Result = PerfectHashServerPrepareBulkCreateDirectoryPayload( + Pipe, + ClientEventHandle, + ClientResultHandle + ); + if (FAILED(Result)) { + goto Error; + } + + Result = PerfectHashServerEnqueueBulkRequest(Request, &KeysDirectory); + if (FAILED(Result)) { + // + // Enqueue failures are recorded and signaled by the enqueue routine. + // + } + + if (AdjustedArgvW) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); + } + + return S_OK; + +Error: + + if (AdjustedArgvW && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); + } + + if (ClientProcess) { + CloseHandle(ClientProcess); + } + + if (BulkResult) { + UnmapViewOfFile(BulkResult); + } + + if (ResultHandle) { + CloseHandle(ResultHandle); + } + + if (EventHandle) { + CloseHandle(EventHandle); + } + + if (Request) { + if (Request->ArgvW) { + LocalFree(Request->ArgvW); + Request->ArgvW = NULL; + } + + if (Request->NodeOutstandingCounts) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->NodeOutstandingCounts + ); + } + + if (Request->BaseOutputDirectoryBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->BaseOutputDirectoryBuffer + ); + } + + if (Request->CommandLineBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Request->CommandLineBuffer + ); + } + + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Request); + } + + if (ParseContext) { + CleanupResult = CleanupTableCreateParameters(&TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + Result = CleanupResult; + } + + ParseContext->Vtbl->Release(ParseContext); + } + + return Result; +} + +static +HRESULT +PerfectHashServerDispatchRequest( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + HRESULT Result; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + PALLOCATOR Allocator; + LPWSTR CommandLine; + LPWSTR *ArgvW; + LPWSTR *AdjustedArgvW; + LPWSTR *ArgvToUse; + ULONG NumberOfArguments; + ULONG AdjustedArguments; + PRTL Rtl; + BOOLEAN ArgvWOwned; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Rtl) { + return E_UNEXPECTED; + } + + ContextIocp = Pipe->Server ? Pipe->Server->ContextIocp : NULL; + RequestType = Pipe->RequestHeader.RequestType; + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + + ZeroMemory(&Pipe->ResponseHeader, sizeof(Pipe->ResponseHeader)); + Pipe->ResponseHeader.SizeOfStruct = + sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER); + Pipe->ResponseHeader.Version = PERFECT_HASH_SERVER_MESSAGE_VERSION; + Pipe->ResponseHeader.RequestId = Pipe->RequestHeader.RequestId; + Pipe->ResponseHeader.PayloadLength = 0; + + ArgvW = NULL; + AdjustedArgvW = NULL; + ArgvToUse = NULL; + NumberOfArguments = 0; + AdjustedArguments = 0; + CommandLine = NULL; + ArgvWOwned = FALSE; + + switch (RequestType) { + case PerfectHashNullServerRequestType: + case PerfectHashInvalidServerRequestType: + Result = E_INVALIDARG; + break; + + case PerfectHashShutdownServerRequestType: + Result = S_OK; + Pipe->ShutdownAfterSend = TRUE; + break; + + case PerfectHashTableCreateServerRequestType: + case PerfectHashBulkCreateServerRequestType: + case PerfectHashBulkCreateDirectoryServerRequestType: + if (!ContextIocp || !Allocator) { + Result = E_UNEXPECTED; + break; + } + + if (Pipe->PayloadLength == 0 || + (Pipe->PayloadLength % sizeof(WCHAR)) != 0) { + Result = E_INVALIDARG; + break; + } + + if (!Pipe->PayloadBuffer) { + Result = E_UNEXPECTED; + break; + } + + CommandLine = (LPWSTR)Pipe->PayloadBuffer; + if (CommandLine[0] == L'\0') { + Result = E_INVALIDARG; + break; + } + + ArgvW = CommandLineToArgvW(CommandLine, + (PINT)&NumberOfArguments); + if (!ArgvW || NumberOfArguments == 0) { + Result = HRESULT_FROM_WIN32(GetLastError()); + if (Result == S_OK) { + Result = E_INVALIDARG; + } + break; + } + + ArgvToUse = ArgvW; + AdjustedArguments = NumberOfArguments; + + if (RequestType != + PerfectHashBulkCreateDirectoryServerRequestType && + (ArgvW[0][0] == L'-' || ArgvW[0][0] == L'/')) { + AdjustedArgvW = Allocator->Vtbl->Calloc( + Allocator, + NumberOfArguments + 1, + sizeof(*AdjustedArgvW) + ); + if (!AdjustedArgvW) { + Result = E_OUTOFMEMORY; + break; + } + + AdjustedArgvW[0] = L"PerfectHashServer"; + CopyMemory(&AdjustedArgvW[1], + ArgvW, + NumberOfArguments * sizeof(*AdjustedArgvW)); + ArgvToUse = AdjustedArgvW; + AdjustedArguments = NumberOfArguments + 1; + } + + if (RequestType == PerfectHashTableCreateServerRequestType) { + Result = ContextIocp->Vtbl->TableCreateArgvW( + ContextIocp, + AdjustedArguments, + ArgvToUse, + CommandLine + ); + } else if (RequestType == + PerfectHashBulkCreateDirectoryServerRequestType) { + Result = PerfectHashServerDispatchBulkCreateDirectoryRequest( + Pipe, + NumberOfArguments, + ArgvW, + CommandLine, + &ArgvWOwned + ); + } else { + Result = ContextIocp->Vtbl->BulkCreateArgvW( + ContextIocp, + AdjustedArguments, + ArgvToUse, + CommandLine + ); + } + + break; + + default: + Result = E_INVALIDARG; + break; + } + + if (AdjustedArgvW && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); + } + + if (ArgvW && !ArgvWOwned) { + LocalFree(ArgvW); + } + + Pipe->ResponseHeader.Result = Result; + if (FAILED(Result)) { + HRESULT PayloadResult; + + PayloadResult = PerfectHashServerPrepareErrorPayload(Pipe, Result); + if (FAILED(PayloadResult)) { + Pipe->ResponseHeader.PayloadLength = 0; + Pipe->ResponseHeader.Flags = 0; + } + } + Pipe->BytesTransferred = 0; + + return PerfectHashServerIssueWriteResponseHeader(Pipe); +} + +static +HRESULT +PerfectHashServerIocpCompletionCallback( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG_PTR CompletionKey, + LPOVERLAPPED Overlapped, + DWORD NumberOfBytesTransferred, + BOOL Success + ) +{ + HRESULT Result; + DWORD LastError; + ULONG Expected; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_SERVER_PIPE Pipe; + + UNREFERENCED_PARAMETER(ContextIocp); + + if (!Overlapped) { + return E_POINTER; + } + + Pipe = CONTAINING_RECORD(Overlapped, + PERFECT_HASH_SERVER_PIPE, + Iocp.Overlapped); + Server = Pipe->Server; + + if ((ULONG_PTR)Pipe != CompletionKey) { + return E_UNEXPECTED; + } + + if (!Success) { + LastError = GetLastError(); + switch (LastError) { + case ERROR_BROKEN_PIPE: + case ERROR_PIPE_NOT_CONNECTED: + case ERROR_NO_DATA: + case ERROR_OPERATION_ABORTED: + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + default: + SYS_ERROR(GetQueuedCompletionStatus); + PerfectHashServerResetPipe(Pipe, TRUE); + return HRESULT_FROM_WIN32(LastError); + } + } + + switch (Pipe->State) { + case PerfectHashServerPipeStateInvalid: + PerfectHashServerResetPipe(Pipe, TRUE); + return E_UNEXPECTED; + + case PerfectHashServerPipeStateAccepting: + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueReadHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + + case PerfectHashServerPipeStateReadingHeader: + if (NumberOfBytesTransferred == 0) { + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + } + + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = sizeof(PERFECT_HASH_SERVER_REQUEST_HEADER); + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueReadHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->RequestHeader.SizeOfStruct != Expected || + Pipe->RequestHeader.Version != + PERFECT_HASH_SERVER_MESSAGE_VERSION || + !IsValidPerfectHashServerRequestType( + Pipe->RequestHeader.RequestType) || + Pipe->RequestHeader.PayloadLength > + PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + Pipe->ResponseHeader.SizeOfStruct = + sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER); + Pipe->ResponseHeader.Version = + PERFECT_HASH_SERVER_MESSAGE_VERSION; + Pipe->ResponseHeader.RequestId = + Pipe->RequestHeader.RequestId; + Pipe->ResponseHeader.Result = E_INVALIDARG; + Pipe->ResponseHeader.PayloadLength = 0; + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueWriteResponseHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + Pipe->PayloadLength = Pipe->RequestHeader.PayloadLength; + + if (Pipe->PayloadLength == 0) { + Result = PerfectHashServerDispatchRequest(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (!Pipe->PayloadBuffer || + Pipe->PayloadBufferSize < Pipe->PayloadLength) { + SIZE_T AllocationSize; + PALLOCATOR Allocator; + + Allocator = Server ? Server->Allocator : NULL; + if (!Allocator) { + PerfectHashServerResetPipe(Pipe, TRUE); + return E_UNEXPECTED; + } + + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer( + Allocator, + &Pipe->PayloadBuffer + ); + } + + AllocationSize = (SIZE_T)Pipe->PayloadLength + + sizeof(WCHAR); + if (AllocationSize < Pipe->PayloadLength) { + PerfectHashServerResetPipe(Pipe, TRUE); + return E_INVALIDARG; + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc( + Allocator, + 1, + AllocationSize + ); + if (!Pipe->PayloadBuffer) { + PerfectHashServerResetPipe(Pipe, TRUE); + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = (ULONG)AllocationSize; + } + + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueReadPayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + + case PerfectHashServerPipeStateReadingPayload: + if (NumberOfBytesTransferred == 0) { + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + } + + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = Pipe->PayloadLength; + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueReadPayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + Result = PerfectHashServerDispatchRequest(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + + case PerfectHashServerPipeStateWritingResponseHeader: + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = sizeof(PERFECT_HASH_SERVER_RESPONSE_HEADER); + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueWriteResponseHeader(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->ResponseHeader.PayloadLength > 0) { + Pipe->BytesTransferred = 0; + Result = PerfectHashServerIssueWriteResponsePayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->ShutdownAfterSend && Server) { + if (Server->ShutdownEvent) { + SetEvent(Server->ShutdownEvent); + } + } + + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + + case PerfectHashServerPipeStateWritingResponsePayload: + Pipe->BytesTransferred += NumberOfBytesTransferred; + Expected = Pipe->ResponseHeader.PayloadLength; + + if (Pipe->BytesTransferred < Expected) { + Result = PerfectHashServerIssueWriteResponsePayload(Pipe); + if (FAILED(Result)) { + PerfectHashServerResetPipe(Pipe, TRUE); + } + return Result; + } + + if (Pipe->ShutdownAfterSend && Server) { + if (Server->ShutdownEvent) { + SetEvent(Server->ShutdownEvent); + } + } + + PerfectHashServerResetPipe(Pipe, TRUE); + return S_OK; + + default: + PerfectHashServerResetPipe(Pipe, TRUE); + return E_UNEXPECTED; + } +} + +#endif // PH_WINDOWS + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashServer.h b/src/PerfectHash/PerfectHashServer.h new file mode 100644 index 00000000..c231f090 --- /dev/null +++ b/src/PerfectHash/PerfectHashServer.h @@ -0,0 +1,135 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashServer.h + +Abstract: + + This is the private header file for the PERFECT_HASH_SERVER component of + the perfect hash library. It defines the structure, and function pointer + typedefs for private non-vtbl members. + +--*/ + +#pragma once + +#include "stdafx.h" + +typedef union _PERFECT_HASH_SERVER_STATE { + struct { + + ULONG Initialized:1; + ULONG Running:1; + ULONG Stopping:1; + ULONG Stopped:1; + + ULONG Unused:28; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_SERVER_STATE; +C_ASSERT(sizeof(PERFECT_HASH_SERVER_STATE) == sizeof(ULONG)); +typedef PERFECT_HASH_SERVER_STATE *PPERFECT_HASH_SERVER_STATE; + +typedef union _PERFECT_HASH_SERVER_FLAGS { + struct { + + // + // When set, indicates the server should only accept local clients. + // + + ULONG LocalOnly:1; + ULONG EndpointAllocated:1; + + ULONG Unused:30; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_SERVER_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_SERVER_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_SERVER_FLAGS *PPERFECT_HASH_SERVER_FLAGS; + +struct _PERFECT_HASH_SERVER_PIPE; +typedef struct _PERFECT_HASH_SERVER_PIPE *PPERFECT_HASH_SERVER_PIPE; + +typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_SERVER { + + COMMON_COMPONENT_HEADER(PERFECT_HASH_SERVER); + + ULONG MaximumConcurrency; + ULONG NumaNodeCount; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; + ULONG Padding1; + ULONG Padding2; + + // + // The underlying IOCP context used by the server. + // + + struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp; + + // + // Named pipe endpoint (if applicable). + // + + UNICODE_STRING Endpoint; + + // + // Pipe instances. + // + + PPERFECT_HASH_SERVER_PIPE Pipes; + ULONG PipeCount; + ULONG Padding3; + + HANDLE ShutdownEvent; + HANDLE StartedEvent; + + // + // Backing vtbl. + // + + PERFECT_HASH_SERVER_VTBL Interface; + +} PERFECT_HASH_SERVER; +typedef PERFECT_HASH_SERVER *PPERFECT_HASH_SERVER; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_SERVER_INITIALIZE)( + _In_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_INITIALIZE *PPERFECT_HASH_SERVER_INITIALIZE; + +typedef +VOID +(NTAPI PERFECT_HASH_SERVER_RUNDOWN)( + _In_ _Post_ptr_invalid_ PPERFECT_HASH_SERVER Server + ); +typedef PERFECT_HASH_SERVER_RUNDOWN *PPERFECT_HASH_SERVER_RUNDOWN; + +extern PERFECT_HASH_SERVER_INITIALIZE PerfectHashServerInitialize; +extern PERFECT_HASH_SERVER_RUNDOWN PerfectHashServerRundown; + +extern PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY + PerfectHashServerSetMaximumConcurrency; +extern PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY + PerfectHashServerGetMaximumConcurrency; +extern PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK + PerfectHashServerSetNumaNodeMask; +extern PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK + PerfectHashServerGetNumaNodeMask; +extern PERFECT_HASH_SERVER_SET_ENDPOINT PerfectHashServerSetEndpoint; +extern PERFECT_HASH_SERVER_GET_ENDPOINT PerfectHashServerGetEndpoint; +extern PERFECT_HASH_SERVER_SET_LOCAL_ONLY PerfectHashServerSetLocalOnly; +extern PERFECT_HASH_SERVER_GET_LOCAL_ONLY PerfectHashServerGetLocalOnly; +extern PERFECT_HASH_SERVER_START PerfectHashServerStart; +extern PERFECT_HASH_SERVER_STOP PerfectHashServerStop; +extern PERFECT_HASH_SERVER_WAIT PerfectHashServerWait; +extern PERFECT_HASH_SERVER_SUBMIT_REQUEST PerfectHashServerSubmitRequest; + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashTable.c b/src/PerfectHash/PerfectHashTable.c index d5d69f1a..4aa00222 100644 --- a/src/PerfectHash/PerfectHashTable.c +++ b/src/PerfectHash/PerfectHashTable.c @@ -152,6 +152,30 @@ Return Value: Allocator = Table->Allocator; + // + // Detach any files still linked to a parent directory. This breaks the + // directory/file reference cycle prior to releasing file objects. + // + +#define EXPAND_AS_DETACH_FILE( \ + Verb, VUpper, Name, Upper, \ + EofType, EofValue, \ + Suffix, Extension, Stream, Base \ +) \ + if (Table->Name && Table->Name->ParentDirectory) { \ + RemoveResult = Table->Name->ParentDirectory->Vtbl->RemoveFile( \ + Table->Name->ParentDirectory, \ + Table->Name \ + ); \ + if (FAILED(RemoveResult)) { \ + PH_ERROR(PerfectHashDirectoryRemoveFile, RemoveResult); \ + } \ + } + + FILE_WORK_TABLE_ENTRY(EXPAND_AS_DETACH_FILE); + +#undef EXPAND_AS_DETACH_FILE + // // Free the memory used for the values array, if applicable. // diff --git a/src/PerfectHash/PerfectHashTableCreate.c b/src/PerfectHash/PerfectHashTableCreate.c index 4f41ce37..f7797359 100644 --- a/src/PerfectHash/PerfectHashTableCreate.c +++ b/src/PerfectHash/PerfectHashTableCreate.c @@ -16,6 +16,36 @@ Module Name: --*/ #include "stdafx.h" +#ifdef PH_WINDOWS +#include "Chm01Async.h" +#endif + +// +// Async gating. +// + +#ifdef PH_WINDOWS +static +BOOLEAN +IsChm01AsyncEnabled( + VOID + ) +{ + static LONG Initialized = 0; + static BOOLEAN Enabled = FALSE; + WCHAR Buffer[32]; + DWORD Length; + + if (InterlockedCompareExchange(&Initialized, 1, 0) == 0) { + Length = GetEnvironmentVariableW(L"PERFECT_HASH_IOCP_ASYNC_CHM01", + Buffer, + ARRAYSIZE(Buffer)); + Enabled = (Length > 0); + } + + return Enabled; +} +#endif // // Cap the maximum key set size we're willing to process to 2 billion. @@ -393,7 +423,21 @@ Return Value: // Dispatch remaining creation work to the algorithm-specific routine. // +#ifdef PH_WINDOWS + if (AlgorithmId == PerfectHashChm01AlgorithmId && + Table->Context && + Table->Context->FileWorkIoCompletionPort && + IsChm01AsyncEnabled()) { + Result = CreatePerfectHashTableImplChm01Async( + Table, + Table->Context->FileWorkIoCompletionPort + ); + } else { + Result = CreationRoutines[AlgorithmId](Table); + } +#else Result = CreationRoutines[AlgorithmId](Table); +#endif if (Table->OutputDirectory) { CloseResult = Table->OutputDirectory->Vtbl->Close(Table->OutputDirectory); diff --git a/src/PerfectHash/PerfectHashTls.h b/src/PerfectHash/PerfectHashTls.h index 491eca39..149f05a9 100644 --- a/src/PerfectHash/PerfectHashTls.h +++ b/src/PerfectHash/PerfectHashTls.h @@ -49,7 +49,7 @@ typedef union _PERFECT_HASH_TLS_CONTEXT_FLAGS { // Unused bits. (Consume these before the Unused2 bits.) // - ULONG Unused1:3; + ULONG Unused1:6; // // The following bits, when set, prevent the global component logic @@ -88,7 +88,7 @@ typedef union _PERFECT_HASH_TLS_CONTEXT_FLAGS { // Remaining unused bits. (Consume Unused1 before using these.) // - ULONG Unused2:24; + ULONG Unused2:21; }; LONG AsLong; diff --git a/src/PerfectHash/stdafx.h b/src/PerfectHash/stdafx.h index 865334ba..242a3d28 100644 --- a/src/PerfectHash/stdafx.h +++ b/src/PerfectHash/stdafx.h @@ -166,6 +166,12 @@ Module Name: #include "Rng.h" #include "GraphCu.h" #include "PerfectHashContext.h" +#include "PerfectHashContextIocp.h" +#ifdef PH_WINDOWS +#include "PerfectHashAsync.h" +#endif +#include "PerfectHashServer.h" +#include "PerfectHashClient.h" #include "PerfectHashConstants.h" #include "PerfectHashErrorHandling.h" #include "GraphImpl.h" diff --git a/src/PerfectHashClientExe/CMakeLists.txt b/src/PerfectHashClientExe/CMakeLists.txt new file mode 100644 index 00000000..9f5f2b20 --- /dev/null +++ b/src/PerfectHashClientExe/CMakeLists.txt @@ -0,0 +1,65 @@ +set(target PerfectHashClientExe) + +################################################################################ +# Source groups +################################################################################ +set(Header_Files + "stdafx.h" + "targetver.h" +) +source_group("Header Files" FILES ${Header_Files}) + +set(Source_Files + "PerfectHashClientExe.c" + #"stdafx.c" +) +source_group("Source Files" FILES ${Source_Files}) + +set(ALL_FILES + ${Header_Files} + ${Source_Files} +) + +set(Resource_Files + "PerfectHashClientExe.rc" +) +source_group("Resource Files" FILES ${Resource_Files}) + +if(WIN32) + list( + APPEND ALL_FILES + ${Resource_Files} + ) +endif() + +add_executable(${target} ${ALL_FILES}) + +################################################################################ +# Target +################################################################################ + +perfecthash_apply_common_settings(${target}) + +################################################################################ +# Output executable name +################################################################################ +set_target_properties(${target} PROPERTIES + OUTPUT_NAME "PerfectHashClient" +) +################################################################################ +# Compile definitions +################################################################################ + +################################################################################ +# Compile and link options +################################################################################ +################################################################################ +# Dependencies +################################################################################ +add_dependencies(${target} + PerfectHash +) + +target_link_libraries(${target} PRIVATE PerfectHash ${CMAKE_DL_LIBS}) + +# vim:set ts=8 sw=4 sts=4 tw=80 expandtab syntax=cmake : diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.c b/src/PerfectHashClientExe/PerfectHashClientExe.c new file mode 100644 index 00000000..5491e6ff --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.c @@ -0,0 +1,450 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashClientExe.c + +Abstract: + + This module implements the main entry point for the perfect hash client. + It loads the perfect hash library, creates a client instance, then attempts + to connect and optionally submit a request. + +--*/ + +#include "stdafx.h" +#include + +typedef struct _PERFECT_HASH_CLIENT_CLI_OPTIONS { + UNICODE_STRING Endpoint; + UNICODE_STRING CommandLine; + PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; + BOOLEAN EndpointPresent; + BOOLEAN CommandLinePresent; + BOOLEAN RequestTypePresent; + UCHAR Padding1[9]; +} PERFECT_HASH_CLIENT_CLI_OPTIONS; +typedef PERFECT_HASH_CLIENT_CLI_OPTIONS *PPERFECT_HASH_CLIENT_CLI_OPTIONS; + +static +VOID +PrintUsage( + VOID + ) +{ + wprintf(L"Usage: PerfectHashClient [--Endpoint=] " + L"[--Shutdown|--TableCreate=|--BulkCreate=|" + L"--BulkCreateDirectory=]\n"); + wprintf(L" --TableCreate= PerfectHashCreate-style arguments\n"); + wprintf(L" --BulkCreate= PerfectHashBulkCreate-style arguments\n"); + wprintf(L" --BulkCreateDirectory= BulkCreate args with single token\n"); +} + +static +HRESULT +ParseClientArgs( + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _Inout_ PPERFECT_HASH_CLIENT_CLI_OPTIONS Options + ) +{ + ULONG Index; + + Options->Endpoint.Buffer = NULL; + Options->Endpoint.Length = 0; + Options->Endpoint.MaximumLength = 0; + Options->EndpointPresent = FALSE; + Options->CommandLine.Buffer = NULL; + Options->CommandLine.Length = 0; + Options->CommandLine.MaximumLength = 0; + Options->CommandLinePresent = FALSE; + Options->RequestType = PerfectHashNullServerRequestType; + Options->RequestTypePresent = FALSE; + + for (Index = 1; Index < NumberOfArguments; Index++) { + PCWSTR Arg = ArgvW[Index]; + + if (!Arg) { + continue; + } + + if (Arg[0] != L'-' || Arg[1] != L'-') { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Arg += 2; + + if (_wcsicmp(Arg, L"Help") == 0 || + _wcsicmp(Arg, L"?") == 0) { + PrintUsage(); + return S_FALSE; + } + + if (_wcsnicmp(Arg, L"Endpoint=", 9) == 0) { + PCWSTR Value; + ULONG Length; + + Value = Arg + 9; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->Endpoint.Buffer = (PWSTR)Value; + Options->Endpoint.Length = (USHORT)Length; + Options->Endpoint.MaximumLength = (USHORT)Length + sizeof(WCHAR); + Options->EndpointPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"Shutdown") == 0) { + if (Options->RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + Options->RequestType = PerfectHashShutdownServerRequestType; + Options->RequestTypePresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"TableCreate=", 12) == 0) { + PCWSTR Value; + ULONG Length; + + if (Options->RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Value = Arg + 12; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->CommandLine.Buffer = (PWSTR)Value; + Options->CommandLine.Length = (USHORT)Length; + Options->CommandLine.MaximumLength = (USHORT)Length + + sizeof(WCHAR); + Options->CommandLinePresent = TRUE; + Options->RequestType = PerfectHashTableCreateServerRequestType; + Options->RequestTypePresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"BulkCreate=", 11) == 0) { + PCWSTR Value; + ULONG Length; + + if (Options->RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Value = Arg + 11; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->CommandLine.Buffer = (PWSTR)Value; + Options->CommandLine.Length = (USHORT)Length; + Options->CommandLine.MaximumLength = (USHORT)Length + + sizeof(WCHAR); + Options->CommandLinePresent = TRUE; + Options->RequestType = PerfectHashBulkCreateServerRequestType; + Options->RequestTypePresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"BulkCreateDirectory=", 20) == 0) { + PCWSTR Value; + ULONG Length; + + if (Options->RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Value = Arg + 20; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->CommandLine.Buffer = (PWSTR)Value; + Options->CommandLine.Length = (USHORT)Length; + Options->CommandLine.MaximumLength = (USHORT)Length + + sizeof(WCHAR); + Options->CommandLinePresent = TRUE; + Options->RequestType = + PerfectHashBulkCreateDirectoryServerRequestType; + Options->RequestTypePresent = TRUE; + continue; + } + + return PH_E_INVALID_COMMANDLINE_ARG; + } + + if (Options->RequestTypePresent) { + if (Options->RequestType == PerfectHashShutdownServerRequestType) { + if (Options->CommandLinePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + } else if (!Options->CommandLinePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + } + + return S_OK; +} + +static +HRESULT +PerfectHashClientWaitForBulkCreateToken( + _In_ PPERFECT_HASH_CLIENT Client, + _Out_opt_ PPERFECT_HASH_SERVER_BULK_RESULT BulkResult + ) +{ + HRESULT Result; + DWORD WaitResult; + int Count; + ULONG ResponseFlags; + ULONGLONG EventHandleValue = 0; + ULONGLONG ResultHandleValue = 0; + HANDLE EventHandle = NULL; + HANDLE ResultHandle = NULL; + PPERFECT_HASH_SERVER_BULK_RESULT Mapping = NULL; + UNICODE_STRING ResponsePayload; + + if (!ARGUMENT_PRESENT(Client)) { + return E_POINTER; + } + + if (ARGUMENT_PRESENT(BulkResult)) { + ZeroMemory(BulkResult, sizeof(*BulkResult)); + } + + Result = Client->Vtbl->GetLastResponse(Client, + &ResponsePayload, + &ResponseFlags); + if (FAILED(Result)) { + return Result; + } + + if (!(ResponseFlags & + PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN)) { + return E_UNEXPECTED; + } + + if (!ResponsePayload.Buffer || ResponsePayload.Length == 0) { + return E_UNEXPECTED; + } + + Count = swscanf_s(ResponsePayload.Buffer, + PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT, + &EventHandleValue, + &ResultHandleValue); + if (Count != 2) { + return E_INVALIDARG; + } + + if (EventHandleValue == 0 || ResultHandleValue == 0) { + return E_INVALIDARG; + } + + EventHandle = (HANDLE)(ULONG_PTR)EventHandleValue; + ResultHandle = (HANDLE)(ULONG_PTR)ResultHandleValue; + + WaitResult = WaitForSingleObject(EventHandle, INFINITE); + if (WaitResult != WAIT_OBJECT_0) { + if (WaitResult == WAIT_FAILED) { + Result = HRESULT_FROM_WIN32(GetLastError()); + } else { + Result = HRESULT_FROM_WIN32(WaitResult); + } + goto End; + } + + Mapping = (PPERFECT_HASH_SERVER_BULK_RESULT)( + MapViewOfFile(ResultHandle, FILE_MAP_READ, 0, 0, 0) + ); + if (!Mapping) { + Result = PH_E_SYSTEM_CALL_FAILED; + goto End; + } + + if (Mapping->SizeOfStruct != sizeof(*Mapping) || + Mapping->Version != PERFECT_HASH_SERVER_BULK_RESULT_VERSION) { + Result = E_UNEXPECTED; + goto End; + } + + if (ARGUMENT_PRESENT(BulkResult)) { + *BulkResult = *Mapping; + } + + Result = Mapping->Result; + +End: + + if (Mapping) { + UnmapViewOfFile(Mapping); + } + + if (ResultHandle) { + CloseHandle(ResultHandle); + } + + if (EventHandle) { + CloseHandle(EventHandle); + } + + return Result; +} + +#ifdef PH_WINDOWS +DECLSPEC_NORETURN +VOID +WINAPI +mainCRTStartup( + VOID + ) +{ + HMODULE Module = NULL; + HRESULT Result = S_OK; + LPWSTR *ArgvW; + LPWSTR CommandLineW; + PICLASSFACTORY ClassFactory; + PPERFECT_HASH_CLIENT Client = NULL; + PERFECT_HASH_CLIENT_CLI_OPTIONS Options = { 0 }; + PERFECT_HASH_SERVER_REQUEST Request = { 0 }; + PERFECT_HASH_SERVER_BULK_RESULT BulkResult = { 0 }; + PPERFECT_HASH_PRINT_ERROR PerfectHashPrintError = NULL; + PPERFECT_HASH_PRINT_MESSAGE PerfectHashPrintMessage; + PICLASSFACTORY_CREATE_INSTANCE CreateInstance; + INT NumberOfArguments = 0; + + CommandLineW = GetCommandLineW(); + ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); + + Result = PerfectHashBootstrap(&ClassFactory, + &PerfectHashPrintError, + &PerfectHashPrintMessage, + &Module); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashBootstrap, Result); + } + goto Error; + } + + CreateInstance = ClassFactory->Vtbl->CreateInstance; + + Result = CreateInstance(ClassFactory, + NULL, + &IID_PERFECT_HASH_CLIENT, + &Client); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashClientCreateInstance, Result); + } + goto Error; + } + + Result = ParseClientArgs(NumberOfArguments, ArgvW, &Options); + if (Result == S_FALSE) { + Result = S_OK; + goto End; + } else if (FAILED(Result)) { + PrintUsage(); + goto Error; + } + + Result = Client->Vtbl->Connect(Client, + Options.EndpointPresent ? + &Options.Endpoint : + NULL); + if (FAILED(Result)) { + goto Error; + } + + if (Options.RequestTypePresent) { + Request.SizeOfStruct = sizeof(Request); + Request.RequestType = Options.RequestType; + Request.RequestId = 1; + if (Options.CommandLinePresent) { + Request.CommandLine = Options.CommandLine; + } + + Result = Client->Vtbl->SubmitRequest(Client, &Request); + if (FAILED(Result)) { + UNICODE_STRING ResponsePayload; + ULONG ResponseFlags = 0; + HRESULT PayloadResult; + + PayloadResult = Client->Vtbl->GetLastResponse(Client, + &ResponsePayload, + &ResponseFlags); + if (SUCCEEDED(PayloadResult) && + ResponsePayload.Buffer && + (ResponseFlags & + PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE)) { + wprintf(L"%.*s\n", + (int)(ResponsePayload.Length / sizeof(WCHAR)), + ResponsePayload.Buffer); + } + goto Error; + } + + if (Options.RequestType == + PerfectHashBulkCreateDirectoryServerRequestType) { + Result = PerfectHashClientWaitForBulkCreateToken(Client, + &BulkResult); + if (FAILED(Result)) { + goto Error; + } + + Result = BulkResult.Result; + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(BulkCreateDirectory, Result); + } + } + goto End; + } + } + + goto End; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashClient, Result); + } + +End: + + if (Client) { + Client->Vtbl->Release(Client); + } + + if (ClassFactory) { + ClassFactory->Vtbl->Release(ClassFactory); + } + + if (Module) { + FreeLibrary(Module); + } + + ExitProcess((ULONG)Result); +} + +#else // PH_WINDOWS + +int +main( + int NumberOfArguments, + char **ArgvA + ) +{ + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + return 0; +} + +#endif // PH_WINDOWS + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.rc b/src/PerfectHashClientExe/PerfectHashClientExe.rc new file mode 100644 index 00000000..f743c6b0 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.rc @@ -0,0 +1,49 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + PerfectHashClientExe.rc + +Abstract: + + This is the main resource compiler for the perfect hash client executable. + It is responsible for providing version information. + +--*/ + +#include "../PerfectHashVersion.rc" + +#define VER_FILEDESCRIPTION_STR "Perfect Hash Client Application" +#define VER_ORIGINALFILENAME_STR "PerfectHashClient.exe" +#define VER_INTERNALNAME_STR VER_ORIGINALFILENAME_STR + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_PRODUCTVERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +FILEFLAGS VER_FILEFLAGS +FILEOS VOS_NT +FILETYPE VFT_APP +FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + BEGIN + VALUE "CompanyName", VER_COMPANYNAME_STR + VALUE "FileDescription", VER_FILEDESCRIPTION_STR + VALUE "LegalCopyright", VER_LEGALCOPYRIGHT_STR + VALUE "OriginalFilename", VER_ORIGINALFILENAME_STR + VALUE "ProductName", VER_PRODUCTNAME_STR + VALUE "ProductVersion", VER_PRODUCTVERSION_STR + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END + diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj new file mode 100644 index 00000000..9ff29072 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj @@ -0,0 +1,71 @@ + + + + + Debug + x64 + + + Release + x64 + + + PGInstrument + x64 + + + PGUpdate + x64 + + + PGOptimize + x64 + + + + {EC0BFDD4-B3C5-4932-BB90-ED4D5D942198} + PerfectHashClientExe + 10.0 + PerfectHashClientExe + + + PerfectHashClient + .exe + + + + Application + + + + + + + + + + + + + + false + + + + + + + + + Create + + + + + + + + + + + diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters new file mode 100644 index 00000000..42882767 --- /dev/null +++ b/src/PerfectHashClientExe/PerfectHashClientExe.vcxproj.filters @@ -0,0 +1,41 @@ + + + + + {F15B494D-F723-4443-9E03-3A04103D2DB9} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {AC13CE39-13FD-4BA5-9F10-CAAACA7CC3EE} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {6F13BC5B-A638-48FA-9F17-58A620D37510} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + + + Resource Files + + + Resource Files + + + diff --git a/src/PerfectHashClientExe/stdafx.c b/src/PerfectHashClientExe/stdafx.c new file mode 100644 index 00000000..fd4f341c --- /dev/null +++ b/src/PerfectHashClientExe/stdafx.c @@ -0,0 +1 @@ +#include "stdafx.h" diff --git a/src/PerfectHashClientExe/stdafx.h b/src/PerfectHashClientExe/stdafx.h new file mode 100644 index 00000000..95bfe414 --- /dev/null +++ b/src/PerfectHashClientExe/stdafx.h @@ -0,0 +1,21 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + stdafx.h + +Abstract: + + This is the precompiled header file for the perfect hash client + executable. + +--*/ + +#pragma once + +#include +#include + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashClientExe/targetver.h b/src/PerfectHashClientExe/targetver.h new file mode 100644 index 00000000..cfae052b --- /dev/null +++ b/src/PerfectHashClientExe/targetver.h @@ -0,0 +1 @@ +#include diff --git a/src/PerfectHashCreateExe/CMakeLists.txt b/src/PerfectHashCreateExe/CMakeLists.txt index 23bb578c..db672b36 100644 --- a/src/PerfectHashCreateExe/CMakeLists.txt +++ b/src/PerfectHashCreateExe/CMakeLists.txt @@ -60,6 +60,10 @@ add_dependencies(${target} PerfectHash ) +if(WIN32) + target_link_libraries(${target} PRIVATE Dbghelp) +endif() + target_link_libraries(${target} PRIVATE PerfectHash ${CMAKE_DL_LIBS}) # vim:set ts=8 sw=4 sts=4 tw=80 expandtab syntax=cmake : diff --git a/src/PerfectHashCreateExe/PerfectHashCreateExe.c b/src/PerfectHashCreateExe/PerfectHashCreateExe.c index b7fe31f4..14e545cb 100644 --- a/src/PerfectHashCreateExe/PerfectHashCreateExe.c +++ b/src/PerfectHashCreateExe/PerfectHashCreateExe.c @@ -17,6 +17,251 @@ Module Name: #include "stdafx.h" +#include +#include + +#ifdef PH_WINDOWS +#pragma warning(push) +#pragma warning(disable: 4820 4255) +#include +#pragma warning(pop) +#endif + +#ifdef PH_WINDOWS +#define PH_CREATE_MAX_STACK_FRAMES 64 + +static +VOID +PerfectHashCreateWriteString( + _In_z_ PCSTR String + ) +/*++ + +Routine Description: + + Writes a null-terminated string to STDERR. + +Arguments: + + String - Supplies a pointer to a NULL-terminated string. + +Return Value: + + None. + +--*/ +{ + DWORD BytesWritten; + size_t Length; + HANDLE ErrorHandle; + + ErrorHandle = GetStdHandle(STD_ERROR_HANDLE); + if (ErrorHandle == INVALID_HANDLE_VALUE || ErrorHandle == NULL) { + return; + } + + Length = strlen(String); + if (Length == 0 || Length > MAXDWORD) { + return; + } + + WriteFile(ErrorHandle, + String, + (DWORD)Length, + &BytesWritten, + NULL); +} + +static +VOID +PerfectHashCreateDumpStack( + _In_ PCONTEXT Context + ) +/*++ + +Routine Description: + + Dumps a best-effort stack trace to STDERR using DbgHelp. Intended for + diagnosing unhandled exceptions during table creation in debug builds. + +Arguments: + + Context - Supplies a pointer to a CONTEXT record. + +Return Value: + + None. + +--*/ +{ + HANDLE Process; + HANDLE Thread; + DWORD MachineType; + DWORD64 Displacement; + DWORD LineDisplacement; + ULONG FrameIndex; + STACKFRAME64 Frame = { 0 }; + PCSTR SymbolPath = NULL; + CHAR ModulePath[MAX_PATH]; + CHAR *LastSlash; + CHAR Buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME] = { 0 }; + PSYMBOL_INFO Symbol = (PSYMBOL_INFO)Buffer; + IMAGEHLP_LINE64 Line = { 0 }; + CHAR LineBuffer[512]; + + if (!ARGUMENT_PRESENT(Context)) { + return; + } + + Process = GetCurrentProcess(); + Thread = GetCurrentThread(); + + SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_UNDNAME | SYMOPT_LOAD_LINES); + + ModulePath[0] = '\0'; + if (GetModuleFileNameA(NULL, ModulePath, ARRAYSIZE(ModulePath)) != 0) { + LastSlash = strrchr(ModulePath, '\\'); + if (LastSlash) { + *LastSlash = '\0'; + SymbolPath = ModulePath; + } + } + + if (!SymInitialize(Process, SymbolPath, TRUE)) { + return; + } + +#if defined(_M_AMD64) || defined(_M_X64) + MachineType = IMAGE_FILE_MACHINE_AMD64; + Frame.AddrPC.Offset = Context->Rip; + Frame.AddrFrame.Offset = Context->Rbp; + Frame.AddrStack.Offset = Context->Rsp; +#elif defined(_M_IX86) + MachineType = IMAGE_FILE_MACHINE_I386; + Frame.AddrPC.Offset = Context->Eip; + Frame.AddrFrame.Offset = Context->Ebp; + Frame.AddrStack.Offset = Context->Esp; +#else + MachineType = IMAGE_FILE_MACHINE_UNKNOWN; +#endif + + Frame.AddrPC.Mode = AddrModeFlat; + Frame.AddrFrame.Mode = AddrModeFlat; + Frame.AddrStack.Mode = AddrModeFlat; + + PerfectHashCreateWriteString("Unhandled exception stack trace:\n"); + + for (FrameIndex = 0; FrameIndex < PH_CREATE_MAX_STACK_FRAMES; FrameIndex++) { + + if (!StackWalk64(MachineType, + Process, + Thread, + &Frame, + Context, + NULL, + SymFunctionTableAccess64, + SymGetModuleBase64, + NULL)) { + break; + } + + if (Frame.AddrPC.Offset == 0) { + break; + } + + Symbol->SizeOfStruct = sizeof(*Symbol); + Symbol->MaxNameLen = MAX_SYM_NAME; + Displacement = 0; + + Line.SizeOfStruct = sizeof(Line); + LineDisplacement = 0; + + if (SymFromAddr(Process, + Frame.AddrPC.Offset, + &Displacement, + Symbol)) { + + if (SymGetLineFromAddr64(Process, + Frame.AddrPC.Offset, + &LineDisplacement, + &Line)) { + sprintf_s(LineBuffer, + sizeof(LineBuffer), + " %02lu: %s + 0x%llx (%s:%lu)\n", + FrameIndex, + Symbol->Name, + (unsigned long long)Displacement, + Line.FileName, + Line.LineNumber); + } else { + sprintf_s(LineBuffer, + sizeof(LineBuffer), + " %02lu: %s + 0x%llx\n", + FrameIndex, + Symbol->Name, + (unsigned long long)Displacement); + } + + } else { + sprintf_s(LineBuffer, + sizeof(LineBuffer), + " %02lu: 0x%llx\n", + FrameIndex, + (unsigned long long)Frame.AddrPC.Offset); + } + + PerfectHashCreateWriteString(LineBuffer); + } + + SymCleanup(Process); +} + +static +LONG +WINAPI +PerfectHashCreateUnhandledExceptionFilter( + _In_ PEXCEPTION_POINTERS ExceptionPointers + ) +/*++ + +Routine Description: + + Unhandled exception filter that logs the exception code and stack trace. + +Arguments: + + ExceptionPointers - Supplies a pointer to the exception information. + +Return Value: + + EXCEPTION_EXECUTE_HANDLER. + +--*/ +{ + CHAR Buffer[128]; + ULONG Code; + + if (!ARGUMENT_PRESENT(ExceptionPointers) || + !ARGUMENT_PRESENT(ExceptionPointers->ExceptionRecord)) { + return EXCEPTION_EXECUTE_HANDLER; + } + + Code = ExceptionPointers->ExceptionRecord->ExceptionCode; + + sprintf_s(Buffer, + sizeof(Buffer), + "Unhandled exception: 0x%08lX\n", + Code); + PerfectHashCreateWriteString(Buffer); + + if (ARGUMENT_PRESENT(ExceptionPointers->ContextRecord)) { + PerfectHashCreateDumpStack(ExceptionPointers->ContextRecord); + } + + return EXCEPTION_EXECUTE_HANDLER; +} +#endif + // // Main entry point. // @@ -43,6 +288,10 @@ mainCRTStartup( CommandLineW = GetCommandLineW(); ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); +#ifdef PH_WINDOWS + SetUnhandledExceptionFilter(PerfectHashCreateUnhandledExceptionFilter); +#endif + Result = PerfectHashBootstrap(&ClassFactory, &PerfectHashPrintError, &PerfectHashPrintMessage, diff --git a/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj b/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj index 8a69ee39..64795d5e 100644 --- a/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj +++ b/src/PerfectHashCreateExe/PerfectHashCreateExe.vcxproj @@ -50,6 +50,9 @@ false + + Dbghelp.lib;%(AdditionalDependencies) + diff --git a/src/PerfectHashServerExe/CMakeLists.txt b/src/PerfectHashServerExe/CMakeLists.txt new file mode 100644 index 00000000..8a087229 --- /dev/null +++ b/src/PerfectHashServerExe/CMakeLists.txt @@ -0,0 +1,65 @@ +set(target PerfectHashServerExe) + +################################################################################ +# Source groups +################################################################################ +set(Header_Files + "stdafx.h" + "targetver.h" +) +source_group("Header Files" FILES ${Header_Files}) + +set(Source_Files + "PerfectHashServerExe.c" + #"stdafx.c" +) +source_group("Source Files" FILES ${Source_Files}) + +set(ALL_FILES + ${Header_Files} + ${Source_Files} +) + +set(Resource_Files + "PerfectHashServerExe.rc" +) +source_group("Resource Files" FILES ${Resource_Files}) + +if(WIN32) + list( + APPEND ALL_FILES + ${Resource_Files} + ) +endif() + +add_executable(${target} ${ALL_FILES}) + +################################################################################ +# Target +################################################################################ + +perfecthash_apply_common_settings(${target}) + +################################################################################ +# Output executable name +################################################################################ +set_target_properties(${target} PROPERTIES + OUTPUT_NAME "PerfectHashServer" +) +################################################################################ +# Compile definitions +################################################################################ + +################################################################################ +# Compile and link options +################################################################################ +################################################################################ +# Dependencies +################################################################################ +add_dependencies(${target} + PerfectHash +) + +target_link_libraries(${target} PRIVATE PerfectHash ${CMAKE_DL_LIBS}) + +# vim:set ts=8 sw=4 sts=4 tw=80 expandtab syntax=cmake : diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.c b/src/PerfectHashServerExe/PerfectHashServerExe.c new file mode 100644 index 00000000..c05b4ced --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.c @@ -0,0 +1,795 @@ +/*++ + +Copyright (c) 2018-2025 Trent Nelson + +Module Name: + + PerfectHashServerExe.c + +Abstract: + + This module implements the main entry point for the perfect hash server. + It loads the perfect hash library, creates a server instance, applies any + command line options, then starts and waits on the server. + +--*/ + +#include "stdafx.h" +#include +#include +#include + +#ifdef PH_WINDOWS + +typedef DWORD MINIDUMP_TYPE; + +#define MiniDumpWithDataSegs ((MINIDUMP_TYPE)0x00000001) + +#pragma warning(push) +#pragma warning(disable: 4820) + +typedef struct _MINIDUMP_EXCEPTION_INFORMATION { + DWORD ThreadId; + PEXCEPTION_POINTERS ExceptionPointers; + BOOL ClientPointers; +} MINIDUMP_EXCEPTION_INFORMATION, *PMINIDUMP_EXCEPTION_INFORMATION; + +#pragma warning(pop) + +typedef PVOID PMINIDUMP_USER_STREAM_INFORMATION; +typedef PVOID PMINIDUMP_CALLBACK_INFORMATION; + +#endif + +typedef struct _PERFECT_HASH_SERVER_CLI_OPTIONS { + UNICODE_STRING Endpoint; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; + ULONG MaximumConcurrency; + BOOLEAN EndpointPresent; + BOOLEAN MaximumConcurrencyPresent; + BOOLEAN NumaNodeMaskPresent; + BOOLEAN LocalOnly; + BOOLEAN LocalOnlyPresent; + UCHAR Padding1[7]; +} PERFECT_HASH_SERVER_CLI_OPTIONS; +typedef PERFECT_HASH_SERVER_CLI_OPTIONS *PPERFECT_HASH_SERVER_CLI_OPTIONS; + +static PPERFECT_HASH_SERVER GlobalServer; + +#ifdef PH_WINDOWS + +typedef BOOL (WINAPI *PMINIDUMP_WRITE_DUMP)( + HANDLE Process, + DWORD ProcessId, + HANDLE File, + MINIDUMP_TYPE DumpType, + PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam, + PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam, + PMINIDUMP_CALLBACK_INFORMATION CallbackParam + ); + +static +LONG +WINAPI +PerfectHashServerUnhandledExceptionFilter( + _In_ struct _EXCEPTION_POINTERS *ExceptionPointers + ) +{ + HMODULE DbgHelpModule; + HANDLE FileHandle = INVALID_HANDLE_VALUE; + PMINIDUMP_WRITE_DUMP MiniDumpWriteDump; + MINIDUMP_EXCEPTION_INFORMATION ExceptionInfo; + MINIDUMP_TYPE DumpType; + BOOL DumpResult = FALSE; + BOOL FallbackResult = FALSE; + BOOL TriedFallback = FALSE; + DWORD DumpError = 0; + DWORD FallbackError = 0; + HANDLE LogHandle = INVALID_HANDLE_VALUE; + DWORD BytesWritten; + PVOID BackTrace[64]; + USHORT Index; + USHORT Frames; + CHAR Buffer[256]; + HMODULE Module; + HMODULE ExeBase; + ULONG_PTR Offset; + BOOLEAN ForceFallback = FALSE; + WCHAR ModulePathW[MAX_PATH]; + CHAR ModulePathA[MAX_PATH]; + DWORD ModulePathLength; + INT ModulePathChars; + + LogHandle = CreateFileW(L"PerfectHashServerCrash.log", + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (LogHandle != INVALID_HANDLE_VALUE) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + Module = NULL; + if (GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)ExceptionPointers->ExceptionRecord->ExceptionAddress, + &Module)) { + Offset = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress - + (ULONG_PTR)Module; + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionModule: 0x%p Offset: 0x%Ix\r\n", + Module, + Offset); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ThreadId: %lu\r\n", + GetCurrentThreadId()); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + DbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); + if (!DbgHelpModule) { + goto End; + } + +#pragma warning(push) +#pragma warning(disable: 4191) + MiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( + GetProcAddress(DbgHelpModule, "MiniDumpWriteDump") + ); +#pragma warning(pop) + + if (!MiniDumpWriteDump) { + goto End; + } + + FileHandle = CreateFileW(L"PerfectHashServerCrash.dmp", + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (FileHandle == INVALID_HANDLE_VALUE) { + goto End; + } + + ExceptionInfo.ThreadId = GetCurrentThreadId(); + ExceptionInfo.ExceptionPointers = ExceptionPointers; + ExceptionInfo.ClientPointers = FALSE; + + DumpType = MiniDumpWithDataSegs; + + ForceFallback = (GetEnvironmentVariableW( + L"PH_SERVER_MINIDUMP_FORCE_FALLBACK", + NULL, + 0 + ) > 0); + + if (!ForceFallback) { + DumpResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + &ExceptionInfo, + NULL, + NULL); + } else { + DumpResult = FALSE; + DumpError = ERROR_NOACCESS; + } + + if (!DumpResult) { + if (!ForceFallback) { + DumpError = GetLastError(); + } + TriedFallback = TRUE; + + CloseHandle(FileHandle); + FileHandle = CreateFileW(L"PerfectHashServerCrash.dmp", + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + + if (FileHandle != INVALID_HANDLE_VALUE) { + DumpType = 0; + FallbackResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + &ExceptionInfo, + NULL, + NULL); + if (!FallbackResult) { + FallbackError = GetLastError(); + } + } + } + + if (LogHandle == INVALID_HANDLE_VALUE) { + LogHandle = CreateFileW(L"PerfectHashServerCrash.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + } + + if (LogHandle == INVALID_HANDLE_VALUE) { + goto End; + } + + SetFilePointer(LogHandle, 0, NULL, FILE_END); + + ExeBase = GetModuleHandleW(NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExeBase: 0x%p\r\n", + ExeBase); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "MiniDumpForceFallback: %s\r\n", + ForceFallback ? "true" : "false"); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "MiniDumpWriteDump: %s (Error=0x%08lX Win32=%lu)\r\n", + DumpResult ? "OK" : "FAILED", + DumpError, + (DumpError & 0xFFFF)); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + if (TriedFallback) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "MiniDumpWriteDumpFallback: %s (Error=0x%08lX Win32=%lu)\r\n", + FallbackResult ? "OK" : "FAILED", + FallbackError, + (FallbackError & 0xFFFF)); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); + + WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + + Frames = RtlCaptureStackBackTrace(0, + ARRAYSIZE(BackTrace), + BackTrace, + NULL); + + for (Index = 0; Index < Frames; Index++) { + Module = NULL; + Offset = 0; + ModulePathLength = 0; + ModulePathChars = 0; + ModulePathA[0] = '?'; + ModulePathA[1] = '\0'; + + if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)BackTrace[Index], + &Module)) { + Offset = (ULONG_PTR)BackTrace[Index] - (ULONG_PTR)Module; + + ModulePathLength = GetModuleFileNameW(Module, + ModulePathW, + ARRAYSIZE(ModulePathW)); + + if (ModulePathLength > 0) { + ModulePathW[ARRAYSIZE(ModulePathW) - 1] = L'\0'; + ModulePathChars = WideCharToMultiByte(CP_ACP, + 0, + ModulePathW, + ModulePathLength, + ModulePathA, + ARRAYSIZE(ModulePathA) - 1, + NULL, + NULL); + + if (ModulePathChars > 0) { + ModulePathA[ModulePathChars] = '\0'; + } + } + } + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Frame %02hu: 0x%p (base=0x%p offset=0x%Ix module=%s)\r\n", + Index, + BackTrace[Index], + Module, + Offset, + ModulePathA); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + +End: + + if (FileHandle != INVALID_HANDLE_VALUE) { + CloseHandle(FileHandle); + } + + if (LogHandle != INVALID_HANDLE_VALUE) { + CloseHandle(LogHandle); + } + + if (DbgHelpModule) { + FreeLibrary(DbgHelpModule); + } + + return EXCEPTION_EXECUTE_HANDLER; +} + +static +VOID +InstallPerfectHashServerCrashHandler( + VOID + ) +{ + DWORD Length; + + Length = GetEnvironmentVariableW(L"PH_LOG_SERVER_CRASH", NULL, 0); + if (Length == 0) { + return; + } + + SetUnhandledExceptionFilter(PerfectHashServerUnhandledExceptionFilter); +} + +#endif +static +VOID +PrintUsage( + VOID + ) +{ + wprintf(L"Usage: PerfectHashServer [--MaxConcurrency=] " + L"[--Numa=All|] [--Endpoint=] " + L"[--AllowRemote|--LocalOnly]\n"); + wprintf(L" --MaxConcurrency= Total worker threads (default: all)\n"); + wprintf(L" --Numa=All|0,1|0-3 NUMA node selection mask\n"); + wprintf(L" --Endpoint= Named pipe endpoint\n"); + wprintf(L" --AllowRemote Allow remote named pipe clients\n"); + wprintf(L" --LocalOnly Reject remote named pipe clients\n"); +} + +static +HRESULT +ParseUnsignedInteger( + _In_ PCWSTR Value, + _Out_ PULONG Result + ) +{ + PWSTR End = NULL; + ULONG Local; + + Local = wcstoul(Value, &End, 10); + if (!End || End == Value || *End != L'\0') { + return E_INVALIDARG; + } + + *Result = Local; + return S_OK; +} + +static +HRESULT +ParseNumaNodeMaskValue( + _In_ PCWSTR Value, + _Out_ PPERFECT_HASH_NUMA_NODE_MASK Mask + ) +{ + PCWSTR Ptr; + PERFECT_HASH_NUMA_NODE_MASK LocalMask; + + if (_wcsicmp(Value, L"All") == 0) { + *Mask = PERFECT_HASH_NUMA_NODE_MASK_ALL; + return S_OK; + } + + LocalMask = 0; + Ptr = Value; + + while (*Ptr) { + ULONG Start; + ULONG End; + PWSTR Next = NULL; + + Start = wcstoul(Ptr, &Next, 10); + if (!Next || Next == Ptr) { + return E_INVALIDARG; + } + + End = Start; + + if (*Next == L'-') { + Ptr = Next + 1; + End = wcstoul(Ptr, &Next, 10); + if (!Next || Next == Ptr) { + return E_INVALIDARG; + } + } + + if (End < Start) { + return E_INVALIDARG; + } + + if (End >= 64) { + return E_INVALIDARG; + } + + for (; Start <= End; Start++) { + LocalMask |= (1ULL << Start); + } + + if (*Next == L',') { + Ptr = Next + 1; + continue; + } else if (*Next == L'\0') { + break; + } else { + return E_INVALIDARG; + } + } + + if (LocalMask == 0) { + return E_INVALIDARG; + } + + *Mask = LocalMask; + return S_OK; +} + +static +HRESULT +ParseServerArgs( + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _Inout_ PPERFECT_HASH_SERVER_CLI_OPTIONS Options + ) +{ + ULONG Index; + + Options->MaximumConcurrency = 0; + Options->NumaNodeMask = PERFECT_HASH_NUMA_NODE_MASK_ALL; + Options->Endpoint.Buffer = NULL; + Options->Endpoint.Length = 0; + Options->Endpoint.MaximumLength = 0; + Options->EndpointPresent = FALSE; + Options->MaximumConcurrencyPresent = FALSE; + Options->NumaNodeMaskPresent = FALSE; + Options->LocalOnly = TRUE; + Options->LocalOnlyPresent = FALSE; + + for (Index = 1; Index < NumberOfArguments; Index++) { + PCWSTR Arg = ArgvW[Index]; + + if (!Arg) { + continue; + } + + if (Arg[0] != L'-' || Arg[1] != L'-') { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Arg += 2; + + if (_wcsicmp(Arg, L"Help") == 0 || + _wcsicmp(Arg, L"?") == 0) { + PrintUsage(); + return S_FALSE; + } + + if (_wcsnicmp(Arg, L"MaxConcurrency=", 15) == 0) { + HRESULT Result; + Arg += 15; + Result = ParseUnsignedInteger(Arg, &Options->MaximumConcurrency); + if (FAILED(Result)) { + return Result; + } + Options->MaximumConcurrencyPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"Numa=", 5) == 0) { + HRESULT Result; + Arg += 5; + Result = ParseNumaNodeMaskValue(Arg, &Options->NumaNodeMask); + if (FAILED(Result)) { + return Result; + } + Options->NumaNodeMaskPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"Endpoint=", 9) == 0) { + PCWSTR Value; + ULONG Length; + + Value = Arg + 9; + Length = (ULONG)wcslen(Value) * sizeof(WCHAR); + + Options->Endpoint.Buffer = (PWSTR)Value; + Options->Endpoint.Length = (USHORT)Length; + Options->Endpoint.MaximumLength = (USHORT)Length + sizeof(WCHAR); + Options->EndpointPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"AllowRemote") == 0) { + Options->LocalOnly = FALSE; + Options->LocalOnlyPresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"LocalOnly") == 0) { + Options->LocalOnly = TRUE; + Options->LocalOnlyPresent = TRUE; + continue; + } + + return PH_E_INVALID_COMMANDLINE_ARG; + } + + return S_OK; +} + +#ifdef PH_WINDOWS +static +BOOL +WINAPI +PerfectHashServerConsoleCtrlHandler( + _In_ DWORD ControlType + ) +{ + switch (ControlType) { + case CTRL_C_EVENT: + case CTRL_CLOSE_EVENT: + case CTRL_SHUTDOWN_EVENT: + if (GlobalServer) { + GlobalServer->Vtbl->Stop(GlobalServer); + } + return TRUE; + default: + return FALSE; + } +} +#endif + +// +// Main entry point. +// + +#ifdef PH_WINDOWS +DECLSPEC_NORETURN +VOID +WINAPI +mainCRTStartup( + VOID + ) +{ + HMODULE Module = NULL; + HRESULT Result = S_OK; + LPWSTR *ArgvW; + LPWSTR CommandLineW; + PICLASSFACTORY ClassFactory; + PPERFECT_HASH_SERVER Server = NULL; + PERFECT_HASH_SERVER_CLI_OPTIONS Options = { 0 }; + PPERFECT_HASH_PRINT_ERROR PerfectHashPrintError = NULL; + PPERFECT_HASH_PRINT_MESSAGE PerfectHashPrintMessage; + PICLASSFACTORY_CREATE_INSTANCE CreateInstance; + INT NumberOfArguments = 0; + + CommandLineW = GetCommandLineW(); + ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); + + InstallPerfectHashServerCrashHandler(); + + Result = PerfectHashBootstrap(&ClassFactory, + &PerfectHashPrintError, + &PerfectHashPrintMessage, + &Module); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashBootstrap, Result); + } + goto Error; + } + + CreateInstance = ClassFactory->Vtbl->CreateInstance; + + Result = CreateInstance(ClassFactory, + NULL, + &IID_PERFECT_HASH_SERVER, + &Server); + + if (FAILED(Result)) { + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashServerCreateInstance, Result); + } + goto Error; + } + + Result = ParseServerArgs(NumberOfArguments, ArgvW, &Options); + if (Result == S_FALSE) { + Result = S_OK; + goto End; + } else if (FAILED(Result)) { + PrintUsage(); + goto Error; + } + + if (Options.MaximumConcurrencyPresent) { + Result = Server->Vtbl->SetMaximumConcurrency( + Server, + Options.MaximumConcurrency + ); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.NumaNodeMaskPresent) { + Result = Server->Vtbl->SetNumaNodeMask(Server, + Options.NumaNodeMask); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.EndpointPresent) { + Result = Server->Vtbl->SetEndpoint(Server, &Options.Endpoint); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.LocalOnlyPresent) { + Result = Server->Vtbl->SetLocalOnly(Server, Options.LocalOnly); + if (FAILED(Result)) { + goto Error; + } + } + + GlobalServer = Server; + SetConsoleCtrlHandler(PerfectHashServerConsoleCtrlHandler, TRUE); + + Result = Server->Vtbl->Start(Server); + if (FAILED(Result)) { + goto Error; + } + + Result = Server->Vtbl->Wait(Server); + if (FAILED(Result)) { + goto Error; + } + + goto End; + +Error: + + if (Result == S_OK) { + Result = E_UNEXPECTED; + } + + if (PerfectHashPrintError != NULL) { + PH_ERROR(PerfectHashServer, Result); + } + +End: + + if (Server) { + Server->Vtbl->Release(Server); + } + + if (ClassFactory) { + ClassFactory->Vtbl->Release(ClassFactory); + } + + if (Module) { + FreeLibrary(Module); + } + + ExitProcess((ULONG)Result); +} + +#else // PH_WINDOWS + +int +main( + int NumberOfArguments, + char **ArgvA + ) +{ + UNREFERENCED_PARAMETER(NumberOfArguments); + UNREFERENCED_PARAMETER(ArgvA); + + return 0; +} + +#endif // PH_WINDOWS + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.rc b/src/PerfectHashServerExe/PerfectHashServerExe.rc new file mode 100644 index 00000000..02be9198 --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.rc @@ -0,0 +1,49 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + PerfectHashServerExe.rc + +Abstract: + + This is the main resource compiler for the perfect hash server executable. + It is responsible for providing version information. + +--*/ + +#include "../PerfectHashVersion.rc" + +#define VER_FILEDESCRIPTION_STR "Perfect Hash Server Application" +#define VER_ORIGINALFILENAME_STR "PerfectHashServer.exe" +#define VER_INTERNALNAME_STR VER_ORIGINALFILENAME_STR + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_PRODUCTVERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +FILEFLAGS VER_FILEFLAGS +FILEOS VOS_NT +FILETYPE VFT_APP +FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + BEGIN + VALUE "CompanyName", VER_COMPANYNAME_STR + VALUE "FileDescription", VER_FILEDESCRIPTION_STR + VALUE "LegalCopyright", VER_LEGALCOPYRIGHT_STR + VALUE "OriginalFilename", VER_ORIGINALFILENAME_STR + VALUE "ProductName", VER_PRODUCTNAME_STR + VALUE "ProductVersion", VER_PRODUCTVERSION_STR + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END + diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj new file mode 100644 index 00000000..469f31ec --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj @@ -0,0 +1,71 @@ + + + + + Debug + x64 + + + Release + x64 + + + PGInstrument + x64 + + + PGUpdate + x64 + + + PGOptimize + x64 + + + + {B269AB7F-EF35-490A-B66D-031E431C4481} + PerfectHashServerExe + 10.0 + PerfectHashServerExe + + + PerfectHashServer + .exe + + + + Application + + + + + + + + + + + + + + false + + + + + + + + + Create + + + + + + + + + + + diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters new file mode 100644 index 00000000..0373fa97 --- /dev/null +++ b/src/PerfectHashServerExe/PerfectHashServerExe.vcxproj.filters @@ -0,0 +1,41 @@ + + + + + {F15B494D-F723-4443-9E03-3A04103D2DB9} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {AC13CE39-13FD-4BA5-9F10-CAAACA7CC3EE} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {6F13BC5B-A638-48FA-9F17-58A620D37510} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + + + Resource Files + + + Resource Files + + + diff --git a/src/PerfectHashServerExe/stdafx.c b/src/PerfectHashServerExe/stdafx.c new file mode 100644 index 00000000..fd4f341c --- /dev/null +++ b/src/PerfectHashServerExe/stdafx.c @@ -0,0 +1 @@ +#include "stdafx.h" diff --git a/src/PerfectHashServerExe/stdafx.h b/src/PerfectHashServerExe/stdafx.h new file mode 100644 index 00000000..3790ffec --- /dev/null +++ b/src/PerfectHashServerExe/stdafx.h @@ -0,0 +1,21 @@ +/*++ + +Copyright (c) 2018 Trent Nelson + +Module Name: + + stdafx.h + +Abstract: + + This is the precompiled header file for the perfect hash server + executable. + +--*/ + +#pragma once + +#include +#include + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHashServerExe/targetver.h b/src/PerfectHashServerExe/targetver.h new file mode 100644 index 00000000..cfae052b --- /dev/null +++ b/src/PerfectHashServerExe/targetver.h @@ -0,0 +1 @@ +#include From da8cf156f2955f9164219fe52eead40e5168cbfc Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Thu, 22 Jan 2026 10:22:05 -0800 Subject: [PATCH 2/9] Working (I think) checkpoint-commit of IOCP progress. --- IOCP-LOGS.md | 33 + IOCP-TOOD.md | 13 +- scripts/test-minidump.ps1 | 128 +++ src/PerfectHash/CMakeLists.txt | 1 + src/PerfectHash/Chm01Async.c | 218 +---- src/PerfectHash/Chm01Async.h | 5 +- src/PerfectHash/Chm01FileWork.c | 19 + src/PerfectHash/Graph.c | 16 +- src/PerfectHash/GraphCu.c | 4 +- src/PerfectHash/PerfectHash.vcxproj | 1 + src/PerfectHash/PerfectHash.vcxproj.filters | 3 + src/PerfectHash/PerfectHashContext.c | 24 + src/PerfectHash/PerfectHashContext.h | 24 +- .../PerfectHashContextBulkCreate.c | 15 +- src/PerfectHash/PerfectHashContextIocp.c | 147 ++-- src/PerfectHash/PerfectHashContextIocp.h | 14 +- src/PerfectHash/PerfectHashContextIocpArgs.c | 793 ++++++++++++++++++ .../PerfectHashContextTableCreate.c | 96 ++- src/PerfectHash/PerfectHashServer.c | 452 +++++++--- src/PerfectHash/PerfectHashTableCreate.c | 3 +- src/PerfectHash/PerfectHashTls.h | 9 +- .../PerfectHashBulkCreateExe.c | 227 ++++- .../PerfectHashClientExe.c | 15 +- .../PerfectHashServerExe.c | 280 +++++-- 24 files changed, 2065 insertions(+), 475 deletions(-) create mode 100644 scripts/test-minidump.ps1 create mode 100644 src/PerfectHash/PerfectHashContextIocpArgs.c diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index 18549f4c..35447c86 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -58,3 +58,36 @@ - Added a CHM01 async prepare-wait state for the try-larger-table-size path (still returns `PH_E_NOT_IMPLEMENTED` but now waits for prepare file work before closing). - Stress runs against `perfecthash-keys\\hard` still time out; observed repeated `SetEndOfFile`/`PerfectHashFileTruncate` failures (error 1224: mapped section open) and `PrepareFileChm01` failures during IOCP bulk runs. - Added a context flag to skip context-level file work for per-file IOCP contexts; CHM01 async prepare/save/close now bypass context file items and signal events directly to avoid `ERROR_USER_MAPPED_FILE` conflicts. +- Added `CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED` and gated `Chm01AsyncFinalizeClose()` on save completion; updated graph completion to use `Job->Context` and the IOCP pump to wait for `Outstanding == 0` before exiting. +- IOCP stress runs still time out within the 120s tool limit, but bulk create AVs in `Chm01AsyncGraphStep` are no longer reproduced. +- Added per-work-item `LastError` tracking in IOCP bulk logging (`PerfectHashServerBulkCreateFailures.log`) to capture `GetLastError()` at the point of failure. +- Added focused two-file IOCP stress run directory (`build-win\\keys-test`) to isolate failures on `shell32-13803.keys`/`CoreUIComponents-7995.keys`. +- `Mulshrolate4X` is not recognized by `PerfectHashClient` (invalid hash function ID); only `Mulshrolate4RX` exists in current hash table list. +- IOCP bulk-create against the two-file `build-win\\keys-test` set with `Mulshrolate4RX` still hangs (client/server left running until manually terminated; no bulk failure log emitted). +- Normal `PerfectHashBulkCreate.exe` run on the same two-file set with `Mulshrolate4RX` crashed with `0xC0000005` after `CoreUIComponents-7995.keys`; no crash dump/log emitted. +- OG `PerfectHashBulkCreate.exe` run against `perfecthash-keys\\test1` (5 files) crashed with `0xC0000005` after `Hydrogen-43013.keys` even with an absolute output directory; `--DisableCsvOutputFile` did not prevent the crash; no crash dump/log emitted. + +## 2026-01-21 +- Created a clean `D:\\src\\perfecthash-main-clean` worktree at `origin/main` (7c83dc5a), built Debug `PerfectHashBulkCreateExe`, and ran `PerfectHashBulkCreate.exe` against `..\\perfecthash-keys\\test1` with `Chm01 Mulshrolate4RX And 4`; completed successfully with five output directories, indicating the crash is specific to `iocp-dev`. +- Noted that `tests/CMakeLists.txt` includes a bulk CLI test that uses `--NoFileIo` and `MultiplyShiftR`, so it doesn't cover the current file-I/O crash scenario. +- Clean rebuilt `build-win` after `PERFECT_HASH_CONTEXT` layout changes; `PerfectHashBulkCreate.exe` no longer AVs on `test1`, indicating a stale PCH/obj mismatch was responsible. +- Added crash-dir env overrides (`PH_BULK_CREATE_CRASH_DIR`, `PH_SERVER_CRASH_DIR`) and server crash test env (`PH_SERVER_CRASH_TEST`), plus DbgHelp preloading and `MiniDumpIgnoreInaccessibleMemory`. +- Reworked crash-test path to generate dumps without exception pointers (avoid `MiniDumpWriteDump` 998) and added `scripts/test-minidump.ps1`; Debug run now produces non-empty minidumps for bulk-create and server. +- Added a `MiniDumpWriteDump` retry path that drops exception pointers on `ERROR_NOACCESS`, keeping real-crash logs while allowing dumps to be written. +- Removed CHM01 async resize/try-larger-table logic (events, state transitions, and initial resize setup) to keep IOCP async solving focused on single-pass solve behavior. +- IOCP bulk stress against `perfecthash-keys\\hard` (Debug) still times out in ~120s; server left running and only ~12/23 output dirs created, indicating completion/hang persists post-resize removal. +- Attached `cdb` to `PerfectHashServer.exe` ~10s into a Debug IOCP bulk run and captured `~* kb` (`cdb-hard-backtrace.log`): main thread blocked in `PerfectHashServerWait`, IOCP worker threads blocked in `GetQueuedCompletionStatus` (33 waits on one IOCP handle, 24 on another), and ~1539 threads waiting in `ZwWaitForWorkViaWorkerFactory`, indicating no active IOCP completions at capture time and unexpected threadpool worker proliferation. +- Added IOCP-native argument extraction helpers (`PerfectHashContextIocpArgs.c`) and wired into build files. +- Added TLS flag plumbing for threadpool-less context creation and IOCP helper to create contexts with `CreateContextWithoutThreadpool` set. +- Updated server bulk/table create to avoid legacy context parsing and use IOCP-native arg extraction. +- Added IOCP table-create dispatch in server (context creation, file-work IOCP wiring, hook/RNG init, TLS context) and removed threadpool priority application. +- Routed server BulkCreate requests through the bulk-create directory path to avoid legacy context invocation. +- Clean rebuild (Debug) succeeded; IOCP stress on `perfecthash-keys\\hard` with `Mulshrolate4RX` and concurrency 32 timed out in ~330s; 23/23 output directories were created but the client hung waiting for completion, then was terminated. +- Debugger run with cdb breakpoints (`cdb-server-bp5.log`, `cdb-client-bp5.log`) showed client hits `PerfectHashClientWaitForBulkCreateToken`/`WaitForSingleObject`, server hits `PerfectHashServerBulkCreateWorkItemCallback`, but `PerfectHashServerCompleteBulkRequest` never hit within 120s; only 9 work-item callbacks logged and 5/23 output dirs created before termination. +- Debugger run against `perfecthash-keys\\test1` (5 files) showed `PerfectHashServerBulkCreateWorkItemCallback` hit 3 times, `PerfectHashServerCompleteBulkRequest` never hit, and only 1 output dir created (`cdb-server-bp-test1.log`, `cdb-client-bp-test1.log`). +- Repeated test1 run with server `--MaxConcurrency=1` showed only 1 work-item callback hit, no bulk completion, and 1 output dir created (`cdb-server-bp-test1-1.log`, `cdb-client-bp-test1-1.log`). +- Added cdb request-state logging using struct offsets from `cdb-types.log`. On test1, `PerfectHashServerEnqueueBulkRequest` hit once, and three work-item callbacks logged `TotalWorkItems=5` with `OutstandingWorkItems` at 2/4/6 and `DispatchComplete=0` at callback entry. `PerfectHashServerCompleteBulkRequest` never hit and only 1 output dir was created (`cdb-server-bp-test1-B.log`, `cdb-client-bp-test1-B.log`). +- `--NoFileIo` test (test1) under cdb: enqueue hit once, two work-item callbacks observed, no bulk completion hit, and no output directories created (`cdb-server-bp-test1-nofile.log`, `cdb-client-bp-test1-nofile.log`). +- Diagnosed `TpIsTimerSet` invalid-parameter crash in IOCP worker threads to `SubmitThreadpoolWork(Context->FinishedWork)` from `GraphSolve`; guarded finished-work submissions in `Graph.c`/`GraphCu.c` when threadpool init is skipped. +- Rebuilt Debug and reran IOCP bulk create on `perfecthash-keys\\test1` under cdb with breakpoints on `Chm01AsyncGraphComplete` and `PerfectHashServerCompleteBulkRequest`; graph completion fired repeatedly, bulk completion callback fired once, and all five output directories were created (`cdb-server-graphcomplete.log`). +- Re-ran IOCP bulk create on `perfecthash-keys\\test1` without cdb (`PERFECT_HASH_IOCP_ASYNC_CHM01=1`); completed successfully and emitted all five output directories (`build-win\\iocp-stress-test1-nocdb`). diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md index cb9afe5d..dd90cc36 100644 --- a/IOCP-TOOD.md +++ b/IOCP-TOOD.md @@ -1,19 +1,24 @@ # IOCP TODO - Validate bulk-create directory request end-to-end (sys32 stress) and tune per-file concurrency defaults. -- Run IOCP sys32 stress pass (Release, max concurrency) after crash fixes. -- Investigate Release bulk-create crash in `PerfectHashBulkCreate.exe` (0xC0000005) during sys32 stress. +- Run IOCP sys32 stress pass (Release, max concurrency) after clean rebuild and crash fixes. +- Decide how to guard against stale PCH/obj after `PERFECT_HASH_CONTEXT` layout changes (force clean or touch header) and recheck Release bulk-create. - Track down `FlushConsoleInputBuffer` failures during bulk create (seen on `main` and iocp-dev). - Validate access-denied fallback for per-file context threadpool minimum failures at higher concurrency. - Recheck named-pipe endpoint handling if `PerfectHashServer-StressSys32` continues to fail. - Decide whether BulkCreateDirectory should accept a single-directory short form or keep output dir required. - Exercise IOCP file work dispatch path (bulk create) to confirm outstanding event signaling and non-threadpool file work callbacks. - Validate CHM01 async path with smaller keysets (e.g. `hard`) and record timing vs legacy. +- Identify root cause of remaining IOCP bulk-create failures (`E_UNEXPECTED` on `shell32-13803.keys` / `CoreUIComponents-7995.keys`), using new `LastError` logging. +- Verify `MiniDumpWriteDump` retry-with-NULL covers real crash scenarios; decide whether to keep exception-pointer dumps as the primary path. - Investigate IOCP bulk-create async hang on `hard` keyset (client wait never completes). +- Confirm bulk request completion signaling and outstanding-count decrements when all files complete (hard run produced all 23 outputs but token never signaled). +- Investigate unexpected ~1500 threadpool worker threads (`ZwWaitForWorkViaWorkerFactory`) observed during IOCP server runs; identify source (RPC/TP APIs) and whether it contributes to the hang. - Validate context file work skip fix for `SetEndOfFile`/`PerfectHashFileTruncate` 1224 failures; decide whether to generate context files once per bulk request. -- Handle CHM01 async resize/try-larger-table-size path (currently returns `PH_E_NOT_IMPLEMENTED`). - Wire IOCP server bulk path to CHM01 async jobs and request completion callbacks. -- Complete native IOCP table/bulk create workflows (replace legacy-context delegation). +- Run a clean build after the IOCP-native arg parsing/context creation changes and confirm no duplicate symbol issues. +- Decide whether `PerfectHashClientExe` should wait on bulk-create tokens for `BulkCreate=` requests now that the server routes them through the bulk-directory path. +- Choose a node-selection policy for single table-create server requests (round-robin vs node 0 vs affinity). - Add IOCP work item queueing/state to drive per-request pipelines. - Integrate per-request concurrency caps and queueing policies. - Draft IOCP-README.md once protocol and dispatch behavior settle. diff --git a/scripts/test-minidump.ps1 b/scripts/test-minidump.ps1 new file mode 100644 index 00000000..43520eb0 --- /dev/null +++ b/scripts/test-minidump.ps1 @@ -0,0 +1,128 @@ +# Copyright (c) 2026 Trent Nelson + +param( + [string]$BuildDir = "build-win", + [string]$Config = "Debug", + [string]$OutputDir = "build-win\\minidump-test", + [switch]$ForceFallback +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$bulkExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashBulkCreate.exe" -f $Config) +$serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) + +if (-not (Test-Path $bulkExe)) { + throw "Bulk-create exe not found: $bulkExe" +} + +if (-not (Test-Path $serverExe)) { + throw "Server exe not found: $serverExe" +} + +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null +$dumpDir = (Resolve-Path $OutputDir).Path + +$bulkDump = Join-Path $dumpDir "PerfectHashBulkCreateCrash.dmp" +$bulkLog = Join-Path $dumpDir "PerfectHashBulkCreateCrash.log" +$serverDump = Join-Path $dumpDir "PerfectHashServerCrash.dmp" +$serverLog = Join-Path $dumpDir "PerfectHashServerCrash.log" + +Remove-Item $bulkDump, $bulkLog, $serverDump, $serverLog -ErrorAction SilentlyContinue + +$envSnapshot = @{ + PH_LOG_BULK_CREATE_CRASH = $env:PH_LOG_BULK_CREATE_CRASH + PH_BULK_CREATE_CRASH_TEST = $env:PH_BULK_CREATE_CRASH_TEST + PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK = $env:PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK + PH_BULK_CREATE_CRASH_DIR = $env:PH_BULK_CREATE_CRASH_DIR + PH_LOG_SERVER_CRASH = $env:PH_LOG_SERVER_CRASH + PH_SERVER_CRASH_TEST = $env:PH_SERVER_CRASH_TEST + PH_SERVER_MINIDUMP_FORCE_FALLBACK = $env:PH_SERVER_MINIDUMP_FORCE_FALLBACK + PH_SERVER_CRASH_DIR = $env:PH_SERVER_CRASH_DIR +} + +try { + $env:PH_BULK_CREATE_CRASH_DIR = $dumpDir + $env:PH_LOG_BULK_CREATE_CRASH = "1" + $env:PH_BULK_CREATE_CRASH_TEST = "AV" + if ($ForceFallback) { + $env:PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK = "1" + } else { + Remove-Item Env:PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK ` + -ErrorAction SilentlyContinue + } + + $bulkProcess = Start-Process -FilePath $bulkExe ` + -PassThru ` + -NoNewWindow ` + -Wait + + if ($bulkProcess.ExitCode -eq 0) { + throw ("Bulk-create crash test exited with code {0}" -f ` + $bulkProcess.ExitCode) + } + + if (-not (Test-Path $bulkDump)) { + throw "Bulk-create minidump not found." + } + + if ((Get-Item $bulkDump).Length -eq 0) { + throw "Bulk-create minidump is empty." + } + + if (-not (Test-Path $bulkLog)) { + throw "Bulk-create crash log not found." + } + + if ((Get-Item $bulkLog).Length -eq 0) { + throw "Bulk-create crash log is empty." + } + + $env:PH_SERVER_CRASH_DIR = $dumpDir + $env:PH_LOG_SERVER_CRASH = "1" + $env:PH_SERVER_CRASH_TEST = "AV" + if ($ForceFallback) { + $env:PH_SERVER_MINIDUMP_FORCE_FALLBACK = "1" + } else { + Remove-Item Env:PH_SERVER_MINIDUMP_FORCE_FALLBACK ` + -ErrorAction SilentlyContinue + } + + $serverProcess = Start-Process -FilePath $serverExe ` + -PassThru ` + -NoNewWindow ` + -Wait + + if ($serverProcess.ExitCode -eq 0) { + throw ("Server crash test exited with code {0}" -f ` + $serverProcess.ExitCode) + } + + if (-not (Test-Path $serverDump)) { + throw "Server minidump not found." + } + + if ((Get-Item $serverDump).Length -eq 0) { + throw "Server minidump is empty." + } + + if (-not (Test-Path $serverLog)) { + throw "Server crash log not found." + } + + if ((Get-Item $serverLog).Length -eq 0) { + throw "Server crash log is empty." + } +} finally { + foreach ($name in $envSnapshot.Keys) { + $value = $envSnapshot[$name] + if ($null -eq $value) { + Remove-Item ("Env:{0}" -f $name) -ErrorAction SilentlyContinue + } else { + Set-Item ("Env:{0}" -f $name) -Value $value + } + } +} + +Write-Host "Minidump test succeeded." diff --git a/src/PerfectHash/CMakeLists.txt b/src/PerfectHash/CMakeLists.txt index a65b6fc5..ab08ab37 100644 --- a/src/PerfectHash/CMakeLists.txt +++ b/src/PerfectHash/CMakeLists.txt @@ -208,6 +208,7 @@ set(Source_Files "PerfectHashAllocator.c" "PerfectHashContext.c" "PerfectHashContextIocp.c" + "PerfectHashContextIocpArgs.c" "PerfectHashContextTableCreate.c" "PerfectHashClient.c" "PerfectHashKeys.c" diff --git a/src/PerfectHash/Chm01Async.c b/src/PerfectHash/Chm01Async.c index a6e91e22..9e6d67fd 100644 --- a/src/PerfectHash/Chm01Async.c +++ b/src/PerfectHash/Chm01Async.c @@ -1,6 +1,6 @@ /*++ -Copyright (c) 2018-2025 Trent Nelson +Copyright (c) 2018-2026 Trent Nelson Module Name: @@ -24,9 +24,9 @@ Module Name: // Async job flags. // -#define CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES 0x00000001 -#define CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED 0x00000002 -#define CHM01_ASYNC_JOB_FLAG_TRY_LARGER_TABLE_SIZE 0x00000004 +#define CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES 0x00000001 +#define CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED 0x00000002 +#define CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED 0x00000008 typedef struct _CHM01_ASYNC_GRAPH_WORK { PERFECT_HASH_ASYNC_WORK Work; @@ -86,12 +86,6 @@ Chm01AsyncPollSolveEvents( _In_ PCHM01_ASYNC_JOB Job ); -static -HRESULT -Chm01AsyncFinalizeWaitPrepare( - _In_ PCHM01_ASYNC_JOB Job - ); - static HRESULT Chm01AsyncFinalizeVerify( @@ -146,7 +140,7 @@ Chm01AsyncGraphStep( GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); Graph = GraphWork->Graph; - Context = Graph->Context; + Context = GraphWork->Job->Context; if (!GraphWork->Started) { GraphWork->Started = TRUE; @@ -263,7 +257,6 @@ Chm01AsyncGraphComplete( { PGRAPH Graph; PHANDLE Event; - ULONG WaitResult; PPERFECT_HASH_CONTEXT Context; PCHM01_ASYNC_JOB Job; PCHM01_ASYNC_GRAPH_WORK GraphWork; @@ -273,25 +266,22 @@ Chm01AsyncGraphComplete( GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); Job = GraphWork->Job; Graph = GraphWork->Graph; - Context = Graph->Context; + Context = Job->Context; if (GraphWork->Started) { InterlockedDecrement(&Context->ActiveSolvingLoops); } if (InterlockedDecrement(&Context->RemainingSolverLoops) == 0) { - WaitResult = WaitForSingleObject(Context->TryLargerTableSizeEvent, 0); - if (WaitResult != WAIT_OBJECT_0) { - if (Context->FinishedCount == 0) { - Event = &Context->FailedEvent; - } else { - Event = &Context->SucceededEvent; - } - if (!SetEvent(*Event)) { - SYS_ERROR(SetEvent); - } - SetStopSolving(Context); + if (Context->FinishedCount == 0) { + Event = &Context->FailedEvent; + } else { + Event = &Context->SucceededEvent; } + if (!SetEvent(*Event)) { + SYS_ERROR(SetEvent); + } + SetStopSolving(Context); } if (InterlockedDecrement(&Job->ActiveGraphs) == 0) { @@ -409,10 +399,7 @@ Chm01AsyncInitializeJob( PCHM01_ASYNC_JOB Job ) { - PRTL Rtl; USHORT Index; - DOUBLE Limit; - DOUBLE Current; HRESULT Result; BOOLEAN LimitConcurrency; ULONG Concurrency; @@ -434,7 +421,6 @@ Chm01AsyncInitializeJob( Table = Job->Table; Context = Table->Context; - Rtl = Table->Rtl; Allocator = Table->Allocator; TableCreateFlags.AsULongLong = Table->TableCreateFlags.AsULongLong; @@ -446,8 +432,7 @@ Chm01AsyncInitializeJob( Job->Events[1] = Context->CompletedEvent; Job->Events[2] = Context->ShutdownEvent; Job->Events[3] = Context->FailedEvent; - Job->Events[4] = Context->TryLargerTableSizeEvent; - Job->Events[5] = Context->LowMemoryEvent; + Job->Events[4] = Context->LowMemoryEvent; { PHANDLE SaveEvent = Job->SaveEvents; @@ -519,59 +504,8 @@ Chm01AsyncInitializeJob( Job->NumberOfGraphs = NumberOfGraphs; - if (WantsAutoResizeIfKeysToEdgesRatioExceedsLimit(Table)) { - Limit = Table->AutoResizeWhenKeysToEdgesRatioExceeds; - Current = Table->Keys->KeysToEdgesRatio; - if (Current > Limit) { - Context->InitialResizes = 1; - } - } - - if (Context->InitialResizes > 0) { - ULONG InitialResizes; - ULARGE_INTEGER NumberOfEdges; - ULARGE_INTEGER NumberOfVertices; - - NumberOfEdges.QuadPart = Table->Keys->NumberOfKeys.QuadPart; - ASSERT(NumberOfEdges.HighPart == 0); - - NumberOfEdges.QuadPart = ( - Rtl->RoundUpPowerOfTwo32( - NumberOfEdges.LowPart - ) - ); - - if (NumberOfEdges.QuadPart < 8) { - NumberOfEdges.QuadPart = 8; - } - - if (NumberOfEdges.HighPart) { - Result = PH_E_TOO_MANY_EDGES; - goto Error; - } - - NumberOfVertices.QuadPart = ( - Rtl->RoundUpNextPowerOfTwo32(NumberOfEdges.LowPart) - ); - - Table->RequestedNumberOfTableElements.QuadPart = ( - NumberOfVertices.QuadPart - ); - - for (InitialResizes = Context->InitialResizes; - InitialResizes > 0; - InitialResizes--) { - - Table->RequestedNumberOfTableElements.QuadPart <<= 1ULL; - - if (Table->RequestedNumberOfTableElements.HighPart) { - Result = PH_I_REQUESTED_NUMBER_OF_TABLE_ELEMENTS_TOO_LARGE; - goto Error; - } - } - - Context->NumberOfTableResizeEvents = Context->InitialResizes; - } + Context->InitialResizes = 0; + Context->NumberOfTableResizeEvents = 0; Result = PrepareGraphInfoChm01(Table, &Job->GraphInfo, NULL); if (FAILED(Result)) { @@ -687,7 +621,6 @@ Chm01AsyncInitializeJob( Context->State.SolveTimeoutExpired = FALSE; Context->State.FixedAttemptsReached = FALSE; Context->State.MaxAttemptsReached = FALSE; - Context->State.UserRequestedResize = FALSE; ClearStopSolving(Context); Job->Attempt = 1; @@ -717,7 +650,7 @@ Chm01AsyncInitializeJob( // Configure solve timeout if requested. // - if (Table->MaxSolveTimeInSeconds > 0) { + if (Table->MaxSolveTimeInSeconds > 0 && Context->SolveTimeout) { SetThreadpoolTimer(Context->SolveTimeout, &Table->RelativeMaxSolveTimeInFiletime.AsFileTime, 0, @@ -817,7 +750,6 @@ Chm01AsyncPollSolveEvents( ) { ULONG WaitResult; - BOOLEAN TryLargerTableSize; PPERFECT_HASH_CONTEXT Context; Context = Job->Context; @@ -837,86 +769,14 @@ Chm01AsyncPollSolveEvents( return PH_E_CTRL_C_PRESSED; } - if (WaitResult == WAIT_OBJECT_0 + 5) { + if (WaitResult == WAIT_OBJECT_0 + 4) { InterlockedIncrement(&Context->LowMemoryObserved); return PH_I_LOW_MEMORY; } - TryLargerTableSize = ( - WaitResult == WAIT_OBJECT_0 + 4 || ( - WaitForSingleObject(Context->TryLargerTableSizeEvent, 0) == - WAIT_OBJECT_0 - ) - ); - - if (TryLargerTableSize) { - Job->Flags |= CHM01_ASYNC_JOB_FLAG_TRY_LARGER_TABLE_SIZE; - Job->LastResult = PH_E_NOT_IMPLEMENTED; - return S_OK; - } - return S_OK; } -_Use_decl_annotations_ -static -HRESULT -Chm01AsyncFinalizeWaitPrepare( - PCHM01_ASYNC_JOB Job - ) -{ - HRESULT Result; - ULONG WaitResult; - - WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->PrepareEvents), - Job->PrepareEvents, - TRUE, - 0); - - if (WaitResult == WAIT_TIMEOUT) { - return S_FALSE; - } - - if (WaitResult != WAIT_OBJECT_0) { - SYS_ERROR(WaitForMultipleObjects); - return PH_E_SYSTEM_CALL_FAILED; - } - - Result = S_OK; - -#undef EXPAND_AS_CHECK_ERRORS -#define EXPAND_AS_CHECK_ERRORS( \ - Verb, VUpper, Name, Upper, \ - EofType, EofValue, \ - Suffix, Extension, Stream, Base \ -) \ - if (Job->Verb##Name.NumberOfErrors > 0) { \ - Result = Job->Verb##Name.LastResult; \ - if (Result == S_OK || Result == E_UNEXPECTED) { \ - Result = PH_E_ERROR_DURING_##VUpper##_##Upper; \ - } \ - PH_ERROR( \ - CreatePerfectHashTableImplChm01_ErrorDuring##Verb##Name, \ - Result \ - ); \ - goto Error; \ - } - - PREPARE_FILE_WORK_TABLE_ENTRY(EXPAND_AS_CHECK_ERRORS) - -#undef EXPAND_AS_CHECK_ERRORS - - return S_OK; - -Error: - - if (Result == S_OK) { - Result = E_UNEXPECTED; - } - - return Result; -} - _Use_decl_annotations_ static HRESULT @@ -1092,6 +952,7 @@ Chm01AsyncFinalizeVerify( #undef SUBMIT_SAVE_FILE_WORK #undef EXPAND_AS_SUBMIT_FILE_WORK + Job->Flags |= CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED; } else { @@ -1277,6 +1138,20 @@ Chm01AsyncFinalizeClose( } } + if (Job->Flags & CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED) { + WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->SaveEvents), + Job->SaveEvents, + TRUE, + 0); + if (WaitResult == WAIT_TIMEOUT) { + return S_FALSE; + } + if (WaitResult != WAIT_OBJECT_0) { + SYS_ERROR(WaitForMultipleObjects); + return PH_E_SYSTEM_CALL_FAILED; + } + } + if (!NoFileIo(Table)) { WaitResult = WaitForMultipleObjects(ARRAYSIZE(Job->PrepareEvents), Job->PrepareEvents, @@ -1464,26 +1339,7 @@ Chm01AsyncStep( Job->State = Chm01AsyncStateFinalizeClose; return S_FALSE; } - if (Job->Flags & CHM01_ASYNC_JOB_FLAG_TRY_LARGER_TABLE_SIZE) { - Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; - Job->State = Chm01AsyncStateFinalizeWaitPrepare; - } else { - Job->State = Chm01AsyncStateFinalizeVerify; - } - return S_FALSE; - - case Chm01AsyncStateFinalizeWaitPrepare: - Result = Chm01AsyncFinalizeWaitPrepare(Job); - if (Result == S_FALSE) { - return S_FALSE; - } - if (FAILED(Result)) { - Job->LastResult = Result; - Job->Flags |= CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES; - Job->State = Chm01AsyncStateFinalizeClose; - return S_FALSE; - } - Job->State = Chm01AsyncStateFinalizeClose; + Job->State = Chm01AsyncStateFinalizeVerify; return S_FALSE; case Chm01AsyncStateFinalizeVerify: @@ -1693,7 +1549,9 @@ Chm01AsyncPumpIoCompletionPort( for (;;) { if (WaitForSingleObject(Job->CompletionEvent, 0) == WAIT_OBJECT_0) { - break; + if (Job->Async.Outstanding == 0) { + break; + } } Success = GetQueuedCompletionStatus(IoCompletionPort, diff --git a/src/PerfectHash/Chm01Async.h b/src/PerfectHash/Chm01Async.h index 54b53912..a7855b27 100644 --- a/src/PerfectHash/Chm01Async.h +++ b/src/PerfectHash/Chm01Async.h @@ -1,6 +1,6 @@ /*++ -Copyright (c) 2018-2025 Trent Nelson +Copyright (c) 2018-2026 Trent Nelson Module Name: @@ -20,7 +20,6 @@ Module Name: typedef enum _CHM01_ASYNC_STATE { Chm01AsyncStateInitialize = 0, Chm01AsyncStateSolveGraphs, - Chm01AsyncStateFinalizeWaitPrepare, Chm01AsyncStateFinalizeVerify, Chm01AsyncStateFinalizeWaitSave, Chm01AsyncStateFinalizeClose, @@ -53,7 +52,7 @@ typedef struct _CHM01_ASYNC_JOB { GRAPH_INFO PrevGraphInfo; GRAPH_INFO_ON_DISK GraphInfoOnDisk; PTABLE_INFO_ON_DISK TableInfoOnDisk; - HANDLE Events[6]; + HANDLE Events[5]; HANDLE SaveEvents[NUMBER_OF_SAVE_FILE_EVENTS]; HANDLE PrepareEvents[NUMBER_OF_PREPARE_FILE_EVENTS]; PCHM01_ASYNC_GRAPH_WORK *GraphWorkItems; diff --git a/src/PerfectHash/Chm01FileWork.c b/src/PerfectHash/Chm01FileWork.c index bf06ac66..701a8c78 100644 --- a/src/PerfectHash/Chm01FileWork.c +++ b/src/PerfectHash/Chm01FileWork.c @@ -179,6 +179,25 @@ Return Value: Item->FileId = FileId = FileWorkIdToFileId(FileWorkId); + if (SkipContextFileWork(Context) && + IsValidContextFileId((CONTEXT_FILE_ID)FileId)) { + Item->Flags.IsContextFile = TRUE; + Item->LastResult = S_OK; + Item->LastError = 0; + if (Event) { +#ifdef PH_WINDOWS + if (Item->Instance) { + SetEventWhenCallbackReturns(Item->Instance, Event); + } else { + SetEvent(Event); + } +#else + SetEvent(Event); +#endif + } + return; + } + // // Determine if this is a context file. Context files get treated // differently to normal table output files in that they are only diff --git a/src/PerfectHash/Graph.c b/src/PerfectHash/Graph.c index 574e11e5..d20a1a79 100644 --- a/src/PerfectHash/Graph.c +++ b/src/PerfectHash/Graph.c @@ -632,7 +632,9 @@ Return Value: } } InsertHeadFinishedWork(Context, &Graph->ListEntry); - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } return PH_S_STOP_GRAPH_SOLVING; } @@ -3175,7 +3177,9 @@ Return Value: // the graph contexts to finish and then assessing the context. // - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } // // Clear the caller's NewGraphPointer, as we're not going to be doing @@ -4261,7 +4265,9 @@ Return Value: if (Result != S_OK) { CONTEXT_END_TIMERS(Solve); SetStopSolving(Context); - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } return Result; } @@ -6276,7 +6282,9 @@ Return Value: // the graph contexts to finish and then assessing the context. // - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } // // Clear the caller's NewGraphPointer, as we're not going to be doing diff --git a/src/PerfectHash/GraphCu.c b/src/PerfectHash/GraphCu.c index b689a6c5..338dace5 100644 --- a/src/PerfectHash/GraphCu.c +++ b/src/PerfectHash/GraphCu.c @@ -967,7 +967,9 @@ Return Value: if (Result != S_OK) { CONTEXT_END_TIMERS(Solve); SetStopSolving(Context); - SubmitThreadpoolWork(Context->FinishedWork); + if (!SkipThreadpoolInitialization(Context)) { + SubmitThreadpoolWork(Context->FinishedWork); + } return Result; } diff --git a/src/PerfectHash/PerfectHash.vcxproj b/src/PerfectHash/PerfectHash.vcxproj index 1bd09b3f..9279a6cb 100644 --- a/src/PerfectHash/PerfectHash.vcxproj +++ b/src/PerfectHash/PerfectHash.vcxproj @@ -254,6 +254,7 @@ + diff --git a/src/PerfectHash/PerfectHash.vcxproj.filters b/src/PerfectHash/PerfectHash.vcxproj.filters index d0176d28..e5677f1f 100644 --- a/src/PerfectHash/PerfectHash.vcxproj.filters +++ b/src/PerfectHash/PerfectHash.vcxproj.filters @@ -64,6 +64,9 @@ Source Files + + Source Files + Source Files diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index c748b9b0..ddb0a19b 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -339,6 +339,21 @@ Return Value: Context->State.AsULong = 0; Context->Flags.AsULong = 0; + // + // Honor the TLS flag that requests IOCP-native contexts without + // threadpool initialization. + // + + { + PPERFECT_HASH_TLS_CONTEXT TlsContext; + + TlsContext = PerfectHashTlsGetContext(); + if (TlsContext && + TlsContext->Flags.CreateContextWithoutThreadpool) { + Context->Flags.SkipThreadpoolInitialization = TRUE; + } + } + InitializeSRWLock(&Context->Lock); if (!InitializeCriticalSectionAndSpinCount( @@ -476,6 +491,8 @@ Return Value: ThreadpoolConcurrency = GetMainThreadpoolConcurrency(MaximumConcurrency); + if (!Context->Flags.SkipThreadpoolInitialization) { + #ifdef PH_WINDOWS // @@ -739,6 +756,8 @@ Return Value: #endif + } + // // Initialize the timestamp string. // @@ -1982,6 +2001,11 @@ Return Value: #ifdef PH_WINDOWS if (!Context->MainThreadpool) { + if (Context->Flags.SkipThreadpoolInitialization) { + ReleasePerfectHashContextLockExclusive(Context); + return S_OK; + } + ReleasePerfectHashContextLockExclusive(Context); return E_UNEXPECTED; } diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index b473e3b8..aab10538 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -266,7 +266,29 @@ typedef PERFECT_HASH_CONTEXT_STATE *PPERFECT_HASH_CONTEXT_STATE; #define IsSystemRngId(Id) (Id == PerfectHashRngSystemId) #define IsSystemRng(Context) ((Context)->RngId == PerfectHashRngSystemId) -DEFINE_UNUSED_FLAGS(PERFECT_HASH_CONTEXT); +typedef union _PERFECT_HASH_CONTEXT_FLAGS { + struct { + + // + // When set, skip threadpool initialization for this context. + // + + ULONG SkipThreadpoolInitialization:1; + + // + // Unused bits. + // + + ULONG Unused:31; + }; + LONG AsLong; + ULONG AsULong; +} PERFECT_HASH_CONTEXT_FLAGS; +C_ASSERT(sizeof(PERFECT_HASH_CONTEXT_FLAGS) == sizeof(ULONG)); +typedef PERFECT_HASH_CONTEXT_FLAGS *PPERFECT_HASH_CONTEXT_FLAGS; + +#define SkipThreadpoolInitialization(Context) \ + ((Context)->Flags.SkipThreadpoolInitialization != FALSE) typedef struct _BEST_GRAPH_INFO { diff --git a/src/PerfectHash/PerfectHashContextBulkCreate.c b/src/PerfectHash/PerfectHashContextBulkCreate.c index 327dcf5c..e6deacfc 100644 --- a/src/PerfectHash/PerfectHashContextBulkCreate.c +++ b/src/PerfectHash/PerfectHashContextBulkCreate.c @@ -346,15 +346,18 @@ Return Value: // Get a reference to the stdout handle. // - if (!Context->OutputHandle) { - Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + if (!Silent) { if (!Context->OutputHandle) { - SYS_ERROR(GetStdHandle); - goto Error; + Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + if (!Context->OutputHandle || + Context->OutputHandle == INVALID_HANDLE_VALUE) { + Context->OutputHandle = NULL; + Silent = TRUE; + } } - } - OutputHandle = Context->OutputHandle; + OutputHandle = Context->OutputHandle; + } // // Calculate the size required for a new concatenated wide string buffer diff --git a/src/PerfectHash/PerfectHashContextIocp.c b/src/PerfectHash/PerfectHashContextIocp.c index 564ba1e7..174a3f46 100644 --- a/src/PerfectHash/PerfectHashContextIocp.c +++ b/src/PerfectHash/PerfectHashContextIocp.c @@ -1,6 +1,6 @@ /*++ -Copyright (c) 2018-2025 Trent Nelson +Copyright (c) 2018-2026 Trent Nelson Module Name: @@ -975,6 +975,75 @@ PerfectHashContextIocpGetBaseOutputDirectory( return S_OK; } +PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT + PerfectHashContextIocpCreateTableContext; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpCreateTableContext( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PPERFECT_HASH_CONTEXT *ContextPointer + ) +/*++ + +Routine Description: + + Creates a PERFECT_HASH_CONTEXT instance suitable for IOCP-native + workflows. The TLS flag CreateContextWithoutThreadpool is set for the + duration of the call to suppress threadpool initialization. + +Arguments: + + ContextIocp - Supplies a pointer to the IOCP context. + + ContextPointer - Receives the newly created PERFECT_HASH_CONTEXT instance. + +Return Value: + + S_OK on success, otherwise an error code. + +--*/ +{ + HRESULT Result; + BOOLEAN LocalTlsActive = FALSE; + PPERFECT_HASH_TLS_CONTEXT ActiveTls; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext = { 0 }; + PERFECT_HASH_TLS_CONTEXT_FLAGS SavedFlags = { 0 }; + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ContextPointer)) { + return E_POINTER; + } + + *ContextPointer = NULL; + + ActiveTls = PerfectHashTlsGetContext(); + if (ActiveTls) { + SavedFlags = ActiveTls->Flags; + ActiveTls->Flags.CreateContextWithoutThreadpool = TRUE; + } else { + ActiveTls = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + ActiveTls->Flags.CreateContextWithoutThreadpool = TRUE; + LocalTlsActive = TRUE; + } + + Result = ContextIocp->Vtbl->CreateInstance(ContextIocp, + NULL, + &IID_PERFECT_HASH_CONTEXT, + ContextPointer); + + if (LocalTlsActive) { + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + } else if (ActiveTls) { + ActiveTls->Flags = SavedFlags; + } + + return Result; +} + PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE PerfectHashContextIocpBulkCreate; PERFECT_HASH_CONTEXT_IOCP_BULK_CREATE_ARGVW PerfectHashContextIocpBulkCreateArgvW; PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW @@ -1090,44 +1159,6 @@ PerfectHashContextIocpBulkCreateArgvW( ); } -_Use_decl_annotations_ -HRESULT -PerfectHashContextIocpExtractBulkCreateArgsFromArgvW( - PPERFECT_HASH_CONTEXT_IOCP ContextIocp, - ULONG NumberOfArguments, - LPWSTR *ArgvW, - LPWSTR CommandLineW, - PUNICODE_STRING KeysDirectory, - PUNICODE_STRING BaseOutputDirectory, - PPERFECT_HASH_ALGORITHM_ID AlgorithmId, - PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, - PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, - PULONG MaximumConcurrency, - PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, - PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, - PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, - PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, - PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters - ) -{ - UNREFERENCED_PARAMETER(ContextIocp); - UNREFERENCED_PARAMETER(NumberOfArguments); - UNREFERENCED_PARAMETER(ArgvW); - UNREFERENCED_PARAMETER(CommandLineW); - UNREFERENCED_PARAMETER(KeysDirectory); - UNREFERENCED_PARAMETER(BaseOutputDirectory); - UNREFERENCED_PARAMETER(AlgorithmId); - UNREFERENCED_PARAMETER(HashFunctionId); - UNREFERENCED_PARAMETER(MaskFunctionId); - UNREFERENCED_PARAMETER(MaximumConcurrency); - UNREFERENCED_PARAMETER(ContextBulkCreateFlags); - UNREFERENCED_PARAMETER(KeysLoadFlags); - UNREFERENCED_PARAMETER(TableCreateFlags); - UNREFERENCED_PARAMETER(TableCompileFlags); - UNREFERENCED_PARAMETER(TableCreateParameters); - - return E_NOTIMPL; -} _Use_decl_annotations_ HRESULT @@ -1178,44 +1209,6 @@ PerfectHashContextIocpTableCreateArgvW( ); } -_Use_decl_annotations_ -HRESULT -PerfectHashContextIocpExtractTableCreateArgsFromArgvW( - PPERFECT_HASH_CONTEXT_IOCP ContextIocp, - ULONG NumberOfArguments, - LPWSTR *ArgvW, - LPWSTR CommandLineW, - PUNICODE_STRING KeysPath, - PUNICODE_STRING BaseOutputDirectory, - PPERFECT_HASH_ALGORITHM_ID AlgorithmId, - PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, - PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, - PULONG MaximumConcurrency, - PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags, - PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, - PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, - PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, - PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters - ) -{ - UNREFERENCED_PARAMETER(ContextIocp); - UNREFERENCED_PARAMETER(NumberOfArguments); - UNREFERENCED_PARAMETER(ArgvW); - UNREFERENCED_PARAMETER(CommandLineW); - UNREFERENCED_PARAMETER(KeysPath); - UNREFERENCED_PARAMETER(BaseOutputDirectory); - UNREFERENCED_PARAMETER(AlgorithmId); - UNREFERENCED_PARAMETER(HashFunctionId); - UNREFERENCED_PARAMETER(MaskFunctionId); - UNREFERENCED_PARAMETER(MaximumConcurrency); - UNREFERENCED_PARAMETER(ContextTableCreateFlags); - UNREFERENCED_PARAMETER(KeysLoadFlags); - UNREFERENCED_PARAMETER(TableCreateFlags); - UNREFERENCED_PARAMETER(TableCompileFlags); - UNREFERENCED_PARAMETER(TableCreateParameters); - - return E_NOTIMPL; -} _Use_decl_annotations_ HRESULT diff --git a/src/PerfectHash/PerfectHashContextIocp.h b/src/PerfectHash/PerfectHashContextIocp.h index 8ebe6ae8..b86e78c0 100644 --- a/src/PerfectHash/PerfectHashContextIocp.h +++ b/src/PerfectHash/PerfectHashContextIocp.h @@ -1,6 +1,6 @@ /*++ -Copyright (c) 2018-2025 Trent Nelson +Copyright (c) 2018-2026 Trent Nelson Module Name: @@ -202,10 +202,22 @@ HRESULT typedef PERFECT_HASH_CONTEXT_IOCP_STOP *PPERFECT_HASH_CONTEXT_IOCP_STOP; +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Outptr_ PPERFECT_HASH_CONTEXT *Context + ); +typedef PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT + *PPERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT; + extern PERFECT_HASH_CONTEXT_IOCP_INITIALIZE PerfectHashContextIocpInitialize; extern PERFECT_HASH_CONTEXT_IOCP_RUNDOWN PerfectHashContextIocpRundown; extern PERFECT_HASH_CONTEXT_IOCP_START PerfectHashContextIocpStart; extern PERFECT_HASH_CONTEXT_IOCP_STOP PerfectHashContextIocpStop; +extern PERFECT_HASH_CONTEXT_IOCP_CREATE_TABLE_CONTEXT + PerfectHashContextIocpCreateTableContext; extern PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY PerfectHashContextIocpSetMaximumConcurrency; diff --git a/src/PerfectHash/PerfectHashContextIocpArgs.c b/src/PerfectHash/PerfectHashContextIocpArgs.c new file mode 100644 index 00000000..c52cc042 --- /dev/null +++ b/src/PerfectHash/PerfectHashContextIocpArgs.c @@ -0,0 +1,793 @@ +/*++ + +Copyright (c) 2018-2026 Trent Nelson + +Module Name: + + PerfectHashContextIocpArgs.c + +Abstract: + + This module implements IOCP-native argument extraction helpers for + bulk-create and table-create operations. These routines mirror the + legacy context parsing logic but operate directly on the IOCP context's + RTL/allocator, avoiding the legacy context dependency. + +--*/ + +#include "stdafx.h" + +#define GET_LENGTH(Name) \ + ((USHORT)wcslen((Name)->Buffer) * (USHORT)sizeof(WCHAR)) +#define GET_MAX_LENGTH(Name) ((Name)->Length + sizeof(WCHAR)) + +#define VALIDATE_ID(Name, Upper) \ + if (FAILED(Rtl->RtlUnicodeStringToInteger(String, \ + 10, \ + (PULONG)Name##Id))) { \ + return PH_E_INVALID_##Upper##_ID; \ + } else if (*Name##Id == 0) { \ + Result = PerfectHashLookupIdForName(Rtl, \ + PerfectHash##Name##EnumId, \ + String, \ + (PULONG)Name##Id); \ + if (FAILED(Result)) { \ + return PH_E_INVALID_##Upper##_ID; \ + } \ + } \ + if (!IsValidPerfectHash##Name##Id(*Name##Id)) { \ + return PH_E_INVALID_##Upper##_ID; \ + } + +#define EXTRACT_ID(Name, Upper) \ + CurrentArg++; \ + String->Buffer = *ArgW++; \ + String->Length = GET_LENGTH(String); \ + String->MaximumLength = GET_MAX_LENGTH(String); \ + VALIDATE_ID(Name, Upper) + +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_BULK_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractBulkCreateArgsFromArgvW; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpExtractBulkCreateArgsFromArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW, + PUNICODE_STRING KeysDirectory, + PUNICODE_STRING BaseOutputDirectory, + PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PULONG MaximumConcurrency, + PPERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +/*++ + +Routine Description: + + Extracts arguments for the bulk-create functionality from an argument + vector array, typically obtained from a commandline invocation. + +Arguments: + + ContextIocp - Supplies a pointer to the PERFECT_HASH_CONTEXT_IOCP instance + for which the arguments are to be extracted. + + NumberOfArguments - Supplies the number of elements in the ArgvW array. + + ArgvW - Supplies a pointer to an array of wide C string arguments. + + CommandLineW - Supplies a pointer to the original command line used to + construct the ArgvW array above. This is only used for inclusion in + things like CSV output; it is not used programmatically (and is not + checked for correctness against ArgvW). + + KeysDirectory - Supplies a pointer to a UNICODE_STRING structure that + will be filled out with the keys directory. + + BaseOutputDirectory - Supplies a pointer to a UNICODE_STRING structure that + will be filled out with the output directory. + + AlgorithmId - Supplies the address of a variable that will receive the + algorithm ID. + + HashFunctionId - Supplies the address of a variable that will receive the + hash function ID. + + MaskFunctionId - Supplies the address of a variable that will receive the + mask function ID. + + MaximumConcurrency - Supplies the address of a variable that will receive + the maximum concurrency. + + ContextBulkCreateFlags - Supplies the address of a variable that will + receive the bulk-create flags. + + KeysLoadFlags - Supplies the address of a variable that will receive the + keys load flags. + + TableCreateFlags - Supplies the address of a variable that will receive + the table create flags. + + TableCompileFlags - Supplies the address of a variable that will receive + the table compile flags. + + TableCreateParameters - Supplies the address of a variable that will + receive a pointer to a table create parameters structure. + +Return Value: + + S_OK - Arguments extracted successfully. + + E_POINTER - One or more mandatory parameters were NULL pointers. + + PH_E_CONTEXT_BULK_CREATE_INVALID_NUM_ARGS - Invalid number of arguments. + + PH_E_INVALID_ALGORITHM_ID - Invalid algorithm ID. + + PH_E_INVALID_HASH_FUNCTION_ID - Invalid hash function ID. + + PH_E_INVALID_MASK_FUNCTION_ID - Invalid mask function ID. + + PH_E_INVALID_MAXIMUM_CONCURRENCY - Invalid maximum concurrency. + +--*/ +{ + PRTL Rtl; + LPWSTR *ArgW; + LPWSTR Arg; + HRESULT Result = S_OK; + HRESULT CleanupResult; + ULONG CurrentArg = 1; + PALLOCATOR Allocator; + UNICODE_STRING Temp; + PUNICODE_STRING String; + BOOLEAN InvalidPrefix; + BOOLEAN ValidNumberOfArguments; + + String = &Temp; + + // + // Validate arguments. + // + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ArgvW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(CommandLineW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysDirectory)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectory)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(AlgorithmId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(HashFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaskFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ContextBulkCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysLoadFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCompileFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateParameters)) { + return E_POINTER; + } + + ValidNumberOfArguments = (NumberOfArguments >= 7); + + if (!ValidNumberOfArguments) { + return PH_E_CONTEXT_BULK_CREATE_INVALID_NUM_ARGS; + } + + // + // Argument validation complete, continue. + // + + ArgW = &ArgvW[1]; + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + // + // The first six arguments (keys directory, base output directory, algo ID, + // hash function ID, mask function ID and maximum concurrency) are special + // in that they're mandatory and expected to appear sequentially, prior to + // any additional arguments (i.e. table create parameters) appearing. + // + + // + // Extract keys directory. + // + + CurrentArg++; + KeysDirectory->Buffer = *ArgW++; + KeysDirectory->Length = GET_LENGTH(KeysDirectory); + KeysDirectory->MaximumLength = GET_MAX_LENGTH(KeysDirectory); + + // + // Extract base output directory. + // + + CurrentArg++; + BaseOutputDirectory->Buffer = *ArgW++; + BaseOutputDirectory->Length = GET_LENGTH(BaseOutputDirectory); + BaseOutputDirectory->MaximumLength = GET_MAX_LENGTH(BaseOutputDirectory); + + // + // Extract algo, hash function and mask function IDs. + // + + EXTRACT_ID(Algorithm, ALGORITHM); + EXTRACT_ID(HashFunction, HASH_FUNCTION); + EXTRACT_ID(MaskFunction, MASK_FUNCTION); + + // + // Extract maximum concurrency. + // + + CurrentArg++; + String->Buffer = *ArgW++; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + if (FAILED(Rtl->RtlUnicodeStringToInteger(String, + 10, + MaximumConcurrency))) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + // + // Zero all flags (except for table create flags, as these may have + // default values) and table create parameters. + // + + KeysLoadFlags->AsULong = 0; + TableCompileFlags->AsULong = 0; + ContextBulkCreateFlags->AsULong = 0; + + for (; CurrentArg < NumberOfArguments; CurrentArg++, ArgW++) { + + String->Buffer = Arg = *ArgW; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + // + // If the argument doesn't start with two dashes, report it. + // + + InvalidPrefix = ( + (String->Length <= (sizeof(L'-') + sizeof(L'-'))) || + (!(*Arg++ == L'-' && *Arg++ == L'-')) + ); + + if (InvalidPrefix) { + goto InvalidArg; + } + + // + // Advance the buffer past the two dashes and update lengths + // accordingly. + // + + String->Buffer += 2; + String->Length -= (sizeof(WCHAR) * 2); + String->MaximumLength -= (sizeof(WCHAR) * 2); + + // + // Try each argument extraction routine for this argument; if it + // indicates an error, report it and break out of the loop. If it + // indicates it successfully extracted the argument (Result == S_OK), + // continue onto the next argument. Otherwise, verify it indicates + // that no argument was extracted (S_FALSE), then try the next routine. + // + +#define TRY_EXTRACT_ARG(Name) \ + Result = TryExtractArg##Name##Flags(Rtl, Allocator, String, Name##Flags); \ + if (FAILED(Result)) { \ + PH_ERROR(ExtractBulkCreateArgs_TryExtractArg##Name##Flags, Result); \ + break; \ + } else if (Result == S_OK) { \ + continue; \ + } else { \ + ASSERT(Result == S_FALSE); \ + } + + TRY_EXTRACT_ARG(ContextBulkCreate); + TRY_EXTRACT_ARG(KeysLoad); + TRY_EXTRACT_ARG(TableCreate); + TRY_EXTRACT_ARG(TableCompile); + + // + // If we get here, none of the previous extraction routines claimed the + // argument, so, provide the table create parameters extraction routine + // an opportunity to run. + // + + Result = TryExtractArgTableCreateParameters(Rtl, + String, + TableCreateParameters); + + if (FAILED(Result)) { + PH_ERROR(ExtractBulkCreateArgs_TryExtractTableCreateParams, Result); + break; + } + + if (Result == S_OK) { + continue; + } + + if (Result == PH_E_COMMANDLINE_ARG_MISSING_VALUE) { + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (FAILED(Result)) { + PH_ERROR(ExtractBulkCreateArgs_TryExtractTableCreateParams, Result); + break; + } + + ASSERT(Result == S_FALSE); + +InvalidArg: + + // + // If we get here, we don't recognize the argument. + // + + Result = PH_E_INVALID_COMMANDLINE_ARG; + PH_MESSAGE_ARGS(Result, String); + break; + } + + // + // If we failed, clean up the table create parameters. If that fails, + // report the error, then replace our return value error code with that + // error code. + // + + if (FAILED(Result)) { + CleanupResult = CleanupTableCreateParameters(TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + Result = CleanupResult; + } + } + + return Result; +} + +#undef TRY_EXTRACT_ARG + +PERFECT_HASH_CONTEXT_IOCP_EXTRACT_TABLE_CREATE_ARGS_FROM_ARGVW + PerfectHashContextIocpExtractTableCreateArgsFromArgvW; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpExtractTableCreateArgsFromArgvW( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLineW, + PUNICODE_STRING KeysPath, + PUNICODE_STRING BaseOutputDirectory, + PPERFECT_HASH_ALGORITHM_ID AlgorithmId, + PPERFECT_HASH_HASH_FUNCTION_ID HashFunctionId, + PPERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId, + PULONG MaximumConcurrency, + PPERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags, + PPERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags, + PPERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags, + PPERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags, + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters + ) +/*++ + +Routine Description: + + Extracts arguments for the table create functionality from an argument + vector array, typically obtained from a commandline invocation. + +Arguments: + + ContextIocp - Supplies a pointer to the PERFECT_HASH_CONTEXT_IOCP instance + for which the arguments are to be extracted. + + NumberOfArguments - Supplies the number of elements in the ArgvW array. + + ArgvW - Supplies a pointer to an array of wide C string arguments. + + CommandLineW - Supplies a pointer to the original command line used to + construct the ArgvW array above. This is only used for inclusion in + things like CSV output; it is not used programmatically (and is not + checked for correctness against ArgvW). + + KeysPath - Supplies a pointer to a UNICODE_STRING structure that will be + filled out with the keys path. + + BaseOutputDirectory - Supplies a pointer to a UNICODE_STRING structure that + will be filled out with the output directory. + + AlgorithmId - Supplies the address of a variable that will receive the + algorithm ID. + + HashFunctionId - Supplies the address of a variable that will receive the + hash function ID. + + MaskFunctionId - Supplies the address of a variable that will receive the + mask function ID. + + MaximumConcurrency - Supplies the address of a variable that will receive + the maximum concurrency. + + ContextTableCreateFlags - Supplies the address of a variable that will + receive the context table-create flags. + + KeysLoadFlags - Supplies the address of a variable that will receive the + keys load flags. + + TableCreateFlags - Supplies the address of a variable that will receive + the table create flags. + + TableCompileFlags - Supplies the address of a variable that will receive + the table compile flags. + + TableCreateParameters - Supplies the address of a table create params + structure that will receive any extracted params. + +Return Value: + + S_OK - Arguments extracted successfully. + + E_POINTER - One or more mandatory parameters were NULL pointers. + + PH_E_CONTEXT_TABLE_CREATE_INVALID_NUM_ARGS - Invalid number of arguments. + + PH_E_INVALID_ALGORITHM_ID - Invalid algorithm ID. + + PH_E_INVALID_HASH_FUNCTION_ID - Invalid hash function ID. + + PH_E_INVALID_MASK_FUNCTION_ID - Invalid mask function ID. + + PH_E_INVALID_MAXIMUM_CONCURRENCY - Invalid maximum concurrency. + +--*/ +{ + PRTL Rtl; + LPWSTR *ArgW; + LPWSTR Arg; + HRESULT Result = S_OK; + HRESULT CleanupResult; + HRESULT DebuggerResult; + ULONG CurrentArg = 1; + PALLOCATOR Allocator; + UNICODE_STRING Temp; + PUNICODE_STRING String; + BOOLEAN InvalidPrefix; + DEBUGGER_CONTEXT_FLAGS Flags; + BOOLEAN ValidNumberOfArguments; + PDEBUGGER_CONTEXT DebuggerContext; + + String = &Temp; + + // + // Validate arguments. + // + + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ArgvW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(CommandLineW)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysPath)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(BaseOutputDirectory)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(AlgorithmId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(HashFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaskFunctionId)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumConcurrency)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(ContextTableCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(KeysLoadFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCompileFlags)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(TableCreateParameters)) { + return E_POINTER; + } + + ValidNumberOfArguments = (NumberOfArguments >= 7); + + if (!ValidNumberOfArguments) { + return PH_E_CONTEXT_TABLE_CREATE_INVALID_NUM_ARGS; + } + + // + // Argument validation complete, continue. + // + + ArgW = &ArgvW[1]; + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + // + // Extract keys path. + // + + CurrentArg++; + KeysPath->Buffer = *ArgW++; + KeysPath->Length = GET_LENGTH(KeysPath); + KeysPath->MaximumLength = GET_MAX_LENGTH(KeysPath); + + // + // Extract base output directory. + // + + CurrentArg++; + BaseOutputDirectory->Buffer = *ArgW++; + BaseOutputDirectory->Length = GET_LENGTH(BaseOutputDirectory); + BaseOutputDirectory->MaximumLength = GET_MAX_LENGTH(BaseOutputDirectory); + + // + // Extract algorithm ID, hash function and mask function. + // + + EXTRACT_ID(Algorithm, ALGORITHM); + EXTRACT_ID(HashFunction, HASH_FUNCTION); + EXTRACT_ID(MaskFunction, MASK_FUNCTION); + + // + // Extract maximum concurrency. + // + + CurrentArg++; + String->Buffer = *ArgW++; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + if (FAILED(Rtl->RtlUnicodeStringToInteger(String, + 10, + MaximumConcurrency))) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + // + // Zero all flags (except for table create flags, as these may have + // default values) and table create parameters. + // + + KeysLoadFlags->AsULong = 0; + TableCompileFlags->AsULong = 0; + + for (; CurrentArg < NumberOfArguments; CurrentArg++, ArgW++) { + + String->Buffer = Arg = *ArgW; + String->Length = GET_LENGTH(String); + String->MaximumLength = GET_MAX_LENGTH(String); + + // + // If the argument doesn't start with two dashes, report it. + // + + InvalidPrefix = ( + (String->Length <= (sizeof(L'-') + sizeof(L'-'))) || + (!(*Arg++ == L'-' && *Arg++ == L'-')) + ); + + if (InvalidPrefix) { + goto InvalidArg; + } + + // + // Advance the buffer past the two dashes and update lengths + // accordingly. + // + + String->Buffer += 2; + String->Length -= (sizeof(WCHAR) * 2); + String->MaximumLength -= (sizeof(WCHAR) * 2); + + // + // Try each argument extraction routine for this argument; if it + // indicates an error, report it and break out of the loop. If it + // indicates it successfully extracted the argument (Result == S_OK), + // continue onto the next argument. Otherwise, verify it indicates + // that no argument was extracted (S_FALSE), then try the next routine. + // + +#define TRY_EXTRACT_ARG(Name) \ + Result = TryExtractArg##Name##Flags(Rtl, Allocator, String, Name##Flags); \ + if (FAILED(Result)) { \ + PH_ERROR(ExtractTableCreateArgs_TryExtractArg##Name##Flags, Result); \ + break; \ + } else if (Result == S_OK) { \ + continue; \ + } else { \ + ASSERT(Result == S_FALSE); \ + } + + TRY_EXTRACT_ARG(ContextTableCreate); + TRY_EXTRACT_ARG(KeysLoad); + TRY_EXTRACT_ARG(TableCreate); + TRY_EXTRACT_ARG(TableCompile); + + // + // If we get here, none of the previous extraction routines claimed the + // argument, so, provide the table create parameters extraction routine + // an opportunity to run. + // + + Result = TryExtractArgTableCreateParameters(Rtl, + String, + TableCreateParameters); + + if (Result == S_OK) { + continue; + } + + if (Result == PH_E_COMMANDLINE_ARG_MISSING_VALUE) { + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (FAILED(Result)) { + PH_ERROR(ExtractBulkCreateArgs_TryExtractTableCreateParams, Result); + break; + } + + ASSERT(Result == S_FALSE); + +InvalidArg: + + // + // If we get here, we don't recognize the argument. + // + + Result = PH_E_INVALID_COMMANDLINE_ARG; + PH_MESSAGE_ARGS(Result, String); + break; + } + + if (SUCCEEDED(Result)) { + + // + // Initialize the debugger flags from the table create flags, initialize + // the debugger context, then, maybe wait for a debugger attach. This + // is a no-op on Windows, or if no debugger has been requested. + // + + Flags.AsULong = 0; + Flags.WaitForGdb = (TableCreateFlags->WaitForGdb != FALSE); + Flags.WaitForCudaGdb = (TableCreateFlags->WaitForCudaGdb != FALSE); + Flags.UseGdbForHostDebugging = ( + TableCreateFlags->UseGdbForHostDebugging != FALSE + ); + + DebuggerContext = &Rtl->DebuggerContext; + DebuggerResult = InitializeDebuggerContext(DebuggerContext, &Flags); + + if (FAILED(DebuggerResult)) { + + PH_ERROR(InitializeDebuggerContext, DebuggerResult); + + // + // *Now* we can propagate the debugger result back as the primary + // result, which ensures the cleanup code below runs. + // + + Result = DebuggerResult; + + } else { + + // + // Debugger context was successfully initialized, so, maybe wait for + // a debugger to attach (depending on what flags were supplied). + // + + Result = MaybeWaitForDebuggerAttach(DebuggerContext); + if (FAILED(Result)) { + PH_ERROR(MaybeWaitForDebuggerAttach, Result); + } + + } + } + + // + // If we failed, clean up the table create parameters. If that fails, + // report the error, then replace our return value error code with that + // error code. + // + + if (FAILED(Result)) { + CleanupResult = CleanupTableCreateParameters(TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + Result = CleanupResult; + } + } + + return Result; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashContextTableCreate.c b/src/PerfectHash/PerfectHashContextTableCreate.c index 7333bd38..46b45d2c 100644 --- a/src/PerfectHash/PerfectHashContextTableCreate.c +++ b/src/PerfectHash/PerfectHashContextTableCreate.c @@ -44,6 +44,64 @@ HRESULT typedef PREPARE_TABLE_CREATE_CSV_FILE *PPREPARE_TABLE_CREATE_CSV_FILE; extern PREPARE_TABLE_CREATE_CSV_FILE PrepareTableCreateCsvFile; +static +VOID +PerfectHashLogTableCreateFailure( + _In_opt_ PCUNICODE_STRING KeysPath, + _In_ ULONG Stage, + _In_ HRESULT Result + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[1024]; + ULONG PathChars = 0; + + if (GetEnvironmentVariableW(L"PH_LOG_TABLE_CREATE_FAILURES", NULL, 0) == 0) { + return; + } + + if (ARGUMENT_PRESENT(KeysPath) && KeysPath->Buffer) { + PathChars = KeysPath->Length / sizeof(WCHAR); + } + + LogHandle = CreateFileW(L"PerfectHashTableCreateFailures.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Result=0x%08lX Path=%.*S\r\n", + Stage, + Result, + (int)PathChars, + (PathChars ? KeysPath->Buffer : L"")); + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(KeysPath); + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(Result); +#endif +} + // // Method implementations. // @@ -169,6 +227,7 @@ Return Value: ASSIGNED_MEMORY_COVERAGE EmptyCoverage; PASSIGNED_MEMORY_COVERAGE Coverage; BOOLEAN UnknownTableCreateResult = FALSE; + ULONG Stage = 0; // // Validate arguments. @@ -214,16 +273,19 @@ Return Value: return E_INVALIDARG; } else { + Stage = 1; Result = Context->Vtbl->SetBaseOutputDirectory(Context, BaseOutputDirectory); if (FAILED(Result)) { PH_ERROR(PerfectHashContextSetBaseOutputDirectory, Result); + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } Result = Context->Vtbl->GetBaseOutputDirectory(Context, &BaseOutputDir); if (FAILED(Result)) { PH_ERROR(PerfectHashContextGetBaseOutputDirectory, Result); + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } RELEASE(BaseOutputDir); @@ -250,16 +312,19 @@ Return Value: ZeroStruct(EmptyCoverage); MonitorLowMemory = (ContextTableCreateFlags.MonitorLowMemory != FALSE); + Stage = 2; Result = PerfectHashContextInitializeLowMemoryMonitor( Context, MonitorLowMemory ); if (FAILED(Result)) { PH_ERROR(PerfectHashContextInitializeLowMemoryMonitor, Result); + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } #ifdef PH_WINDOWS + Stage = 3; Result = PerfectHashContextTryPrepareCallbackTableValuesFile( Context, TableCreateFlags @@ -281,6 +346,7 @@ Return Value: NumberOfPages = TABLE_CREATE_CSV_ROW_BUFFER_NUMBER_OF_PAGES; } + Stage = 4; Result = Rtl->Vtbl->CreateBuffer(Rtl, &ProcessHandle, NumberOfPages, @@ -290,6 +356,7 @@ Return Value: if (FAILED(Result)) { Result = E_OUTOFMEMORY; + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); return Result; } @@ -302,20 +369,25 @@ Return Value: // Get a reference to the stdout handle. // - if (!Context->OutputHandle) { - Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + Stage = 5; + if (!Silent) { if (!Context->OutputHandle) { - SYS_ERROR(GetStdHandle); - goto Error; + Context->OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE); + if (!Context->OutputHandle || + Context->OutputHandle == INVALID_HANDLE_VALUE) { + Context->OutputHandle = NULL; + Silent = TRUE; + } } - } - OutputHandle = Context->OutputHandle; + OutputHandle = Context->OutputHandle; + } // // Prepare the .csv file if applicable. // + Stage = 6; if (TableCreateFlags.DisableCsvOutputFile == FALSE) { NumberOfRows = 1; Result = PrepareTableCreateCsvFile(Context, @@ -350,6 +422,7 @@ Return Value: // We pass this value to Keys->Vtbl->Load(). // + Stage = 7; Result = PerfectHashContextInitializeKeySize(&KeysLoadFlags, TableCreateParameters, &KeySizeInBytes); @@ -362,6 +435,7 @@ Return Value: // Create a keys instance. // + Stage = 8; Result = Context->Vtbl->CreateInstance(Context, NULL, &IID_PERFECT_HASH_KEYS, @@ -376,6 +450,7 @@ Return Value: // Load the keys. // + Stage = 9; Result = Keys->Vtbl->Load(Keys, &KeysLoadFlags, KeysPath, @@ -386,6 +461,7 @@ Return Value: goto Error; } + Stage = 10; Result = Keys->Vtbl->GetFlags(Keys, sizeof(KeysFlags), &KeysFlags); @@ -395,6 +471,7 @@ Return Value: goto Error; } + Stage = 11; Result = Keys->Vtbl->GetAddress(Keys, &KeysBaseAddress, &NumberOfKeys); @@ -429,6 +506,7 @@ Return Value: ASSERT(Table == NULL); + Stage = 12; Result = Context->Vtbl->CreateInstance(Context, NULL, &IID_PERFECT_HASH_TABLE, @@ -439,6 +517,7 @@ Return Value: goto Error; } + Stage = 13; Result = Table->Vtbl->Create(Table, Context, AlgorithmId, @@ -477,6 +556,7 @@ Return Value: if (!ContextTableCreateFlags.SkipTestAfterCreate) { + Stage = 14; Result = Table->Vtbl->Test(Table, Keys, FALSE); if (FAILED(Result)) { @@ -491,6 +571,7 @@ Return Value: if (ContextTableCreateFlags.Compile) { + Stage = 15; Result = Table->Vtbl->Compile(Table, &TableCompileFlags, CpuArchId); @@ -506,6 +587,7 @@ Return Value: // Write the .csv row if applicable. // + Stage = 16; if (TableCreateFlags.DisableCsvOutputFile != FALSE) { goto End; } @@ -550,6 +632,8 @@ Return Value: Result = E_UNEXPECTED; } + PerfectHashLogTableCreateFailure(KeysPath, Stage, Result); + // // Intentional follow-on to End. // diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c index a2ea2ffb..66434c5a 100644 --- a/src/PerfectHash/PerfectHashServer.c +++ b/src/PerfectHash/PerfectHashServer.c @@ -1,6 +1,6 @@ /*++ -Copyright (c) 2018-2025 Trent Nelson +Copyright (c) 2018-2026 Trent Nelson Module Name: @@ -62,7 +62,6 @@ typedef struct _PERFECT_HASH_SERVER_BULK_REQUEST { ULONG PerFileMaximumConcurrency; ULONG Padding1; PPERFECT_HASH_SERVER Server; - PPERFECT_HASH_CONTEXT ParseContext; LPWSTR *ArgvW; PWSTR CommandLineBuffer; PERFECT_HASH_CONTEXT_BULK_CREATE_FLAGS ContextBulkCreateFlags; @@ -99,6 +98,8 @@ typedef struct _PERFECT_HASH_SERVER_BULK_WORK_ITEM { ULONG Padding1; UNICODE_STRING KeysPath; PWSTR KeysPathBuffer; + ULONG LastError; + ULONG Padding2; } PERFECT_HASH_SERVER_BULK_WORK_ITEM; typedef PERFECT_HASH_SERVER_BULK_WORK_ITEM *PPERFECT_HASH_SERVER_BULK_WORK_ITEM; @@ -167,6 +168,15 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( _Out_ PBOOLEAN ArgvWOwned ); +static +HRESULT +PerfectHashServerDispatchTableCreateRequest( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe, + _In_ ULONG NumberOfArguments, + _In_ LPWSTR *ArgvW, + _In_ LPWSTR CommandLine + ); + static HRESULT PerfectHashServerPrepareErrorPayload( @@ -263,7 +273,7 @@ Return Value: UNREFERENCED_PARAMETER(Server); return E_NOTIMPL; #else - HRESULT Result; + HRESULT Result = E_UNEXPECTED; PPERFECT_HASH_CONTEXT_IOCP ContextIocp; if (!ARGUMENT_PRESENT(Server)) { @@ -394,7 +404,7 @@ PerfectHashServerSetMaximumConcurrency( ULONG MaximumConcurrency ) { - HRESULT Result; + HRESULT Result = E_UNEXPECTED; if (!ARGUMENT_PRESENT(Server)) { return E_POINTER; @@ -747,6 +757,7 @@ PerfectHashServerCreatePipes( ULONG PipeCount; HRESULT Result; PALLOCATOR Allocator; + PRTL Rtl; PPERFECT_HASH_CONTEXT_IOCP ContextIocp; if (!ARGUMENT_PRESENT(Server)) { @@ -759,8 +770,9 @@ PerfectHashServerCreatePipes( ContextIocp = Server->ContextIocp; Allocator = Server->Allocator; + Rtl = Server->Rtl; - if (!ContextIocp || !Allocator) { + if (!ContextIocp || !Allocator || !Rtl) { return E_UNEXPECTED; } @@ -1433,7 +1445,7 @@ PerfectHashServerCompleteBulkRequest( CloseHandle(Request->CompletionEvent); } - if (Request->ParseContext) { + if (Request->TableCreateParameters.SizeOfStruct != 0) { HRESULT CleanupResult; CleanupResult = CleanupTableCreateParameters( @@ -1442,8 +1454,6 @@ PerfectHashServerCompleteBulkRequest( if (FAILED(CleanupResult)) { PH_ERROR(CleanupTableCreateParameters, CleanupResult); } - - Request->ParseContext->Vtbl->Release(Request->ParseContext); } if (Request->ArgvW) { @@ -1492,9 +1502,11 @@ PerfectHashServerLogBulkCreateException( DWORD BytesWritten; ULONG NodeIndex = 0; ULONG_PTR Address = 0; + ULONG_PTR Offset = 0; DWORD Code = 0; DWORD ThreadId; CHAR Buffer[256]; + HMODULE Module = NULL; if (!ARGUMENT_PRESENT(ExceptionPointers)) { return; @@ -1523,15 +1535,28 @@ PerfectHashServerLogBulkCreateException( Code = ExceptionPointers->ExceptionRecord->ExceptionCode; Address = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress; - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "Stage=%lu Code=0x%08lX Address=0x%p ThreadId=%lu Node=%lu\r\n", - Stage, - Code, - (PVOID)Address, - ThreadId, - NodeIndex); + if (GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)ExceptionPointers->ExceptionRecord->ExceptionAddress, + &Module)) { + Offset = Address - (ULONG_PTR)Module; + } + + _snprintf_s( + Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Code=0x%08lX Address=0x%p Module=0x%p Offset=0x%Ix " + "ThreadId=%lu Node=%lu\r\n", + Stage, + Code, + (PVOID)Address, + Module, + Offset, + ThreadId, + NodeIndex + ); WriteFile(LogHandle, Buffer, @@ -1547,6 +1572,69 @@ PerfectHashServerLogBulkCreateException( #endif } +static +VOID +PerfectHashServerLogBulkCreateFailure( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_ HRESULT Result + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[1024]; + ULONG PathChars = 0; + + if (GetEnvironmentVariableW(L"PH_LOG_BULK_CREATE_FAILURES", NULL, 0) == 0) { + return; + } + + if (!WorkItem || !WorkItem->KeysPath.Buffer) { + return; + } + + PathChars = WorkItem->KeysPath.Length / sizeof(WCHAR); + + LogHandle = CreateFileW(L"PerfectHashServerBulkCreateFailures.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Result=0x%08lX LastError=%lu (0x%08lX) " + "Path=%.*S\r\n", + Stage, + Result, + WorkItem->LastError, + WorkItem->LastError, + (int)PathChars, + WorkItem->KeysPath.Buffer); + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(WorkItem); + UNREFERENCED_PARAMETER(Result); +#endif +} + static LONG PerfectHashServerBulkCreateExceptionFilter( @@ -1584,20 +1672,21 @@ PerfectHashServerBulkCreateWorkItemCallback( ULONG Stage = 0; BOOLEAN ExceptionRaised = FALSE; - UNREFERENCED_PARAMETER(ContextIocp); UNREFERENCED_PARAMETER(NumberOfBytesTransferred); - if (!Success) { - Failure = HRESULT_FROM_WIN32(GetLastError()); - } else { - Failure = S_OK; - } - WorkItem = CONTAINING_RECORD(Overlapped, PERFECT_HASH_SERVER_BULK_WORK_ITEM, Iocp.Overlapped); Request = WorkItem->Request; Allocator = (Request && Request->Server) ? Request->Server->Allocator : NULL; + WorkItem->LastError = ERROR_SUCCESS; + + if (!Success) { + WorkItem->LastError = GetLastError(); + Failure = HRESULT_FROM_WIN32(WorkItem->LastError); + } else { + Failure = S_OK; + } __try { Stage = 1; @@ -1615,12 +1704,8 @@ PerfectHashServerBulkCreateWorkItemCallback( } Stage = 2; - Result = Request->Server->Vtbl->CreateInstance( - Request->Server, - NULL, - &IID_PERFECT_HASH_CONTEXT, - &Context - ); + Result = PerfectHashContextIocpCreateTableContext(ContextIocp, + &Context); if (FAILED(Result)) { Failure = Result; InterlockedIncrement(&Request->FailedWorkItems); @@ -1681,11 +1766,6 @@ PerfectHashServerBulkCreateWorkItemCallback( goto Complete; } - PerfectHashContextApplyThreadpoolPriorities( - Context, - &Request->TableCreateParameters - ); - Stage = 4; Result = PerfectHashContextInitializeRng( Context, @@ -1726,6 +1806,7 @@ PerfectHashServerBulkCreateWorkItemCallback( PerfectHashTlsClearContextIfActive(&LocalTlsContext); if (FAILED(Result)) { + WorkItem->LastError = GetLastError(); Failure = Result; InterlockedIncrement(&Request->FailedWorkItems); InterlockedCompareExchange(&Request->FirstFailure, @@ -1759,6 +1840,10 @@ PerfectHashServerBulkCreateWorkItemCallback( Complete: + if (FAILED(Failure)) { + PerfectHashServerLogBulkCreateFailure(Stage, WorkItem, Failure); + } + if (Context) { Context->Vtbl->Release(Context); } @@ -2106,7 +2191,6 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( PALLOCATOR Allocator; PRTL Rtl; PPERFECT_HASH_SERVER Server; - PPERFECT_HASH_CONTEXT ParseContext = NULL; PPERFECT_HASH_CONTEXT_IOCP ContextIocp; PPERFECT_HASH_SERVER_BULK_REQUEST Request = NULL; PPERFECT_HASH_SERVER_BULK_RESULT BulkResult = NULL; @@ -2123,6 +2207,7 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags = { 0 }; PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters = { 0 }; LPWSTR *AdjustedArgvW = NULL; + BOOLEAN ParsedArgs = FALSE; if (!ARGUMENT_PRESENT(Pipe)) { return E_POINTER; @@ -2147,21 +2232,13 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( *ArgvWOwned = FALSE; - Result = Server->Vtbl->CreateInstance(Server, - NULL, - &IID_PERFECT_HASH_CONTEXT, - &ParseContext); - if (FAILED(Result)) { - return Result; - } - Result = LoadDefaultTableCreateFlags(&TableCreateFlags); if (FAILED(Result)) { goto Error; } TableCreateParameters.SizeOfStruct = sizeof(TableCreateParameters); - TableCreateParameters.Allocator = ParseContext->Allocator; + TableCreateParameters.Allocator = Allocator; AdjustedArguments = NumberOfArguments + 1; AdjustedArgvW = Allocator->Vtbl->Calloc(Allocator, @@ -2177,8 +2254,8 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( ArgvW, NumberOfArguments * sizeof(*AdjustedArgvW)); - Result = ParseContext->Vtbl->ExtractBulkCreateArgsFromArgvW( - ParseContext, + Result = ContextIocp->Vtbl->ExtractBulkCreateArgsFromArgvW( + ContextIocp, AdjustedArguments, AdjustedArgvW, CommandLine, @@ -2197,6 +2274,7 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( if (FAILED(Result)) { goto Error; } + ParsedArgs = TRUE; ContextTableCreateFlags.AsULong = 0; ContextTableCreateFlags.SkipTestAfterCreate = @@ -2221,7 +2299,6 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( Request->SizeOfStruct = sizeof(*Request); Request->Server = Server; - Request->ParseContext = ParseContext; Request->ContextBulkCreateFlags = ContextBulkCreateFlags; Request->ContextTableCreateFlags = ContextTableCreateFlags; Request->KeysLoadFlags = KeysLoadFlags; @@ -2438,14 +2515,235 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Request); } - if (ParseContext) { + if (ParsedArgs) { CleanupResult = CleanupTableCreateParameters(&TableCreateParameters); if (FAILED(CleanupResult)) { PH_ERROR(CleanupTableCreateParameters, CleanupResult); Result = CleanupResult; } + } + + return Result; +} - ParseContext->Vtbl->Release(ParseContext); +static +HRESULT +PerfectHashServerDispatchTableCreateRequest( + PPERFECT_HASH_SERVER_PIPE Pipe, + ULONG NumberOfArguments, + LPWSTR *ArgvW, + LPWSTR CommandLine + ) +{ + HRESULT Result; + HRESULT CleanupResult; + ULONG CommandLineChars; + ULONG CommandLineBytes; + ULONG AdjustedArguments; + ULONG MaximumConcurrency = 0; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_SERVER Server; + PPERFECT_HASH_CONTEXT Context = NULL; + PPERFECT_HASH_CONTEXT_IOCP ContextIocp; + PPERFECT_HASH_IOCP_NODE Node = NULL; + LPWSTR *AdjustedArgvW = NULL; + LPWSTR *ArgvToUse = NULL; + PWSTR CommandLineBuffer = NULL; + UNICODE_STRING KeysPath = { 0 }; + UNICODE_STRING BaseOutputDirectory = { 0 }; + PERFECT_HASH_ALGORITHM_ID AlgorithmId = 0; + PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId = 0; + PERFECT_HASH_MASK_FUNCTION_ID MaskFunctionId = 0; + PERFECT_HASH_CONTEXT_TABLE_CREATE_FLAGS ContextTableCreateFlags = { 0 }; + PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags = { 0 }; + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags = { 0 }; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParameters = { 0 }; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PERFECT_HASH_TLS_CONTEXT LocalTlsContext = { 0 }; + BOOLEAN ParsedArgs = FALSE; + BOOLEAN TlsContextSet = FALSE; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Server = Pipe->Server; + if (!Server) { + return E_UNEXPECTED; + } + + ContextIocp = Server->ContextIocp; + Allocator = Server->Allocator; + Rtl = Server->Rtl; + + if (!ContextIocp || !Allocator || !Rtl) { + return E_UNEXPECTED; + } + + if (!ARGUMENT_PRESENT(ArgvW) || NumberOfArguments == 0) { + return E_INVALIDARG; + } + + Result = LoadDefaultTableCreateFlags(&TableCreateFlags); + if (FAILED(Result)) { + return Result; + } + + TableCreateParameters.SizeOfStruct = sizeof(TableCreateParameters); + TableCreateParameters.Allocator = Allocator; + + ArgvToUse = ArgvW; + AdjustedArguments = NumberOfArguments; + + if (ArgvW[0][0] == L'-' || ArgvW[0][0] == L'/') { + AdjustedArgvW = Allocator->Vtbl->Calloc(Allocator, + NumberOfArguments + 1, + sizeof(*AdjustedArgvW)); + if (!AdjustedArgvW) { + Result = E_OUTOFMEMORY; + goto Error; + } + + AdjustedArgvW[0] = L"PerfectHashTableCreate"; + CopyMemory(&AdjustedArgvW[1], + ArgvW, + NumberOfArguments * sizeof(*AdjustedArgvW)); + ArgvToUse = AdjustedArgvW; + AdjustedArguments = NumberOfArguments + 1; + } + + Result = ContextIocp->Vtbl->ExtractTableCreateArgsFromArgvW( + ContextIocp, + AdjustedArguments, + ArgvToUse, + CommandLine, + &KeysPath, + &BaseOutputDirectory, + &AlgorithmId, + &HashFunctionId, + &MaskFunctionId, + &MaximumConcurrency, + &ContextTableCreateFlags, + &KeysLoadFlags, + &TableCreateFlags, + &TableCompileFlags, + &TableCreateParameters + ); + if (FAILED(Result)) { + goto Error; + } + ParsedArgs = TRUE; + + Result = PerfectHashContextIocpCreateTableContext(ContextIocp, + &Context); + if (FAILED(Result)) { + goto Error; + } + + if (CommandLine) { + CommandLineChars = (ULONG)wcslen(CommandLine); + if (CommandLineChars > ((ULONG_MAX / sizeof(WCHAR)) - 1)) { + Result = E_INVALIDARG; + goto Error; + } + + CommandLineBytes = (CommandLineChars + 1) * sizeof(WCHAR); + CommandLineBuffer = (PWSTR)Allocator->Vtbl->Calloc(Allocator, + 1, + CommandLineBytes); + if (!CommandLineBuffer) { + Result = E_OUTOFMEMORY; + goto Error; + } + + CopyMemory(CommandLineBuffer, CommandLine, CommandLineBytes); + Context->CommandLineW = CommandLineBuffer; + } + + if (ContextIocp->NodeCount > 0 && ContextIocp->Nodes) { + Node = &ContextIocp->Nodes[0]; + if (Node->IoCompletionPort) { + Context->FileWorkIoCompletionPort = Node->IoCompletionPort; + } + } + + SetContextSkipContextFileWork(Context); + + Result = PerfectHashContextInitializeFunctionHookCallbackDll( + Context, + &TableCreateFlags, + &TableCreateParameters + ); + if (FAILED(Result)) { + goto Error; + } + + if (MaximumConcurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency(Context, + MaximumConcurrency); + if (FAILED(Result)) { + if (Result == PH_E_SET_MAXIMUM_CONCURRENCY_FAILED && + GetLastError() == ERROR_ACCESS_DENIED) { + Result = S_OK; + } + } + if (FAILED(Result)) { + goto Error; + } + } + + Result = PerfectHashContextInitializeRng(Context, + &TableCreateFlags, + &TableCreateParameters); + if (FAILED(Result)) { + goto Error; + } + + TlsContext = PerfectHashTlsGetOrSetContext(&LocalTlsContext); + TlsContext->Context = Context; + TlsContextSet = TRUE; + + Result = Context->Vtbl->TableCreate(Context, + &KeysPath, + &BaseOutputDirectory, + AlgorithmId, + HashFunctionId, + MaskFunctionId, + &ContextTableCreateFlags, + &KeysLoadFlags, + &TableCreateFlags, + &TableCompileFlags, + &TableCreateParameters); + +Error: + + if (TlsContextSet) { + PerfectHashTlsClearContextIfActive(&LocalTlsContext); + } + + if (ParsedArgs) { + CleanupResult = CleanupTableCreateParameters(&TableCreateParameters); + if (FAILED(CleanupResult)) { + PH_ERROR(CleanupTableCreateParameters, CleanupResult); + if (SUCCEEDED(Result)) { + Result = CleanupResult; + } + } + } + + if (Context) { + Context->Vtbl->Release(Context); + } + + if (CommandLineBuffer && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&CommandLineBuffer); + } + + if (AdjustedArgvW && Allocator) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); } return Result; @@ -2457,16 +2755,13 @@ PerfectHashServerDispatchRequest( PPERFECT_HASH_SERVER_PIPE Pipe ) { - HRESULT Result; + HRESULT Result = E_UNEXPECTED; PPERFECT_HASH_CONTEXT_IOCP ContextIocp; PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; PALLOCATOR Allocator; LPWSTR CommandLine; LPWSTR *ArgvW; - LPWSTR *AdjustedArgvW; - LPWSTR *ArgvToUse; ULONG NumberOfArguments; - ULONG AdjustedArguments; PRTL Rtl; BOOLEAN ArgvWOwned; @@ -2491,10 +2786,7 @@ PerfectHashServerDispatchRequest( Pipe->ResponseHeader.PayloadLength = 0; ArgvW = NULL; - AdjustedArgvW = NULL; - ArgvToUse = NULL; NumberOfArguments = 0; - AdjustedArguments = 0; CommandLine = NULL; ArgvWOwned = FALSE; @@ -2544,39 +2836,16 @@ PerfectHashServerDispatchRequest( break; } - ArgvToUse = ArgvW; - AdjustedArguments = NumberOfArguments; - - if (RequestType != - PerfectHashBulkCreateDirectoryServerRequestType && - (ArgvW[0][0] == L'-' || ArgvW[0][0] == L'/')) { - AdjustedArgvW = Allocator->Vtbl->Calloc( - Allocator, - NumberOfArguments + 1, - sizeof(*AdjustedArgvW) - ); - if (!AdjustedArgvW) { - Result = E_OUTOFMEMORY; - break; - } - - AdjustedArgvW[0] = L"PerfectHashServer"; - CopyMemory(&AdjustedArgvW[1], - ArgvW, - NumberOfArguments * sizeof(*AdjustedArgvW)); - ArgvToUse = AdjustedArgvW; - AdjustedArguments = NumberOfArguments + 1; - } - if (RequestType == PerfectHashTableCreateServerRequestType) { - Result = ContextIocp->Vtbl->TableCreateArgvW( - ContextIocp, - AdjustedArguments, - ArgvToUse, + Result = PerfectHashServerDispatchTableCreateRequest( + Pipe, + NumberOfArguments, + ArgvW, CommandLine ); } else if (RequestType == - PerfectHashBulkCreateDirectoryServerRequestType) { + PerfectHashBulkCreateDirectoryServerRequestType || + RequestType == PerfectHashBulkCreateServerRequestType) { Result = PerfectHashServerDispatchBulkCreateDirectoryRequest( Pipe, NumberOfArguments, @@ -2584,13 +2853,6 @@ PerfectHashServerDispatchRequest( CommandLine, &ArgvWOwned ); - } else { - Result = ContextIocp->Vtbl->BulkCreateArgvW( - ContextIocp, - AdjustedArguments, - ArgvToUse, - CommandLine - ); } break; @@ -2600,10 +2862,6 @@ PerfectHashServerDispatchRequest( break; } - if (AdjustedArgvW && Allocator) { - Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&AdjustedArgvW); - } - if (ArgvW && !ArgvWOwned) { LocalFree(ArgvW); } diff --git a/src/PerfectHash/PerfectHashTableCreate.c b/src/PerfectHash/PerfectHashTableCreate.c index f7797359..81d2f3e7 100644 --- a/src/PerfectHash/PerfectHashTableCreate.c +++ b/src/PerfectHash/PerfectHashTableCreate.c @@ -427,7 +427,8 @@ Return Value: if (AlgorithmId == PerfectHashChm01AlgorithmId && Table->Context && Table->Context->FileWorkIoCompletionPort && - IsChm01AsyncEnabled()) { + (IsChm01AsyncEnabled() || + SkipThreadpoolInitialization(Table->Context))) { Result = CreatePerfectHashTableImplChm01Async( Table, Table->Context->FileWorkIoCompletionPort diff --git a/src/PerfectHash/PerfectHashTls.h b/src/PerfectHash/PerfectHashTls.h index 149f05a9..f428c1a4 100644 --- a/src/PerfectHash/PerfectHashTls.h +++ b/src/PerfectHash/PerfectHashTls.h @@ -45,11 +45,18 @@ typedef union _PERFECT_HASH_TLS_CONTEXT_FLAGS { ULONG GlobalComponentLockAcquired:1; + // + // When set, indicates new PERFECT_HASH_CONTEXT instances should skip + // threadpool initialization (IOCP-native contexts). + // + + ULONG CreateContextWithoutThreadpool:1; + // // Unused bits. (Consume these before the Unused2 bits.) // - ULONG Unused1:6; + ULONG Unused1:5; // // The following bits, when set, prevent the global component logic diff --git a/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c b/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c index 294a5b82..d59663a9 100644 --- a/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c +++ b/src/PerfectHashBulkCreateExe/PerfectHashBulkCreateExe.c @@ -25,6 +25,7 @@ Module Name: typedef DWORD MINIDUMP_TYPE; #define MiniDumpWithDataSegs ((MINIDUMP_TYPE)0x00000001) +#define MiniDumpIgnoreInaccessibleMemory ((MINIDUMP_TYPE)0x00020000) #pragma warning(push) #pragma warning(disable: 4820) @@ -58,6 +59,79 @@ typedef BOOL (WINAPI *PMINIDUMP_WRITE_DUMP)( PMINIDUMP_CALLBACK_INFORMATION CallbackParam ); +static HMODULE BulkCreateDbgHelpModule = NULL; +static PMINIDUMP_WRITE_DUMP BulkCreateMiniDumpWriteDump = NULL; +static WCHAR BulkCreateCrashDir[MAX_PATH]; + +static +VOID +BuildCrashPath( + _In_opt_ PCWSTR CrashDir, + _In_ PCWSTR FileName, + _Out_writes_(BufferChars) PWSTR Buffer, + _In_ ULONG BufferChars + ) +{ + size_t Length; + + if (!CrashDir || !*CrashDir) { + wcscpy_s(Buffer, BufferChars, FileName); + return; + } + + Length = wcslen(CrashDir); + if (CrashDir[Length - 1] == L'\\' || CrashDir[Length - 1] == L'/') { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, FileName); + } else { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, L"\\"); + wcscat_s(Buffer, BufferChars, FileName); + } +} + +static +VOID +InitializeBulkCreateCrashDir( + VOID + ) +{ + DWORD Length; + + BulkCreateCrashDir[0] = L'\0'; + Length = GetEnvironmentVariableW(L"PH_BULK_CREATE_CRASH_DIR", + BulkCreateCrashDir, + ARRAYSIZE(BulkCreateCrashDir)); + if (Length == 0 || Length >= ARRAYSIZE(BulkCreateCrashDir)) { + BulkCreateCrashDir[0] = L'\0'; + } else { + BulkCreateCrashDir[Length] = L'\0'; + } +} + +static +VOID +PreloadBulkCreateDbgHelp( + VOID + ) +{ + if (BulkCreateMiniDumpWriteDump) { + return; + } + + BulkCreateDbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); + if (!BulkCreateDbgHelpModule) { + return; + } + +#pragma warning(push) +#pragma warning(disable: 4191) + BulkCreateMiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( + GetProcAddress(BulkCreateDbgHelpModule, "MiniDumpWriteDump") + ); +#pragma warning(pop) +} + static LONG WINAPI @@ -69,6 +143,7 @@ BulkCreateUnhandledExceptionFilter( HANDLE FileHandle = INVALID_HANDLE_VALUE; PMINIDUMP_WRITE_DUMP MiniDumpWriteDump; MINIDUMP_EXCEPTION_INFORMATION ExceptionInfo; + PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam = NULL; MINIDUMP_TYPE DumpType; BOOL DumpResult = FALSE; BOOL FallbackResult = FALSE; @@ -89,24 +164,32 @@ BulkCreateUnhandledExceptionFilter( CHAR ModulePathA[MAX_PATH]; DWORD ModulePathLength; INT ModulePathChars; + WCHAR DumpPath[MAX_PATH]; + WCHAR LogPath[MAX_PATH]; - DbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); - if (!DbgHelpModule) { - goto End; + BuildCrashPath(BulkCreateCrashDir, + L"PerfectHashBulkCreateCrash.dmp", + DumpPath, + ARRAYSIZE(DumpPath)); + + BuildCrashPath(BulkCreateCrashDir, + L"PerfectHashBulkCreateCrash.log", + LogPath, + ARRAYSIZE(LogPath)); + + if (!BulkCreateMiniDumpWriteDump) { + PreloadBulkCreateDbgHelp(); } -#pragma warning(push) -#pragma warning(disable: 4191) - MiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( - GetProcAddress(DbgHelpModule, "MiniDumpWriteDump") - ); -#pragma warning(pop) + DbgHelpModule = BulkCreateDbgHelpModule; + MiniDumpWriteDump = BulkCreateMiniDumpWriteDump; if (!MiniDumpWriteDump) { - goto End; + DumpError = GetLastError(); + goto LogOnly; } - FileHandle = CreateFileW(L"PerfectHashBulkCreateCrash.dmp", + FileHandle = CreateFileW(DumpPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -115,14 +198,18 @@ BulkCreateUnhandledExceptionFilter( NULL); if (FileHandle == INVALID_HANDLE_VALUE) { - goto End; + DumpError = GetLastError(); + goto LogOnly; } - ExceptionInfo.ThreadId = GetCurrentThreadId(); - ExceptionInfo.ExceptionPointers = ExceptionPointers; - ExceptionInfo.ClientPointers = FALSE; + if (ExceptionPointers) { + ExceptionInfo.ThreadId = GetCurrentThreadId(); + ExceptionInfo.ExceptionPointers = ExceptionPointers; + ExceptionInfo.ClientPointers = FALSE; + ExceptionParam = &ExceptionInfo; + } - DumpType = MiniDumpWithDataSegs; + DumpType = MiniDumpWithDataSegs | MiniDumpIgnoreInaccessibleMemory; ForceFallback = (GetEnvironmentVariableW( L"PH_BULK_CREATE_MINIDUMP_FORCE_FALLBACK", @@ -135,7 +222,7 @@ BulkCreateUnhandledExceptionFilter( GetCurrentProcessId(), FileHandle, DumpType, - &ExceptionInfo, + ExceptionParam, NULL, NULL); } else { @@ -147,10 +234,28 @@ BulkCreateUnhandledExceptionFilter( if (!ForceFallback) { DumpError = GetLastError(); } + + if (DumpError == ERROR_NOACCESS && ExceptionParam) { + DumpResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + NULL, + NULL, + NULL); + if (!DumpResult) { + DumpError = GetLastError(); + } + } + + if (DumpResult) { + goto LogOnly; + } + TriedFallback = TRUE; CloseHandle(FileHandle); - FileHandle = CreateFileW(L"PerfectHashBulkCreateCrash.dmp", + FileHandle = CreateFileW(DumpPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -159,12 +264,12 @@ BulkCreateUnhandledExceptionFilter( NULL); if (FileHandle != INVALID_HANDLE_VALUE) { - DumpType = 0; + DumpType = MiniDumpIgnoreInaccessibleMemory; FallbackResult = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), FileHandle, DumpType, - &ExceptionInfo, + ExceptionParam, NULL, NULL); if (!FallbackResult) { @@ -173,7 +278,9 @@ BulkCreateUnhandledExceptionFilter( } } - LogHandle = CreateFileW(L"PerfectHashBulkCreateCrash.log", +LogOnly: + + LogHandle = CreateFileW(LogPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -229,21 +336,53 @@ BulkCreateUnhandledExceptionFilter( NULL); } - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "ExceptionCode: 0x%08lX\r\n", - ExceptionPointers->ExceptionRecord->ExceptionCode); + if (ExceptionPointers) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); - WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "ExceptionAddress: 0x%p\r\n", - ExceptionPointers->ExceptionRecord->ExceptionAddress); + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); - WriteFile(LogHandle, Buffer, (DWORD)strlen(Buffer), &BytesWritten, NULL); + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } else { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } Frames = RtlCaptureStackBackTrace(0, ARRAYSIZE(BackTrace), @@ -332,9 +471,23 @@ InstallBulkCreateCrashHandler( return; } + InitializeBulkCreateCrashDir(); + PreloadBulkCreateDbgHelp(); + SetUnhandledExceptionFilter(BulkCreateUnhandledExceptionFilter); } +static +VOID +TriggerBulkCreateCrashTest( + _In_ ULONG ExceptionCode + ) +{ + UNREFERENCED_PARAMETER(ExceptionCode); + BulkCreateUnhandledExceptionFilter(NULL); + ExitProcess((ULONG)E_UNEXPECTED); +} + static VOID MaybeTriggerBulkCreateCrashTest( @@ -343,7 +496,6 @@ MaybeTriggerBulkCreateCrashTest( { DWORD Length; WCHAR Buffer[32]; - Length = GetEnvironmentVariableW(L"PH_BULK_CREATE_CRASH_TEST", Buffer, ARRAYSIZE(Buffer)); @@ -354,12 +506,13 @@ MaybeTriggerBulkCreateCrashTest( Buffer[Length] = L'\0'; + InitializeBulkCreateCrashDir(); + if (_wcsicmp(Buffer, L"AV") == 0 || _wcsicmp(Buffer, L"ACCESS_VIOLATION") == 0) { - volatile int *Ptr = (volatile int *)1; - *Ptr = 1; + TriggerBulkCreateCrashTest(STATUS_ACCESS_VIOLATION); } else { - RaiseException(0xE0000001, 0, 0, NULL); + TriggerBulkCreateCrashTest(0xE0000001); } } DECLSPEC_NORETURN diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.c b/src/PerfectHashClientExe/PerfectHashClientExe.c index 5491e6ff..5d87d497 100644 --- a/src/PerfectHashClientExe/PerfectHashClientExe.c +++ b/src/PerfectHashClientExe/PerfectHashClientExe.c @@ -1,6 +1,6 @@ /*++ -Copyright (c) 2018-2025 Trent Nelson +Copyright (c) 2018-2026 Trent Nelson Module Name: @@ -268,10 +268,11 @@ PerfectHashClientWaitForBulkCreateToken( if (ARGUMENT_PRESENT(BulkResult)) { *BulkResult = *Mapping; + Result = S_OK; + } else { + Result = Mapping->Result; } - Result = Mapping->Result; - End: if (Mapping) { @@ -395,6 +396,14 @@ mainCRTStartup( Result = BulkResult.Result; if (FAILED(Result)) { + if (BulkResult.TotalFiles || BulkResult.FailedFiles) { + wprintf(L"Bulk result: total=%lu succeeded=%lu failed=%lu " + L"first=0x%08lX\n", + BulkResult.TotalFiles, + BulkResult.SucceededFiles, + BulkResult.FailedFiles, + (ULONG)BulkResult.FirstFailure); + } if (PerfectHashPrintError != NULL) { PH_ERROR(BulkCreateDirectory, Result); } diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.c b/src/PerfectHashServerExe/PerfectHashServerExe.c index c05b4ced..d772e48c 100644 --- a/src/PerfectHashServerExe/PerfectHashServerExe.c +++ b/src/PerfectHashServerExe/PerfectHashServerExe.c @@ -24,6 +24,7 @@ Module Name: typedef DWORD MINIDUMP_TYPE; #define MiniDumpWithDataSegs ((MINIDUMP_TYPE)0x00000001) +#define MiniDumpIgnoreInaccessibleMemory ((MINIDUMP_TYPE)0x00020000) #pragma warning(push) #pragma warning(disable: 4820) @@ -68,6 +69,79 @@ typedef BOOL (WINAPI *PMINIDUMP_WRITE_DUMP)( PMINIDUMP_CALLBACK_INFORMATION CallbackParam ); +static HMODULE ServerDbgHelpModule = NULL; +static PMINIDUMP_WRITE_DUMP ServerMiniDumpWriteDump = NULL; +static WCHAR ServerCrashDir[MAX_PATH]; + +static +VOID +BuildCrashPath( + _In_opt_ PCWSTR CrashDir, + _In_ PCWSTR FileName, + _Out_writes_(BufferChars) PWSTR Buffer, + _In_ ULONG BufferChars + ) +{ + size_t Length; + + if (!CrashDir || !*CrashDir) { + wcscpy_s(Buffer, BufferChars, FileName); + return; + } + + Length = wcslen(CrashDir); + if (CrashDir[Length - 1] == L'\\' || CrashDir[Length - 1] == L'/') { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, FileName); + } else { + wcscpy_s(Buffer, BufferChars, CrashDir); + wcscat_s(Buffer, BufferChars, L"\\"); + wcscat_s(Buffer, BufferChars, FileName); + } +} + +static +VOID +InitializeServerCrashDir( + VOID + ) +{ + DWORD Length; + + ServerCrashDir[0] = L'\0'; + Length = GetEnvironmentVariableW(L"PH_SERVER_CRASH_DIR", + ServerCrashDir, + ARRAYSIZE(ServerCrashDir)); + if (Length == 0 || Length >= ARRAYSIZE(ServerCrashDir)) { + ServerCrashDir[0] = L'\0'; + } else { + ServerCrashDir[Length] = L'\0'; + } +} + +static +VOID +PreloadServerDbgHelp( + VOID + ) +{ + if (ServerMiniDumpWriteDump) { + return; + } + + ServerDbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); + if (!ServerDbgHelpModule) { + return; + } + +#pragma warning(push) +#pragma warning(disable: 4191) + ServerMiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( + GetProcAddress(ServerDbgHelpModule, "MiniDumpWriteDump") + ); +#pragma warning(pop) +} + static LONG WINAPI @@ -79,6 +153,7 @@ PerfectHashServerUnhandledExceptionFilter( HANDLE FileHandle = INVALID_HANDLE_VALUE; PMINIDUMP_WRITE_DUMP MiniDumpWriteDump; MINIDUMP_EXCEPTION_INFORMATION ExceptionInfo; + PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam = NULL; MINIDUMP_TYPE DumpType; BOOL DumpResult = FALSE; BOOL FallbackResult = FALSE; @@ -99,8 +174,20 @@ PerfectHashServerUnhandledExceptionFilter( CHAR ModulePathA[MAX_PATH]; DWORD ModulePathLength; INT ModulePathChars; + WCHAR DumpPath[MAX_PATH]; + WCHAR LogPath[MAX_PATH]; + + BuildCrashPath(ServerCrashDir, + L"PerfectHashServerCrash.dmp", + DumpPath, + ARRAYSIZE(DumpPath)); - LogHandle = CreateFileW(L"PerfectHashServerCrash.log", + BuildCrashPath(ServerCrashDir, + L"PerfectHashServerCrash.log", + LogPath, + ARRAYSIZE(LogPath)); + + LogHandle = CreateFileW(LogPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -109,45 +196,69 @@ PerfectHashServerUnhandledExceptionFilter( NULL); if (LogHandle != INVALID_HANDLE_VALUE) { - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "ExceptionCode: 0x%08lX\r\n", - ExceptionPointers->ExceptionRecord->ExceptionCode); + if (ExceptionPointers) { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: 0x%08lX\r\n", + ExceptionPointers->ExceptionRecord->ExceptionCode); - WriteFile(LogHandle, - Buffer, - (DWORD)strlen(Buffer), - &BytesWritten, - NULL); + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); - _snprintf_s(Buffer, - sizeof(Buffer), - _TRUNCATE, - "ExceptionAddress: 0x%p\r\n", - ExceptionPointers->ExceptionRecord->ExceptionAddress); + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionAddress: 0x%p\r\n", + ExceptionPointers->ExceptionRecord->ExceptionAddress); - WriteFile(LogHandle, - Buffer, - (DWORD)strlen(Buffer), - &BytesWritten, - NULL); + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); - Module = NULL; - if (GetModuleHandleExW( - GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - (LPCWSTR)ExceptionPointers->ExceptionRecord->ExceptionAddress, - &Module)) { - Offset = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress - - (ULONG_PTR)Module; + Module = NULL; + if (GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)ExceptionPointers->ExceptionRecord->ExceptionAddress, + &Module)) { + Offset = (ULONG_PTR)ExceptionPointers->ExceptionRecord->ExceptionAddress - + (ULONG_PTR)Module; + + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionModule: 0x%p Offset: 0x%Ix\r\n", + Module, + Offset); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + } else { + _snprintf_s(Buffer, + sizeof(Buffer), + _TRUNCATE, + "ExceptionCode: \r\n"); + + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); _snprintf_s(Buffer, sizeof(Buffer), _TRUNCATE, - "ExceptionModule: 0x%p Offset: 0x%Ix\r\n", - Module, - Offset); + "ExceptionAddress: \r\n"); WriteFile(LogHandle, Buffer, @@ -169,23 +280,19 @@ PerfectHashServerUnhandledExceptionFilter( NULL); } - DbgHelpModule = LoadLibraryW(L"DbgHelp.dll"); - if (!DbgHelpModule) { - goto End; + if (!ServerMiniDumpWriteDump) { + PreloadServerDbgHelp(); } -#pragma warning(push) -#pragma warning(disable: 4191) - MiniDumpWriteDump = (PMINIDUMP_WRITE_DUMP)( - GetProcAddress(DbgHelpModule, "MiniDumpWriteDump") - ); -#pragma warning(pop) + DbgHelpModule = ServerDbgHelpModule; + MiniDumpWriteDump = ServerMiniDumpWriteDump; if (!MiniDumpWriteDump) { - goto End; + DumpError = GetLastError(); + goto LogOnly; } - FileHandle = CreateFileW(L"PerfectHashServerCrash.dmp", + FileHandle = CreateFileW(DumpPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -194,14 +301,18 @@ PerfectHashServerUnhandledExceptionFilter( NULL); if (FileHandle == INVALID_HANDLE_VALUE) { - goto End; + DumpError = GetLastError(); + goto LogOnly; } - ExceptionInfo.ThreadId = GetCurrentThreadId(); - ExceptionInfo.ExceptionPointers = ExceptionPointers; - ExceptionInfo.ClientPointers = FALSE; + if (ExceptionPointers) { + ExceptionInfo.ThreadId = GetCurrentThreadId(); + ExceptionInfo.ExceptionPointers = ExceptionPointers; + ExceptionInfo.ClientPointers = FALSE; + ExceptionParam = &ExceptionInfo; + } - DumpType = MiniDumpWithDataSegs; + DumpType = MiniDumpWithDataSegs | MiniDumpIgnoreInaccessibleMemory; ForceFallback = (GetEnvironmentVariableW( L"PH_SERVER_MINIDUMP_FORCE_FALLBACK", @@ -214,7 +325,7 @@ PerfectHashServerUnhandledExceptionFilter( GetCurrentProcessId(), FileHandle, DumpType, - &ExceptionInfo, + ExceptionParam, NULL, NULL); } else { @@ -226,10 +337,28 @@ PerfectHashServerUnhandledExceptionFilter( if (!ForceFallback) { DumpError = GetLastError(); } + + if (DumpError == ERROR_NOACCESS && ExceptionParam) { + DumpResult = MiniDumpWriteDump(GetCurrentProcess(), + GetCurrentProcessId(), + FileHandle, + DumpType, + NULL, + NULL, + NULL); + if (!DumpResult) { + DumpError = GetLastError(); + } + } + + if (DumpResult) { + goto LogOnly; + } + TriedFallback = TRUE; CloseHandle(FileHandle); - FileHandle = CreateFileW(L"PerfectHashServerCrash.dmp", + FileHandle = CreateFileW(DumpPath, GENERIC_WRITE, FILE_SHARE_READ, NULL, @@ -238,12 +367,12 @@ PerfectHashServerUnhandledExceptionFilter( NULL); if (FileHandle != INVALID_HANDLE_VALUE) { - DumpType = 0; + DumpType = MiniDumpIgnoreInaccessibleMemory; FallbackResult = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), FileHandle, DumpType, - &ExceptionInfo, + ExceptionParam, NULL, NULL); if (!FallbackResult) { @@ -252,8 +381,10 @@ PerfectHashServerUnhandledExceptionFilter( } } +LogOnly: + if (LogHandle == INVALID_HANDLE_VALUE) { - LogHandle = CreateFileW(L"PerfectHashServerCrash.log", + LogHandle = CreateFileW(LogPath, FILE_APPEND_DATA, FILE_SHARE_READ, NULL, @@ -415,9 +546,51 @@ InstallPerfectHashServerCrashHandler( return; } + InitializeServerCrashDir(); + PreloadServerDbgHelp(); + SetUnhandledExceptionFilter(PerfectHashServerUnhandledExceptionFilter); } +static +VOID +TriggerServerCrashTest( + _In_ ULONG ExceptionCode + ) +{ + UNREFERENCED_PARAMETER(ExceptionCode); + PerfectHashServerUnhandledExceptionFilter(NULL); + ExitProcess((ULONG)E_UNEXPECTED); +} + +static +VOID +MaybeTriggerServerCrashTest( + VOID + ) +{ + DWORD Length; + WCHAR Buffer[32]; + Length = GetEnvironmentVariableW(L"PH_SERVER_CRASH_TEST", + Buffer, + ARRAYSIZE(Buffer)); + + if (Length == 0 || Length >= ARRAYSIZE(Buffer)) { + return; + } + + Buffer[Length] = L'\0'; + + InitializeServerCrashDir(); + + if (_wcsicmp(Buffer, L"AV") == 0 || + _wcsicmp(Buffer, L"ACCESS_VIOLATION") == 0) { + TriggerServerCrashTest(STATUS_ACCESS_VIOLATION); + } else { + TriggerServerCrashTest(0xE0000001); + } +} + #endif static VOID @@ -666,6 +839,7 @@ mainCRTStartup( ArgvW = CommandLineToArgvW(CommandLineW, &NumberOfArguments); InstallPerfectHashServerCrashHandler(); + MaybeTriggerServerCrashTest(); Result = PerfectHashBootstrap(&ClassFactory, &PerfectHashPrintError, From 532eae1411804185eb3be5a642f9311265d92440 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Thu, 22 Jan 2026 12:48:02 -0800 Subject: [PATCH 3/9] More IOCP progress. --- IOCP-LOGS.md | 15 ++ include/PerfectHash.h | 2 + scripts/iocp-smoke.ps1 | 21 +- scripts/stress-sys32-iocp.ps1 | 20 +- src/PerfectHash/Chm01Async.c | 132 ++++++++++- src/PerfectHash/Chm01Async.h | 13 +- src/PerfectHash/PerfectHashServer.c | 178 +++++++++++++- src/PerfectHash/_dtoa.c | 52 ++++- .../PerfectHashClientExe.c | 220 +++++++++++++++--- 9 files changed, 598 insertions(+), 55 deletions(-) diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index 35447c86..fea6a47e 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -91,3 +91,18 @@ - Diagnosed `TpIsTimerSet` invalid-parameter crash in IOCP worker threads to `SubmitThreadpoolWork(Context->FinishedWork)` from `GraphSolve`; guarded finished-work submissions in `Graph.c`/`GraphCu.c` when threadpool init is skipped. - Rebuilt Debug and reran IOCP bulk create on `perfecthash-keys\\test1` under cdb with breakpoints on `Chm01AsyncGraphComplete` and `PerfectHashServerCompleteBulkRequest`; graph completion fired repeatedly, bulk completion callback fired once, and all five output directories were created (`cdb-server-graphcomplete.log`). - Re-ran IOCP bulk create on `perfecthash-keys\\test1` without cdb (`PERFECT_HASH_IOCP_ASYNC_CHM01=1`); completed successfully and emitted all five output directories (`build-win\\iocp-stress-test1-nocdb`). +- Replaced `CHM01_ASYNC_JOB.Events[5]` with an enum-backed count macro and updated initialization to use named event IDs. +- Added client `--WaitForServer`/`--ConnectTimeout=` and ping request/response support to wait for server readiness before submitting work. +- Updated IOCP scripts to pass `--WaitForServer`/`--ConnectTimeout`; ping check against `PerfectHashServer-PingTest` returned 0 for ping/shutdown/server. +- IOCP bulk run against `perfecthash-keys\\hard` (Debug, concurrency 32) still timed out after ~360s; all 23 output directories were created, server exited, but the client remained waiting until terminated (`build-win\\iocp-stress-hard-latest`). +- OG bulk create (Debug) against `perfecthash-keys\\hard` completed in ~0.82s (cmd /c run, concurrency 32), exit 0. +- IOCP bulk run against `perfecthash-keys\\hard` with concurrency 1 under cdb hit `PerfectHashServerEnqueueBulkRequest`/`WorkItem` callbacks, then crashed in `PerfectHash!PhFree` while printing stats (`PrintCurrentContextStatsChm01` -> `Chm01AsyncFinalizeVerify`); client timed out waiting (`cdb-server-hard1.log`). +- Fixed `_dtoa_Allocator` race by resolving allocator from TLS context in `_dtoa.c` (no TLS var in DLL); Debug rebuild succeeded. +- Re-ran IOCP bulk `hard` with concurrency 1 under cdb; no crash, `PerfectHashServerCompleteBulkRequest` hit, client exited cleanly (`cdb-server-hard1b.log`). + +## 2026-01-22 +- Reproduced IOCP bulk-create hang on `perfecthash-keys\\hard` (Debug, concurrency 32); client waited indefinitely while many per-file outputs were created. +- Added optional bulk-count logging (`PH_LOG_BULK_CREATE_COUNTS`) and CHM01 async job logging (`PH_LOG_CHM01_ASYNC_JOB`) to track request completion and async state transitions. +- Identified `Chm01AsyncWaitJob()` pump loop could block in `GetQueuedCompletionStatus()` after job completion, leaving bulk work items stuck mid-callback. +- Added a dummy IOCP wake post in `Chm01AsyncComplete()` and switched the pump loop to a 100ms timeout to re-check job completion state. +- Re-ran IOCP stress on `perfecthash-keys\\hard` with `Mulshrolate4RX`/concurrency 32; run completed successfully and server shutdown cleanly. diff --git a/include/PerfectHash.h b/include/PerfectHash.h index b2912f76..aa55d49a 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -4488,6 +4488,7 @@ typedef enum _PERFECT_HASH_SERVER_REQUEST_TYPE { PerfectHashTableCreateServerRequestType, PerfectHashBulkCreateServerRequestType, PerfectHashBulkCreateDirectoryServerRequestType, + PerfectHashPingServerRequestType, PerfectHashShutdownServerRequestType, PerfectHashInvalidServerRequestType } PERFECT_HASH_SERVER_REQUEST_TYPE; @@ -4523,6 +4524,7 @@ typedef PERFECT_HASH_SERVER_REQUEST *PPERFECT_HASH_SERVER_REQUEST; #define PERFECT_HASH_SERVER_RESPONSE_FLAG_ERROR_MESSAGE 0x00000001 #define PERFECT_HASH_SERVER_RESPONSE_FLAG_BULK_CREATE_TOKEN 0x00000002 +#define PERFECT_HASH_SERVER_RESPONSE_FLAG_PONG 0x00000004 #define PERFECT_HASH_SERVER_BULK_CREATE_TOKEN_FORMAT \ L"EventHandle=%llu ResultHandle=%llu" diff --git a/scripts/iocp-smoke.ps1 b/scripts/iocp-smoke.ps1 index 2e9051b1..0e6637ea 100644 --- a/scripts/iocp-smoke.ps1 +++ b/scripts/iocp-smoke.ps1 @@ -2,7 +2,9 @@ param( [string]$BuildDir = "build-win", [string]$Config = "Debug", [string]$Endpoint = "\\.\\pipe\\PerfectHashServer-Smoke", - [int]$TimeoutSeconds = 10 + [int]$TimeoutSeconds = 10, + [bool]$WaitForServer = $true, + [int]$ConnectTimeoutMs = 10000 ) Set-StrictMode -Version Latest @@ -47,6 +49,12 @@ $tableArg = "--TableCreate=$createCommand" $serverArgs = @("--Endpoint=$Endpoint") $clientArgs = @("--Endpoint=$Endpoint", "--Shutdown") +if ($WaitForServer) { + $clientArgs += "--WaitForServer" + if ($ConnectTimeoutMs -gt 0) { + $clientArgs += ("--ConnectTimeout={0}" -f $ConnectTimeoutMs) + } +} $server = Start-Process -FilePath $serverExe ` -ArgumentList $serverArgs ` @@ -55,9 +63,16 @@ $server = Start-Process -FilePath $serverExe ` Start-Sleep -Milliseconds 500 +$clientCreateArgs = @("--Endpoint=$Endpoint", "`"$tableArg`"") +if ($WaitForServer) { + $clientCreateArgs += "--WaitForServer" + if ($ConnectTimeoutMs -gt 0) { + $clientCreateArgs += ("--ConnectTimeout={0}" -f $ConnectTimeoutMs) + } +} + $clientCreate = Start-Process -FilePath $clientExe ` - -ArgumentList @("--Endpoint=$Endpoint", - "`"$tableArg`"") ` + -ArgumentList $clientCreateArgs ` -PassThru ` -NoNewWindow ` -Wait diff --git a/scripts/stress-sys32-iocp.ps1 b/scripts/stress-sys32-iocp.ps1 index f6ab2089..f1fa87b1 100644 --- a/scripts/stress-sys32-iocp.ps1 +++ b/scripts/stress-sys32-iocp.ps1 @@ -9,7 +9,9 @@ param( [string]$MaskFunction = "And", [int]$MaximumConcurrency = 0, [string[]]$ExtraArgs = @(), - [int]$TimeoutSeconds = 300 + [int]$TimeoutSeconds = 300, + [bool]$WaitForServer = $true, + [int]$ConnectTimeoutMs = 10000 ) Set-StrictMode -Version Latest @@ -57,11 +59,17 @@ $server = Start-Process -FilePath $serverExe ` -PassThru ` -NoNewWindow -Start-Sleep -Milliseconds 500 +$clientArgs = @("--Endpoint=$Endpoint") +if ($WaitForServer) { + $clientArgs += "--WaitForServer" + if ($ConnectTimeoutMs -gt 0) { + $clientArgs += ("--ConnectTimeout={0}" -f $ConnectTimeoutMs) + } +} +$clientCreateArgs = @($clientArgs + "`"$bulkArg`"") $clientCreate = Start-Process -FilePath $clientExe ` - -ArgumentList @("--Endpoint=$Endpoint", - "`"$bulkArg`"") ` + -ArgumentList $clientCreateArgs ` -PassThru ` -NoNewWindow ` -Wait @@ -71,9 +79,9 @@ if ($clientCreate.ExitCode -ne 0 -and $clientCreate.ExitCode -ne $bulkSuccess) { throw ("Client bulk-create exited with code {0}" -f $clientCreate.ExitCode) } +$clientShutdownArgs = @($clientArgs + "--Shutdown") $clientShutdown = Start-Process -FilePath $clientExe ` - -ArgumentList @("--Endpoint=$Endpoint", - "--Shutdown") ` + -ArgumentList $clientShutdownArgs ` -PassThru ` -NoNewWindow ` -Wait diff --git a/src/PerfectHash/Chm01Async.c b/src/PerfectHash/Chm01Async.c index 9e6d67fd..9be308b8 100644 --- a/src/PerfectHash/Chm01Async.c +++ b/src/PerfectHash/Chm01Async.c @@ -27,6 +27,7 @@ Module Name: #define CHM01_ASYNC_JOB_FLAG_CLOSE_DELETES_FILES 0x00000001 #define CHM01_ASYNC_JOB_FLAG_CLOSE_SUBMITTED 0x00000002 #define CHM01_ASYNC_JOB_FLAG_SAVE_SUBMITTED 0x00000008 +#define CHM01_ASYNC_IOCP_PUMP_TIMEOUT_MS 100 typedef struct _CHM01_ASYNC_GRAPH_WORK { PERFECT_HASH_ASYNC_WORK Work; @@ -55,6 +56,99 @@ Chm01AsyncShouldSkipContextFileWork( return IsValidContextFileId((CONTEXT_FILE_ID)FileId); } +static +VOID +Chm01AsyncLogJobState( + _In_ PCHM01_ASYNC_JOB Job, + _In_ ULONG Stage + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[512]; + LONG Outstanding = 0; + LONG ActiveGraphs = 0; + ULONG State = 0; + ULONG Attempt = 0; + ULONG Flags = 0; + HRESULT Result = S_OK; + ULONG NameChars = 0; + PCWSTR NameBuffer = L""; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + PPERFECT_HASH_FILE File; + PPERFECT_HASH_PATH Path; + + if (GetEnvironmentVariableW(L"PH_LOG_CHM01_ASYNC_JOB", NULL, 0) == 0) { + return; + } + + if (!ARGUMENT_PRESENT(Job)) { + return; + } + + Outstanding = Job->Async.Outstanding; + ActiveGraphs = Job->ActiveGraphs; + State = (ULONG)Job->State; + Attempt = Job->Attempt; + Flags = Job->Flags; + Result = Job->LastResult; + + Table = Job->Table; + Keys = Table ? Table->Keys : NULL; + File = Keys ? Keys->File : NULL; + Path = File ? GetActivePath(File) : NULL; + + if (Path && Path->FileName.Buffer) { + NameBuffer = Path->FileName.Buffer; + NameChars = Path->FileName.Length / sizeof(WCHAR); + } + + LogHandle = CreateFileW(L"PerfectHashChm01AsyncJob.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s( + Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu State=%lu Outstanding=%ld ActiveGraphs=%ld Attempt=%lu " + "Flags=0x%08lX Result=0x%08lX Name=%.*S\r\n", + Stage, + State, + Outstanding, + ActiveGraphs, + Attempt, + Flags, + Result, + (int)NameChars, + NameBuffer + ); + + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Job); + UNREFERENCED_PARAMETER(Stage); +#endif +} + static HRESULT Chm01AsyncGraphStep( @@ -428,11 +522,11 @@ Chm01AsyncInitializeJob( // Initialize event arrays. // - Job->Events[0] = Context->SucceededEvent; - Job->Events[1] = Context->CompletedEvent; - Job->Events[2] = Context->ShutdownEvent; - Job->Events[3] = Context->FailedEvent; - Job->Events[4] = Context->LowMemoryEvent; + Job->Events[Chm01AsyncJobEventSucceeded] = Context->SucceededEvent; + Job->Events[Chm01AsyncJobEventCompleted] = Context->CompletedEvent; + Job->Events[Chm01AsyncJobEventShutdown] = Context->ShutdownEvent; + Job->Events[Chm01AsyncJobEventFailed] = Context->FailedEvent; + Job->Events[Chm01AsyncJobEventLowMemory] = Context->LowMemoryEvent; { PHANDLE SaveEvent = Job->SaveEvents; @@ -1415,9 +1509,18 @@ Chm01AsyncComplete( Job = CONTAINING_RECORD(Work, CHM01_ASYNC_JOB, Work); Job->LastResult = Result; + Chm01AsyncLogJobState(Job, 1); + if (Job->CompletionEvent) { SetEvent(Job->CompletionEvent); } + + if (Job->Async.IoCompletionPort) { + PostQueuedCompletionStatus(Job->Async.IoCompletionPort, + 0, + 0, + NULL); + } } _Use_decl_annotations_ @@ -1532,8 +1635,11 @@ Chm01AsyncPumpIoCompletionPort( { BOOL Success; DWORD NumberOfBytes; + DWORD LastError; ULONG_PTR CompletionKey; LPOVERLAPPED Overlapped; + BOOLEAN LoggedCompletionSignaled = FALSE; + BOOLEAN LoggedOutstandingZero = FALSE; HANDLE IoCompletionPort; PPERFECT_HASH_IOCP_WORK WorkItem; const ULONG AllowedFlags = ( @@ -1552,13 +1658,27 @@ Chm01AsyncPumpIoCompletionPort( if (Job->Async.Outstanding == 0) { break; } + if (!LoggedCompletionSignaled) { + Chm01AsyncLogJobState(Job, 2); + LoggedCompletionSignaled = TRUE; + } + } else if (Job->Async.Outstanding == 0 && !LoggedOutstandingZero) { + Chm01AsyncLogJobState(Job, 3); + LoggedOutstandingZero = TRUE; } Success = GetQueuedCompletionStatus(IoCompletionPort, &NumberOfBytes, &CompletionKey, &Overlapped, - INFINITE); + CHM01_ASYNC_IOCP_PUMP_TIMEOUT_MS); + + if (!Success && !Overlapped) { + LastError = GetLastError(); + if (LastError == WAIT_TIMEOUT) { + continue; + } + } if (CompletionKey == PERFECT_HASH_IOCP_SHUTDOWN_KEY) { PostQueuedCompletionStatus(IoCompletionPort, diff --git a/src/PerfectHash/Chm01Async.h b/src/PerfectHash/Chm01Async.h index a7855b27..5d8e2e8f 100644 --- a/src/PerfectHash/Chm01Async.h +++ b/src/PerfectHash/Chm01Async.h @@ -28,6 +28,17 @@ typedef enum _CHM01_ASYNC_STATE { Chm01AsyncStateError } CHM01_ASYNC_STATE; +typedef enum _CHM01_ASYNC_JOB_EVENT_ID { + Chm01AsyncJobEventSucceeded = 0, + Chm01AsyncJobEventCompleted, + Chm01AsyncJobEventShutdown, + Chm01AsyncJobEventFailed, + Chm01AsyncJobEventLowMemory, + Chm01AsyncJobEventInvalid +} CHM01_ASYNC_JOB_EVENT_ID; + +#define NUMBER_OF_CHM01_ASYNC_JOB_EVENTS (Chm01AsyncJobEventInvalid) + typedef struct _CHM01_ASYNC_GRAPH_WORK CHM01_ASYNC_GRAPH_WORK; typedef CHM01_ASYNC_GRAPH_WORK *PCHM01_ASYNC_GRAPH_WORK; @@ -52,7 +63,7 @@ typedef struct _CHM01_ASYNC_JOB { GRAPH_INFO PrevGraphInfo; GRAPH_INFO_ON_DISK GraphInfoOnDisk; PTABLE_INFO_ON_DISK TableInfoOnDisk; - HANDLE Events[5]; + HANDLE Events[NUMBER_OF_CHM01_ASYNC_JOB_EVENTS]; HANDLE SaveEvents[NUMBER_OF_SAVE_FILE_EVENTS]; HANDLE PrepareEvents[NUMBER_OF_PREPARE_FILE_EVENTS]; PCHM01_ASYNC_GRAPH_WORK *GraphWorkItems; diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c index 66434c5a..57a84aae 100644 --- a/src/PerfectHash/PerfectHashServer.c +++ b/src/PerfectHash/PerfectHashServer.c @@ -184,6 +184,12 @@ PerfectHashServerPrepareErrorPayload( _In_ HRESULT Error ); +static +HRESULT +PerfectHashServerPreparePingPayload( + _In_ PPERFECT_HASH_SERVER_PIPE Pipe + ); + static HRESULT PerfectHashServerPrepareBulkCreateDirectoryPayload( @@ -223,6 +229,15 @@ PerfectHashServerLogBulkCreateException( _In_ struct _EXCEPTION_POINTERS *ExceptionPointers ); +static +VOID +PerfectHashServerLogBulkCreateCounts( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_opt_ PPERFECT_HASH_SERVER_BULK_REQUEST Request, + _In_ LONG Outstanding + ); + static LONG PerfectHashServerBulkCreateExceptionFilter( @@ -1315,6 +1330,58 @@ PerfectHashServerPrepareErrorPayload( return S_OK; } +static +HRESULT +PerfectHashServerPreparePingPayload( + PPERFECT_HASH_SERVER_PIPE Pipe + ) +{ + PALLOCATOR Allocator = NULL; + PRTL Rtl = NULL; + WCHAR Pong[] = L"PONG"; + ULONG PayloadLength; + + if (!ARGUMENT_PRESENT(Pipe)) { + return E_POINTER; + } + + Allocator = Pipe->Server ? Pipe->Server->Allocator : NULL; + Rtl = Pipe->Server ? Pipe->Server->Rtl : NULL; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PayloadLength = (ULONG)sizeof(Pong); + if (PayloadLength > PERFECT_HASH_SERVER_MAX_MESSAGE_SIZE) { + return E_INVALIDARG; + } + + if (!Pipe->PayloadBuffer || Pipe->PayloadBufferSize < PayloadLength) { + if (Pipe->PayloadBuffer) { + Allocator->Vtbl->FreePointer(Allocator, + &Pipe->PayloadBuffer); + } + + Pipe->PayloadBuffer = Allocator->Vtbl->Calloc(Allocator, + 1, + PayloadLength); + if (!Pipe->PayloadBuffer) { + return E_OUTOFMEMORY; + } + + Pipe->PayloadBufferSize = PayloadLength; + } else { + ZeroMemory(Pipe->PayloadBuffer, PayloadLength); + } + + CopyMemory(Pipe->PayloadBuffer, Pong, PayloadLength); + + Pipe->ResponseHeader.PayloadLength = PayloadLength; + Pipe->ResponseHeader.Flags |= PERFECT_HASH_SERVER_RESPONSE_FLAG_PONG; + + return S_OK; +} + static HRESULT PerfectHashServerPrepareBulkCreateDirectoryPayload( @@ -1393,6 +1460,7 @@ PerfectHashServerCompleteBulkRequest( HRESULT FirstFailure; PPERFECT_HASH_SERVER_BULK_RESULT BulkResult; PALLOCATOR Allocator; + LONG Outstanding; if (!ARGUMENT_PRESENT(Request)) { return; @@ -1402,6 +1470,9 @@ PerfectHashServerCompleteBulkRequest( return; } + Outstanding = Request->OutstandingWorkItems; + PerfectHashServerLogBulkCreateCounts(3, NULL, Request, Outstanding); + Failed = (ULONG)Request->FailedWorkItems; Total = Request->TotalWorkItems; Succeeded = (Total >= Failed) ? (Total - Failed) : 0; @@ -1572,6 +1643,90 @@ PerfectHashServerLogBulkCreateException( #endif } +static +VOID +PerfectHashServerLogBulkCreateCounts( + _In_ ULONG Stage, + _In_opt_ PPERFECT_HASH_SERVER_BULK_WORK_ITEM WorkItem, + _In_opt_ PPERFECT_HASH_SERVER_BULK_REQUEST Request, + _In_ LONG Outstanding + ) +{ +#ifdef PH_WINDOWS + int Count; + DWORD BytesWritten; + HANDLE LogHandle; + CHAR Buffer[512]; + ULONG PathChars = 0; + LONG DispatchComplete = 0; + LONG Failed = 0; + LONG PendingNodes = 0; + ULONG Total = 0; + ULONG NodeIndex = 0; + + if (GetEnvironmentVariableW(L"PH_LOG_BULK_CREATE_COUNTS", NULL, 0) == 0) { + return; + } + + if (Request) { + DispatchComplete = Request->DispatchComplete; + Failed = Request->FailedWorkItems; + PendingNodes = Request->PendingNodes; + Total = Request->TotalWorkItems; + } + + if (WorkItem) { + NodeIndex = WorkItem->NodeIndex; + if (WorkItem->KeysPath.Buffer) { + PathChars = WorkItem->KeysPath.Length / sizeof(WCHAR); + } + } + + LogHandle = CreateFileW(L"PerfectHashServerBulkCreateCounts.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (!IsValidHandle(LogHandle)) { + return; + } + + Count = _snprintf_s( + Buffer, + sizeof(Buffer), + _TRUNCATE, + "Stage=%lu Outstanding=%ld Dispatch=%ld Total=%lu Failed=%ld " + "PendingNodes=%ld Node=%lu Path=%.*S\r\n", + Stage, + Outstanding, + DispatchComplete, + Total, + Failed, + PendingNodes, + NodeIndex, + (int)PathChars, + WorkItem ? WorkItem->KeysPath.Buffer : L"" + ); + + if (Count > 0) { + WriteFile(LogHandle, + Buffer, + (DWORD)strlen(Buffer), + &BytesWritten, + NULL); + } + + CloseHandle(LogHandle); +#else + UNREFERENCED_PARAMETER(Stage); + UNREFERENCED_PARAMETER(WorkItem); + UNREFERENCED_PARAMETER(Request); + UNREFERENCED_PARAMETER(Outstanding); +#endif +} + static VOID PerfectHashServerLogBulkCreateFailure( @@ -1661,6 +1816,7 @@ PerfectHashServerBulkCreateWorkItemCallback( { HRESULT Result; HRESULT Failure; + LONG Outstanding; LONG NodeOutstanding; PALLOCATOR Allocator = NULL; PPERFECT_HASH_CONTEXT Context = NULL; @@ -1865,8 +2021,10 @@ PerfectHashServerBulkCreateWorkItemCallback( } } - if (InterlockedDecrement(&Request->OutstandingWorkItems) == 0 && - Request->DispatchComplete != 0) { + Outstanding = InterlockedDecrement(&Request->OutstandingWorkItems); + PerfectHashServerLogBulkCreateCounts(1, WorkItem, Request, Outstanding); + + if (Outstanding == 0 && Request->DispatchComplete != 0) { PerfectHashServerCompleteBulkRequest(Request); Request = NULL; } @@ -1892,6 +2050,7 @@ PerfectHashServerEnqueueBulkRequest( ULONG BytesToAllocate; ULONG WildcardBytes; ULONG KeysPathBytes; + LONG Outstanding; PALLOCATOR Allocator = NULL; PRTL Rtl = NULL; PWSTR WildcardBuffer = NULL; @@ -2153,7 +2312,9 @@ PerfectHashServerEnqueueBulkRequest( if (DispatchInitialized) { Request->DispatchComplete = 1; - if (InterlockedDecrement(&Request->OutstandingWorkItems) == 0) { + Outstanding = InterlockedDecrement(&Request->OutstandingWorkItems); + PerfectHashServerLogBulkCreateCounts(2, NULL, Request, Outstanding); + if (Outstanding == 0) { PerfectHashServerCompleteBulkRequest(Request); } } @@ -2796,6 +2957,17 @@ PerfectHashServerDispatchRequest( Result = E_INVALIDARG; break; + case PerfectHashPingServerRequestType: + if (Pipe->PayloadLength != 0) { + Result = E_INVALIDARG; + break; + } + if (Pipe->Server && Pipe->Server->StartedEvent) { + WaitForSingleObject(Pipe->Server->StartedEvent, INFINITE); + } + Result = PerfectHashServerPreparePingPayload(Pipe); + break; + case PerfectHashShutdownServerRequestType: Result = S_OK; Pipe->ShutdownAfterSend = TRUE; diff --git a/src/PerfectHash/_dtoa.c b/src/PerfectHash/_dtoa.c index 74937ac3..18e84593 100644 --- a/src/PerfectHash/_dtoa.c +++ b/src/PerfectHash/_dtoa.c @@ -130,6 +130,36 @@ PALLOCATOR _dtoa_Allocator; +static +PALLOCATOR +PhGetDtoaAllocator( + VOID + ) +{ + PALLOCATOR Allocator; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + + Allocator = _dtoa_Allocator; + if (Allocator) { + return Allocator; + } + + TlsContext = PerfectHashTlsGetContext(); + if (!TlsContext) { + return NULL; + } + + if (TlsContext->Allocator) { + return TlsContext->Allocator; + } + + if (TlsContext->Context) { + return TlsContext->Context->Allocator; + } + + return NULL; +} + _Must_inspect_result_ _Ret_maybenull_ _Success_(return != 0) @@ -139,11 +169,14 @@ PhMalloc( _In_ SIZE_T Size ) { - return _dtoa_Allocator->Vtbl->Calloc( - _dtoa_Allocator, - 1, - Size - ); + PALLOCATOR Allocator; + + Allocator = PhGetDtoaAllocator(); + if (!Allocator) { + return NULL; + } + + return Allocator->Vtbl->Calloc(Allocator, 1, Size); } VOID @@ -151,7 +184,14 @@ PhFree( _Frees_ptr_opt_ PVOID Address ) { - _dtoa_Allocator->Vtbl->Free(_dtoa_Allocator, Address); + PALLOCATOR Allocator; + + Allocator = PhGetDtoaAllocator(); + if (!Allocator) { + return; + } + + Allocator->Vtbl->Free(Allocator, Address); } //#include "float.h" diff --git a/src/PerfectHashClientExe/PerfectHashClientExe.c b/src/PerfectHashClientExe/PerfectHashClientExe.c index 5d87d497..2e456387 100644 --- a/src/PerfectHashClientExe/PerfectHashClientExe.c +++ b/src/PerfectHashClientExe/PerfectHashClientExe.c @@ -17,14 +17,26 @@ Module Name: #include "stdafx.h" #include +typedef union _PERFECT_HASH_CLIENT_CLI_FLAGS { + struct { + ULONG EndpointPresent:1; + ULONG CommandLinePresent:1; + ULONG RequestTypePresent:1; + ULONG WaitForServer:1; + ULONG ConnectTimeoutPresent:1; + ULONG Unused:27; + }; + ULONG AsULong; +} PERFECT_HASH_CLIENT_CLI_FLAGS; +typedef PERFECT_HASH_CLIENT_CLI_FLAGS *PPERFECT_HASH_CLIENT_CLI_FLAGS; + typedef struct _PERFECT_HASH_CLIENT_CLI_OPTIONS { UNICODE_STRING Endpoint; UNICODE_STRING CommandLine; PERFECT_HASH_SERVER_REQUEST_TYPE RequestType; - BOOLEAN EndpointPresent; - BOOLEAN CommandLinePresent; - BOOLEAN RequestTypePresent; - UCHAR Padding1[9]; + ULONG ConnectTimeoutInMilliseconds; + PERFECT_HASH_CLIENT_CLI_FLAGS Flags; + ULONG Padding1; } PERFECT_HASH_CLIENT_CLI_OPTIONS; typedef PERFECT_HASH_CLIENT_CLI_OPTIONS *PPERFECT_HASH_CLIENT_CLI_OPTIONS; @@ -36,10 +48,13 @@ PrintUsage( { wprintf(L"Usage: PerfectHashClient [--Endpoint=] " L"[--Shutdown|--TableCreate=|--BulkCreate=|" - L"--BulkCreateDirectory=]\n"); + L"--BulkCreateDirectory=|--Ping]\n"); wprintf(L" --TableCreate= PerfectHashCreate-style arguments\n"); wprintf(L" --BulkCreate= PerfectHashBulkCreate-style arguments\n"); wprintf(L" --BulkCreateDirectory= BulkCreate args with single token\n"); + wprintf(L" --Ping Wait for server readiness\n"); + wprintf(L" --WaitForServer Wait for server to appear\n"); + wprintf(L" --ConnectTimeout= Cap wait time for --WaitForServer\n"); } static @@ -55,13 +70,12 @@ ParseClientArgs( Options->Endpoint.Buffer = NULL; Options->Endpoint.Length = 0; Options->Endpoint.MaximumLength = 0; - Options->EndpointPresent = FALSE; + Options->Flags.AsULong = 0; Options->CommandLine.Buffer = NULL; Options->CommandLine.Length = 0; Options->CommandLine.MaximumLength = 0; - Options->CommandLinePresent = FALSE; Options->RequestType = PerfectHashNullServerRequestType; - Options->RequestTypePresent = FALSE; + Options->ConnectTimeoutInMilliseconds = 0; for (Index = 1; Index < NumberOfArguments; Index++) { PCWSTR Arg = ArgvW[Index]; @@ -92,16 +106,49 @@ ParseClientArgs( Options->Endpoint.Buffer = (PWSTR)Value; Options->Endpoint.Length = (USHORT)Length; Options->Endpoint.MaximumLength = (USHORT)Length + sizeof(WCHAR); - Options->EndpointPresent = TRUE; + Options->Flags.EndpointPresent = TRUE; continue; } if (_wcsicmp(Arg, L"Shutdown") == 0) { - if (Options->RequestTypePresent) { + if (Options->Flags.RequestTypePresent) { return PH_E_INVALID_COMMANDLINE_ARG; } Options->RequestType = PerfectHashShutdownServerRequestType; - Options->RequestTypePresent = TRUE; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"Ping") == 0) { + if (Options->Flags.RequestTypePresent) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + Options->RequestType = PerfectHashPingServerRequestType; + Options->Flags.RequestTypePresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"WaitForServer") == 0) { + Options->Flags.WaitForServer = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"ConnectTimeout=", 15) == 0) { + PCWSTR Value; + ULONG Timeout; + + Value = Arg + 15; + if (!Value || *Value == L'\0') { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Timeout = wcstoul(Value, NULL, 10); + if (Timeout == 0) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + Options->ConnectTimeoutInMilliseconds = Timeout; + Options->Flags.ConnectTimeoutPresent = TRUE; continue; } @@ -109,7 +156,7 @@ ParseClientArgs( PCWSTR Value; ULONG Length; - if (Options->RequestTypePresent) { + if (Options->Flags.RequestTypePresent) { return PH_E_INVALID_COMMANDLINE_ARG; } @@ -120,9 +167,9 @@ ParseClientArgs( Options->CommandLine.Length = (USHORT)Length; Options->CommandLine.MaximumLength = (USHORT)Length + sizeof(WCHAR); - Options->CommandLinePresent = TRUE; + Options->Flags.CommandLinePresent = TRUE; Options->RequestType = PerfectHashTableCreateServerRequestType; - Options->RequestTypePresent = TRUE; + Options->Flags.RequestTypePresent = TRUE; continue; } @@ -130,7 +177,7 @@ ParseClientArgs( PCWSTR Value; ULONG Length; - if (Options->RequestTypePresent) { + if (Options->Flags.RequestTypePresent) { return PH_E_INVALID_COMMANDLINE_ARG; } @@ -141,9 +188,9 @@ ParseClientArgs( Options->CommandLine.Length = (USHORT)Length; Options->CommandLine.MaximumLength = (USHORT)Length + sizeof(WCHAR); - Options->CommandLinePresent = TRUE; + Options->Flags.CommandLinePresent = TRUE; Options->RequestType = PerfectHashBulkCreateServerRequestType; - Options->RequestTypePresent = TRUE; + Options->Flags.RequestTypePresent = TRUE; continue; } @@ -151,7 +198,7 @@ ParseClientArgs( PCWSTR Value; ULONG Length; - if (Options->RequestTypePresent) { + if (Options->Flags.RequestTypePresent) { return PH_E_INVALID_COMMANDLINE_ARG; } @@ -162,26 +209,111 @@ ParseClientArgs( Options->CommandLine.Length = (USHORT)Length; Options->CommandLine.MaximumLength = (USHORT)Length + sizeof(WCHAR); - Options->CommandLinePresent = TRUE; + Options->Flags.CommandLinePresent = TRUE; Options->RequestType = PerfectHashBulkCreateDirectoryServerRequestType; - Options->RequestTypePresent = TRUE; + Options->Flags.RequestTypePresent = TRUE; continue; } return PH_E_INVALID_COMMANDLINE_ARG; } - if (Options->RequestTypePresent) { - if (Options->RequestType == PerfectHashShutdownServerRequestType) { - if (Options->CommandLinePresent) { + if (Options->Flags.RequestTypePresent) { + if (Options->RequestType == PerfectHashShutdownServerRequestType || + Options->RequestType == PerfectHashPingServerRequestType) { + if (Options->Flags.CommandLinePresent) { return PH_E_INVALID_COMMANDLINE_ARG; } - } else if (!Options->CommandLinePresent) { + } else if (!Options->Flags.CommandLinePresent) { return PH_E_INVALID_COMMANDLINE_ARG; } } + if (Options->Flags.ConnectTimeoutPresent && !Options->Flags.WaitForServer) { + return PH_E_INVALID_COMMANDLINE_ARG; + } + + return S_OK; +} + +static +HRESULT +PerfectHashClientConnectWithWait( + _In_ PPERFECT_HASH_CLIENT Client, + _In_opt_ PCUNICODE_STRING Endpoint, + _In_ BOOLEAN WaitForServer, + _In_ BOOLEAN TimeoutPresent, + _In_ ULONG TimeoutInMilliseconds + ) +{ + HRESULT Result; + ULONGLONG StartTicks; + + if (!WaitForServer) { + return Client->Vtbl->Connect(Client, Endpoint); + } + + StartTicks = GetTickCount64(); + + for (;;) { + Result = Client->Vtbl->Connect(Client, Endpoint); + if (SUCCEEDED(Result)) { + return S_OK; + } + + if (Result != HRESULT_FROM_WIN32(ERROR_PIPE_BUSY) && + Result != HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) { + return Result; + } + + if (TimeoutPresent) { + ULONGLONG Elapsed = GetTickCount64() - StartTicks; + if (Elapsed >= TimeoutInMilliseconds) { + return HRESULT_FROM_WIN32(WAIT_TIMEOUT); + } + } + + Sleep(50); + } +} + +static +HRESULT +PerfectHashClientPing( + _In_ PPERFECT_HASH_CLIENT Client + ) +{ + HRESULT Result; + ULONG ResponseFlags = 0; + UNICODE_STRING ResponsePayload; + PERFECT_HASH_SERVER_REQUEST Request = { 0 }; + + Request.SizeOfStruct = sizeof(Request); + Request.RequestType = PerfectHashPingServerRequestType; + Request.RequestId = 1; + + Result = Client->Vtbl->SubmitRequest(Client, &Request); + if (FAILED(Result)) { + return Result; + } + + Result = Client->Vtbl->GetLastResponse(Client, + &ResponsePayload, + &ResponseFlags); + if (FAILED(Result)) { + return Result; + } + + if (!(ResponseFlags & PERFECT_HASH_SERVER_RESPONSE_FLAG_PONG)) { + return E_UNEXPECTED; + } + + if (!ResponsePayload.Buffer || + _wcsicmp(ResponsePayload.Buffer, L"PONG") != 0) { + return E_UNEXPECTED; + } + return S_OK; } @@ -350,19 +482,47 @@ mainCRTStartup( goto Error; } - Result = Client->Vtbl->Connect(Client, - Options.EndpointPresent ? - &Options.Endpoint : - NULL); + Result = PerfectHashClientConnectWithWait( + Client, + Options.Flags.EndpointPresent ? &Options.Endpoint : NULL, + (BOOLEAN)!!Options.Flags.WaitForServer, + (BOOLEAN)!!Options.Flags.ConnectTimeoutPresent, + Options.ConnectTimeoutInMilliseconds + ); if (FAILED(Result)) { goto Error; } - if (Options.RequestTypePresent) { + if (Options.Flags.WaitForServer || + Options.RequestType == PerfectHashPingServerRequestType) { + Result = PerfectHashClientPing(Client); + if (FAILED(Result)) { + goto Error; + } + + if (Options.RequestType == PerfectHashPingServerRequestType || + !Options.Flags.RequestTypePresent) { + goto End; + } + + Client->Vtbl->Disconnect(Client); + Result = PerfectHashClientConnectWithWait( + Client, + Options.Flags.EndpointPresent ? &Options.Endpoint : NULL, + (BOOLEAN)!!Options.Flags.WaitForServer, + (BOOLEAN)!!Options.Flags.ConnectTimeoutPresent, + Options.ConnectTimeoutInMilliseconds + ); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.Flags.RequestTypePresent) { Request.SizeOfStruct = sizeof(Request); Request.RequestType = Options.RequestType; Request.RequestId = 1; - if (Options.CommandLinePresent) { + if (Options.Flags.CommandLinePresent) { Request.CommandLine = Options.CommandLine; } From 7156a1957809d147b8a4bbaaedc819665b0b7532 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Thu, 22 Jan 2026 18:53:47 -0800 Subject: [PATCH 4/9] Iocp updates. --- IOCP-LOGS.md | 24 +++ IOCP-TOOD.md | 23 +- USAGE.txt | 19 +- include/PerfectHash.h | 83 ++++++++ scripts/iocp-smoke.ps1 | 21 +- scripts/stress-sys32-iocp.ps1 | 25 ++- src/PerfectHash/Chm01.c | 4 +- src/PerfectHash/Chm01Async.c | 199 +++++++++++++++--- src/PerfectHash/Chm01Async.h | 6 + src/PerfectHash/ExtractArg.c | 8 + src/PerfectHash/PerfectHashConstants.c | 12 +- src/PerfectHash/PerfectHashContext.c | 3 + src/PerfectHash/PerfectHashContext.h | 4 + .../PerfectHashContextBulkCreate.c | 21 ++ src/PerfectHash/PerfectHashContextIocp.c | 147 +++++++++---- src/PerfectHash/PerfectHashContextIocp.h | 16 +- src/PerfectHash/PerfectHashContextIocpArgs.c | 66 ++++++ .../PerfectHashContextTableCreate.c | 21 ++ src/PerfectHash/PerfectHashFile.c | 14 +- src/PerfectHash/PerfectHashServer.c | 164 ++++++++++++++- src/PerfectHash/PerfectHashServer.h | 20 +- src/PerfectHash/PerfectHashTableCreate.c | 48 +++++ .../PerfectHashServerExe.c | 114 ++++++++-- 23 files changed, 953 insertions(+), 109 deletions(-) diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index fea6a47e..22922680 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -106,3 +106,27 @@ - Identified `Chm01AsyncWaitJob()` pump loop could block in `GetQueuedCompletionStatus()` after job completion, leaving bulk work items stuck mid-callback. - Added a dummy IOCP wake post in `Chm01AsyncComplete()` and switched the pump loop to a 100ms timeout to re-check job completion state. - Re-ran IOCP stress on `perfecthash-keys\\hard` with `Mulshrolate4RX`/concurrency 32; run completed successfully and server shutdown cleanly. +- Clean Release rebuild after build dir cleanup and ran IOCP sys32 stress (`Mulshrolate4RX`, concurrency 32); run completed in ~197s but emitted repeated `PerfectHashKeysLoadTableSize` invalid EOF messages (0xe0040264). +- Identified 31 sys32 keys files with zero-length `Chm01_Mulshrolate4RX_And.TableSize` streams causing invalid EOF failures. +- Reproduced the invalid EOF with a single file (`aadauthhelper-189.keys`) via `PerfectHashCreate.exe`. +- Fixed `PerfectHashFileCreate()` to extend existing zero-length files when `NoTruncate` is set; single-file create now succeeds. +- Added `ADS.md` with notes on table-size vs table-info ADS usage and risks of removal. +- Created `..\\perfecthash-keys\\win32.temp50` with 50 random sys32 keys and ran IOCP bulk create (Release, `Mulshrolate4RX`, concurrency 32); run completed with 50 output dirs. +- Attempted full sys32 IOCP run (Release, `Mulshrolate4RX`, concurrency 32) to `build-win\\iocp-stress-sys32-release`; run emitted `SetEndOfFile` failures with `ERROR_DISK_FULL` and was halted; output dir grew to ~193 GB with ~13,637 directories before shutdown. +- Added IOCP server verbosity/no-file-IO controls (`--Verbose`, `--NoFileIo`) and vtbl setters/getters; server now defaults to silent output and only prints console legend when console work is enabled. +- Updated IOCP scripts (`scripts/iocp-smoke.ps1`, `scripts/stress-sys32-iocp.ps1`) to pass the new server flags (NoFileIo default true, optional verbose). +- Ran IOCP sys32 `--NoFileIo` stress (Release, `Mulshrolate4RX`, concurrency 32) via script; client/server did not complete within tool timeout, shutdown timed out, and both processes required termination (follow-up needed). +- Ran IOCP bulk create against a 200-file subset of `sys32` (`G:\\Scratch\\sys32-200`, Release, `Mulshrolate4RX`, concurrency 4, `--NoFileIo`); completed successfully within the 30s timeout. +- Captured server thread samples during the 200-file run (Release, concurrency 4). `PerfectHashServer.exe` showed 68 threads total; initial samples had 31-36 running, then all waiting. Wait reasons were dominated by `EventPairLow` (IOCP), with a few `Unknown`/`UserRequest`. +- Re-ran the 200-file sys32 subset with server `--MaxConcurrency=4` (Release, `Mulshrolate4RX`, `--NoFileIo`). Thread sampling showed 8 total threads (4 running initially, then all waiting), with wait reasons dominated by `EventPairLow`; client/server exited cleanly. +- Added separate IOCP concurrency and worker thread controls: `--IocpConcurrency` (`--MaxConcurrency` alias) and `--MaxThreads` (default `IocpConcurrency * 2` when set). IOCP ports now use per-node concurrency, worker threads default per node; scripts accept `-IocpConcurrency`/`-MaxThreads`. +- 200-file sys32 subset run with `--IocpConcurrency=4` (Release, `Mulshrolate4RX`, `--NoFileIo`) showed 12 total server threads (6 running initially), wait reasons dominated by `EventPairLow`; client/server exited cleanly. +- 1000-file random sys32 subset (`G:\\Scratch\\sys32-1000`, Release, `Mulshrolate4RX`, `--NoFileIo`, per-file concurrency 32, server `--IocpConcurrency=32 --MaxThreads=64`) did not complete within 900s; client/server remained running and required termination. +- OG `PerfectHashBulkCreate.exe` run on the same 1000-file subset (Release, `Mulshrolate4RX`, `--NoFileIo`, concurrency 32, `--Silent`) completed in ~1.95s with exit code 0. +- Added per-file concurrency ramp controls (`InitialPerFileConcurrency`, `MaxPerFileConcurrency`, `IncreaseConcurrencyAfterMilliseconds`) with defaults (1, max, 500ms). +- Wired new table-create params into arg parsing and validation, IOCP arg helpers, and `PERFECT_HASH_CONTEXT`. +- Updated `Chm01Async` to dispatch the initial per-file graph work count and enqueue additional graph work after the configured delay. +- Documented new parameters in `USAGE.txt`. +- IOCP sys32-200 subset run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms) completed successfully. +- IOCP `hard` run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms) crashed with `0xC0000005` (server exit code -1073741819); no minidump found. +- Re-ran `hard` under `cdb` attach (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms); client returned `0x2004000F`, shutdown succeeded, and no AV was captured (`build-win\\logs\\cdb-hard-attach-release.log`). diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md index dd90cc36..64ae76e4 100644 --- a/IOCP-TOOD.md +++ b/IOCP-TOOD.md @@ -1,24 +1,21 @@ # IOCP TODO -- Validate bulk-create directory request end-to-end (sys32 stress) and tune per-file concurrency defaults. -- Run IOCP sys32 stress pass (Release, max concurrency) after clean rebuild and crash fixes. -- Decide how to guard against stale PCH/obj after `PERFECT_HASH_CONTEXT` layout changes (force clean or touch header) and recheck Release bulk-create. -- Track down `FlushConsoleInputBuffer` failures during bulk create (seen on `main` and iocp-dev). +- Get a sys32 `--NoFileIo` run to complete (Release, max concurrency); if it hangs, capture bulk-count logs and identify the stuck state. +- Diagnose 1000-file sys32 subset hang with `--IocpConcurrency=32 --MaxThreads=64` (Release, `--NoFileIo`); identify which work items are stuck and why. +- Re-run sys32 with file I/O to `G:\\Scratch` once `--NoFileIo` completes; record timing and correctness. +- Decide how to guard against stale PCH/obj after `PERFECT_HASH_CONTEXT` layout changes (force clean or touch header). +- Track down `FlushConsoleInputBuffer` failures and fully disable console work for IOCP contexts if still triggered. - Validate access-denied fallback for per-file context threadpool minimum failures at higher concurrency. - Recheck named-pipe endpoint handling if `PerfectHashServer-StressSys32` continues to fail. - Decide whether BulkCreateDirectory should accept a single-directory short form or keep output dir required. - Exercise IOCP file work dispatch path (bulk create) to confirm outstanding event signaling and non-threadpool file work callbacks. -- Validate CHM01 async path with smaller keysets (e.g. `hard`) and record timing vs legacy. -- Identify root cause of remaining IOCP bulk-create failures (`E_UNEXPECTED` on `shell32-13803.keys` / `CoreUIComponents-7995.keys`), using new `LastError` logging. -- Verify `MiniDumpWriteDump` retry-with-NULL covers real crash scenarios; decide whether to keep exception-pointer dumps as the primary path. -- Investigate IOCP bulk-create async hang on `hard` keyset (client wait never completes). -- Confirm bulk request completion signaling and outstanding-count decrements when all files complete (hard run produced all 23 outputs but token never signaled). -- Investigate unexpected ~1500 threadpool worker threads (`ZwWaitForWorkViaWorkerFactory`) observed during IOCP server runs; identify source (RPC/TP APIs) and whether it contributes to the hang. +- Identify root cause of remaining IOCP bulk-create failures (`E_UNEXPECTED` on `shell32-13803.keys` / `CoreUIComponents-7995.keys`) if they recur. +- Investigate unexpected ~1500 threadpool worker threads (`ZwWaitForWorkViaWorkerFactory`) observed during IOCP server runs; identify source (RPC/TP APIs) and whether it contributes to hangs. - Validate context file work skip fix for `SetEndOfFile`/`PerfectHashFileTruncate` 1224 failures; decide whether to generate context files once per bulk request. -- Wire IOCP server bulk path to CHM01 async jobs and request completion callbacks. -- Run a clean build after the IOCP-native arg parsing/context creation changes and confirm no duplicate symbol issues. - Decide whether `PerfectHashClientExe` should wait on bulk-create tokens for `BulkCreate=` requests now that the server routes them through the bulk-directory path. - Choose a node-selection policy for single table-create server requests (round-robin vs node 0 vs affinity). - Add IOCP work item queueing/state to drive per-request pipelines. - Integrate per-request concurrency caps and queueing policies. -- Draft IOCP-README.md once protocol and dispatch behavior settle. +- Validate per-file concurrency ramp behavior (`--InitialPerFileConcurrency`, `--MaxPerFileConcurrency`, `--IncreaseConcurrencyAfterMilliseconds`) on `hard` and sys32 subsets; confirm no early failure. +- Capture and diagnose `PerfectHashServer.exe` `0xC0000005` crash on `hard` (Release, `--NoFileIo`, concurrency 4); set `PH_SERVER_CRASH_DIR` for minidumps if needed. +- Draft `IOCP-README.md` once protocol and dispatch behavior settle. diff --git a/USAGE.txt b/USAGE.txt index aa5a9cc4..8caa4a7e 100644 --- a/USAGE.txt +++ b/USAGE.txt @@ -719,6 +719,24 @@ Table Create Parameters: Supplies the maximum number of seconds to try and solve an individual graph. + --InitialPerFileConcurrency=N + + Sets the initial number of graph solver work items to dispatch per + keys file (default: 1). This is useful for IOCP-driven bulk runs + where most key sets solve quickly and only "hard" sets should ramp + up additional concurrency. + + --MaxPerFileConcurrency=N + + Caps the maximum number of graph solver work items per keys file. + Defaults to the maximum concurrency supplied on the command line. + + --IncreaseConcurrencyAfterMilliseconds=N + + When set, and the first solver work item has been running longer than + the supplied interval, an additional solver work item will be queued + (up to --MaxPerFileConcurrency). Defaults to 500ms; use 0 to disable. + --FunctionHookCallbackDllPath= Supplies a fully-qualified path to a .dll file that will be used as the @@ -811,4 +829,3 @@ Mask Functions: ID | Name 2 And - diff --git a/include/PerfectHash.h b/include/PerfectHash.h index aa55d49a..8ff7cea3 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -3763,6 +3763,9 @@ IsValidPerfectHashCuRngId( ENTRY(Seed3Byte1MaskCounts) \ ENTRY(Seed3Byte2MaskCounts) \ ENTRY(MaxSolveTimeInSeconds) \ + ENTRY(InitialPerFileConcurrency) \ + ENTRY(MaxPerFileConcurrency) \ + ENTRY(IncreaseConcurrencyAfterMilliseconds) \ ENTRY(AutoResizeWhenKeysToEdgesRatioExceeds) \ ENTRY(FunctionHookCallbackDllPath) \ ENTRY(FunctionHookCallbackFunctionName) \ @@ -4272,6 +4275,24 @@ HRESULT typedef PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY *PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY; +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG MaximumThreads + ); +typedef PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS + *PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _Out_ PULONG MaximumThreads + ); +typedef PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS + *PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS; + typedef HRESULT (STDAPICALLTYPE PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK)( @@ -4446,6 +4467,8 @@ typedef struct _PERFECT_HASH_CONTEXT_IOCP_VTBL { PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY SetMaximumConcurrency; PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY GetMaximumConcurrency; + PPERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS SetMaximumThreads; + PPERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS GetMaximumThreads; PPERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK SetNumaNodeMask; PPERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK GetNumaNodeMask; PPERFECT_HASH_CONTEXT_IOCP_SET_BASE_OUTPUT_DIRECTORY SetBaseOutputDirectory; @@ -4587,6 +4610,24 @@ HRESULT typedef PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY *PPERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY; +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ ULONG MaximumThreads + ); +typedef PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS + *PPERFECT_HASH_SERVER_SET_MAXIMUM_THREADS; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PULONG MaximumThreads + ); +typedef PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS + *PPERFECT_HASH_SERVER_GET_MAXIMUM_THREADS; + typedef HRESULT (STDAPICALLTYPE PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK)( @@ -4641,6 +4682,42 @@ HRESULT typedef PERFECT_HASH_SERVER_GET_LOCAL_ONLY *PPERFECT_HASH_SERVER_GET_LOCAL_ONLY; +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_VERBOSE)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN Verbose + ); +typedef PERFECT_HASH_SERVER_SET_VERBOSE + *PPERFECT_HASH_SERVER_SET_VERBOSE; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_VERBOSE)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN Verbose + ); +typedef PERFECT_HASH_SERVER_GET_VERBOSE + *PPERFECT_HASH_SERVER_GET_VERBOSE; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_NO_FILE_IO)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN NoFileIo + ); +typedef PERFECT_HASH_SERVER_SET_NO_FILE_IO + *PPERFECT_HASH_SERVER_SET_NO_FILE_IO; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_NO_FILE_IO)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN NoFileIo + ); +typedef PERFECT_HASH_SERVER_GET_NO_FILE_IO + *PPERFECT_HASH_SERVER_GET_NO_FILE_IO; + typedef HRESULT (STDAPICALLTYPE PERFECT_HASH_SERVER_START)( @@ -4676,12 +4753,18 @@ typedef struct _PERFECT_HASH_SERVER_VTBL { PPERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY SetMaximumConcurrency; PPERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY GetMaximumConcurrency; + PPERFECT_HASH_SERVER_SET_MAXIMUM_THREADS SetMaximumThreads; + PPERFECT_HASH_SERVER_GET_MAXIMUM_THREADS GetMaximumThreads; PPERFECT_HASH_SERVER_SET_NUMA_NODE_MASK SetNumaNodeMask; PPERFECT_HASH_SERVER_GET_NUMA_NODE_MASK GetNumaNodeMask; PPERFECT_HASH_SERVER_SET_ENDPOINT SetEndpoint; PPERFECT_HASH_SERVER_GET_ENDPOINT GetEndpoint; PPERFECT_HASH_SERVER_SET_LOCAL_ONLY SetLocalOnly; PPERFECT_HASH_SERVER_GET_LOCAL_ONLY GetLocalOnly; + PPERFECT_HASH_SERVER_SET_VERBOSE SetVerbose; + PPERFECT_HASH_SERVER_GET_VERBOSE GetVerbose; + PPERFECT_HASH_SERVER_SET_NO_FILE_IO SetNoFileIo; + PPERFECT_HASH_SERVER_GET_NO_FILE_IO GetNoFileIo; PPERFECT_HASH_SERVER_START Start; PPERFECT_HASH_SERVER_STOP Stop; PPERFECT_HASH_SERVER_WAIT Wait; diff --git a/scripts/iocp-smoke.ps1 b/scripts/iocp-smoke.ps1 index 0e6637ea..2ea04c9f 100644 --- a/scripts/iocp-smoke.ps1 +++ b/scripts/iocp-smoke.ps1 @@ -3,13 +3,20 @@ param( [string]$Config = "Debug", [string]$Endpoint = "\\.\\pipe\\PerfectHashServer-Smoke", [int]$TimeoutSeconds = 10, + [int]$IocpConcurrency = 0, + [int]$MaxThreads = 0, [bool]$WaitForServer = $true, - [int]$ConnectTimeoutMs = 10000 + [int]$ConnectTimeoutMs = 10000, + [bool]$NoFileIo = $true, + [bool]$VerboseServer = $false ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" +Remove-Item Env:PH_LOG_BULK_CREATE_COUNTS -ErrorAction SilentlyContinue +Remove-Item Env:PH_LOG_CHM01_ASYNC_JOB -ErrorAction SilentlyContinue + $serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) $clientExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashClient.exe" -f $Config) $workDir = Join-Path $BuildDir "iocp-smoke" @@ -48,6 +55,18 @@ $createCommand = "PerfectHashCreate.exe $keysPath $outputPath " + $tableArg = "--TableCreate=$createCommand" $serverArgs = @("--Endpoint=$Endpoint") +if ($IocpConcurrency -gt 0) { + $serverArgs += ("--IocpConcurrency={0}" -f $IocpConcurrency) +} +if ($MaxThreads -gt 0) { + $serverArgs += ("--MaxThreads={0}" -f $MaxThreads) +} +if ($NoFileIo) { + $serverArgs += "--NoFileIo" +} +if ($VerboseServer) { + $serverArgs += "--Verbose" +} $clientArgs = @("--Endpoint=$Endpoint", "--Shutdown") if ($WaitForServer) { $clientArgs += "--WaitForServer" diff --git a/scripts/stress-sys32-iocp.ps1 b/scripts/stress-sys32-iocp.ps1 index f1fa87b1..ee96d349 100644 --- a/scripts/stress-sys32-iocp.ps1 +++ b/scripts/stress-sys32-iocp.ps1 @@ -8,15 +8,22 @@ param( [string]$HashFunction = "Mulshrolate4RX", [string]$MaskFunction = "And", [int]$MaximumConcurrency = 0, + [int]$IocpConcurrency = 0, + [int]$MaxThreads = 0, [string[]]$ExtraArgs = @(), [int]$TimeoutSeconds = 300, [bool]$WaitForServer = $true, - [int]$ConnectTimeoutMs = 10000 + [int]$ConnectTimeoutMs = 10000, + [bool]$NoFileIo = $true, + [bool]$VerboseServer = $false ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" +Remove-Item Env:PH_LOG_BULK_CREATE_COUNTS -ErrorAction SilentlyContinue +Remove-Item Env:PH_LOG_CHM01_ASYNC_JOB -ErrorAction SilentlyContinue + $serverExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashServer.exe" -f $Config) $clientExe = Join-Path $BuildDir ("bin\\{0}\\PerfectHashClient.exe" -f $Config) @@ -54,8 +61,22 @@ if ($ExtraArgs -and $ExtraArgs.Count -gt 0) { $bulkArg = "--BulkCreateDirectory=$bulkCommand" +$serverArgs = @("--Endpoint=$Endpoint") +if ($IocpConcurrency -gt 0) { + $serverArgs += ("--IocpConcurrency={0}" -f $IocpConcurrency) +} +if ($MaxThreads -gt 0) { + $serverArgs += ("--MaxThreads={0}" -f $MaxThreads) +} +if ($NoFileIo) { + $serverArgs += "--NoFileIo" +} +if ($VerboseServer) { + $serverArgs += "--Verbose" +} + $server = Start-Process -FilePath $serverExe ` - -ArgumentList @("--Endpoint=$Endpoint") ` + -ArgumentList $serverArgs ` -PassThru ` -NoNewWindow diff --git a/src/PerfectHash/Chm01.c b/src/PerfectHash/Chm01.c index 5ffad096..5cba535a 100644 --- a/src/PerfectHash/Chm01.c +++ b/src/PerfectHash/Chm01.c @@ -2432,7 +2432,9 @@ Return Value: Result = S_OK; } - PerfectHashPrintMessage(PH_MSG_PERFECT_HASH_CONSOLE_KEYS_HELP); + if (Context->ConsoleWork != NULL) { + PerfectHashPrintMessage(PH_MSG_PERFECT_HASH_CONSOLE_KEYS_HELP); + } // // Intentional follow-on to End. diff --git a/src/PerfectHash/Chm01Async.c b/src/PerfectHash/Chm01Async.c index 9be308b8..45df72d6 100644 --- a/src/PerfectHash/Chm01Async.c +++ b/src/PerfectHash/Chm01Async.c @@ -164,10 +164,17 @@ Chm01AsyncGraphComplete( static HRESULT -Chm01AsyncDispatchGraphWork( +Chm01AsyncPrepareGraphs( _In_ PCHM01_ASYNC_JOB Job ); +static +HRESULT +Chm01AsyncDispatchGraphWork( + _In_ PCHM01_ASYNC_JOB Job, + _In_ ULONG DispatchCount + ); + static HRESULT Chm01AsyncInitializeJob( @@ -180,6 +187,12 @@ Chm01AsyncPollSolveEvents( _In_ PCHM01_ASYNC_JOB Job ); +static +HRESULT +Chm01AsyncMaybeIncreaseConcurrency( + _In_ PCHM01_ASYNC_JOB Job + ); + static HRESULT Chm01AsyncFinalizeVerify( @@ -230,15 +243,25 @@ Chm01AsyncGraphStep( ULONG OldNumberOfVertices; HRESULT Result; PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_JOB Job; PCHM01_ASYNC_GRAPH_WORK GraphWork; GraphWork = CONTAINING_RECORD(Work, CHM01_ASYNC_GRAPH_WORK, Work); Graph = GraphWork->Graph; - Context = GraphWork->Job->Context; + Job = GraphWork->Job; + Context = Job->Context; if (!GraphWork->Started) { GraphWork->Started = TRUE; InterlockedIncrement(&Context->ActiveSolvingLoops); + if (InterlockedCompareExchange(&Job->FirstGraphStarted, 1, 0) == 0) { + Job->FirstGraphStartMilliseconds = GetTickCount64(); + if (Job->IncreaseConcurrencyAfterMilliseconds > 0) { + Job->NextConcurrencyIncreaseMilliseconds = + Job->FirstGraphStartMilliseconds + + Job->IncreaseConcurrencyAfterMilliseconds; + } + } } LockedGraph = Graph; @@ -392,28 +415,18 @@ Chm01AsyncGraphComplete( _Use_decl_annotations_ static HRESULT -Chm01AsyncDispatchGraphWork( +Chm01AsyncPrepareGraphs( PCHM01_ASYNC_JOB Job ) { HRESULT Result; ULONG Index; - ULONG ActiveGraphs; PGRAPH Graph; PGRAPH *Graphs; PPERFECT_HASH_CONTEXT Context; - PCHM01_ASYNC_GRAPH_WORK GraphWork; Context = Job->Context; Graphs = Job->Graphs; - ActiveGraphs = 0; - - if (Job->GraphsCompleteEvent) { - ResetEvent(Job->GraphsCompleteEvent); - } - - ASSERT(Context->MainWorkList->Vtbl->IsEmpty(Context->MainWorkList)); - ASSERT(Context->FinishedWorkList->Vtbl->IsEmpty(Context->FinishedWorkList)); if (FirstSolvedGraphWins(Context)) { ASSERT(Job->NumberOfGraphs == Job->Concurrency); @@ -450,6 +463,61 @@ Chm01AsyncDispatchGraphWork( } Graph->Flags.IsSpare = FALSE; + } + + return S_OK; +} + +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncDispatchGraphWork( + PCHM01_ASYNC_JOB Job, + ULONG DispatchCount + ) +{ + HRESULT Result; + ULONG Index; + ULONG GraphIndex; + ULONG Remaining; + PGRAPH Graph; + PGRAPH *Graphs; + PPERFECT_HASH_CONTEXT Context; + PCHM01_ASYNC_GRAPH_WORK GraphWork; + + Context = Job->Context; + Graphs = Job->Graphs; + + Remaining = Job->Concurrency - Job->DispatchedGraphs; + if (DispatchCount == 0 || Remaining == 0) { + return S_OK; + } + + if (DispatchCount > Remaining) { + DispatchCount = Remaining; + } + + if (Job->GraphsCompleteEvent) { + ResetEvent(Job->GraphsCompleteEvent); + } + + if (Job->DispatchedGraphs == 0) { + ASSERT(Context->MainWorkList->Vtbl->IsEmpty(Context->MainWorkList)); + ASSERT(Context->FinishedWorkList->Vtbl->IsEmpty(Context->FinishedWorkList)); + } + + for (Index = 0; Index < DispatchCount; Index++) { + + GraphIndex = Job->DispatchedGraphs; + if (!FirstSolvedGraphWins(Context)) { + GraphIndex++; + } + + if (GraphIndex >= Job->NumberOfGraphs) { + return PH_E_INVARIANT_CHECK_FAILED; + } + + Graph = Graphs[GraphIndex]; GraphWork = (PCHM01_ASYNC_GRAPH_WORK)( Job->Allocator->Vtbl->Calloc( @@ -469,7 +537,10 @@ Chm01AsyncDispatchGraphWork( GraphWork->Work.Complete = Chm01AsyncGraphComplete; GraphWork->Work.SliceBudget = Job->SliceBudget; - ActiveGraphs++; + InterlockedIncrement(&Context->GraphMemoryFailures); + InterlockedIncrement(&Context->RemainingSolverLoops); + InterlockedIncrement(&Job->ActiveGraphs); + Job->DispatchedGraphs++; Result = PerfectHashAsyncSubmit(&Job->Async, &GraphWork->Work); if (FAILED(Result)) { @@ -477,12 +548,6 @@ Chm01AsyncDispatchGraphWork( } } - Job->ActiveGraphs = (LONG)ActiveGraphs; - - if (ActiveGraphs == 0 && Job->GraphsCompleteEvent) { - SetEvent(Job->GraphsCompleteEvent); - } - return S_OK; } @@ -497,6 +562,7 @@ Chm01AsyncInitializeJob( HRESULT Result; BOOLEAN LimitConcurrency; ULONG Concurrency; + ULONG InitialConcurrency; ULONG NumberOfGraphs; ULONG NumberOfSeedsRequired; ULONG NumberOfSeedsAvailable; @@ -574,7 +640,13 @@ Chm01AsyncInitializeJob( // Determine concurrency. // - Concurrency = Context->MaximumConcurrency; + Concurrency = Context->MaxPerFileConcurrency; + if (Concurrency == 0) { + Concurrency = Context->MaximumConcurrency; + } + if (Concurrency == 0) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } LimitConcurrency = ( Table->PriorPredictedAttempts > 0 && @@ -585,7 +657,27 @@ Chm01AsyncInitializeJob( Concurrency = min(Concurrency, Table->PriorPredictedAttempts); } + if (Concurrency == 0) { + return PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + + InitialConcurrency = Context->InitialPerFileConcurrency; + if (InitialConcurrency == 0) { + InitialConcurrency = 1; + } + + if (InitialConcurrency > Concurrency) { + InitialConcurrency = Concurrency; + } + Job->Concurrency = Concurrency; + Job->InitialConcurrency = InitialConcurrency; + Job->DispatchedGraphs = 0; + Job->IncreaseConcurrencyAfterMilliseconds = + Context->IncreaseConcurrencyAfterMilliseconds; + Job->FirstGraphStarted = 0; + Job->FirstGraphStartMilliseconds = 0; + Job->NextConcurrencyIncreaseMilliseconds = 0; if (FirstSolvedGraphWins(Context)) { NumberOfGraphs = Concurrency; @@ -695,6 +787,15 @@ Chm01AsyncInitializeJob( goto Error; } + // + // Prepare all graphs for solving. + // + + Result = Chm01AsyncPrepareGraphs(Job); + if (FAILED(Result)) { + goto Error; + } + // // Reset all context events. // @@ -800,8 +901,8 @@ Chm01AsyncInitializeJob( // Reset counters. // - Context->GraphMemoryFailures = Concurrency; - Context->RemainingSolverLoops = Concurrency; + Context->GraphMemoryFailures = 0; + Context->RemainingSolverLoops = 0; Context->ActiveSolvingLoops = 0; Context->Attempts = 0; Context->FailedAttempts = 0; @@ -812,7 +913,7 @@ Chm01AsyncInitializeJob( // Dispatch graph work. // - Result = Chm01AsyncDispatchGraphWork(Job); + Result = Chm01AsyncDispatchGraphWork(Job, Job->InitialConcurrency); if (FAILED(Result)) { goto Error; } @@ -843,6 +944,7 @@ Chm01AsyncPollSolveEvents( PCHM01_ASYNC_JOB Job ) { + HRESULT Result; ULONG WaitResult; PPERFECT_HASH_CONTEXT Context; @@ -854,6 +956,10 @@ Chm01AsyncPollSolveEvents( 0); if (WaitResult == WAIT_TIMEOUT) { + Result = Chm01AsyncMaybeIncreaseConcurrency(Job); + if (FAILED(Result)) { + return Result; + } return S_FALSE; } @@ -871,6 +977,51 @@ Chm01AsyncPollSolveEvents( return S_OK; } +_Use_decl_annotations_ +static +HRESULT +Chm01AsyncMaybeIncreaseConcurrency( + PCHM01_ASYNC_JOB Job + ) +{ + HRESULT Result; + ULONGLONG Now; + PPERFECT_HASH_CONTEXT Context; + + Context = Job->Context; + + if (Job->IncreaseConcurrencyAfterMilliseconds == 0) { + return S_OK; + } + + if (Job->FirstGraphStarted == 0) { + return S_OK; + } + + if (Job->DispatchedGraphs >= Job->Concurrency) { + return S_OK; + } + + if (StopSolving(Context)) { + return S_OK; + } + + Now = GetTickCount64(); + if (Now < Job->NextConcurrencyIncreaseMilliseconds) { + return S_OK; + } + + Result = Chm01AsyncDispatchGraphWork(Job, 1); + if (FAILED(Result)) { + return Result; + } + + Job->NextConcurrencyIncreaseMilliseconds = + Now + Job->IncreaseConcurrencyAfterMilliseconds; + + return S_OK; +} + _Use_decl_annotations_ static HRESULT diff --git a/src/PerfectHash/Chm01Async.h b/src/PerfectHash/Chm01Async.h index 5d8e2e8f..1acbdb74 100644 --- a/src/PerfectHash/Chm01Async.h +++ b/src/PerfectHash/Chm01Async.h @@ -57,8 +57,14 @@ typedef struct _CHM01_ASYNC_JOB { PGRAPH *Graphs; ULONG NumberOfGraphs; ULONG Concurrency; + ULONG InitialConcurrency; + ULONG DispatchedGraphs; + ULONG IncreaseConcurrencyAfterMilliseconds; ULONG Attempt; + volatile LONG FirstGraphStarted; ULONG Padding1; + ULONGLONG FirstGraphStartMilliseconds; + ULONGLONG NextConcurrencyIncreaseMilliseconds; GRAPH_INFO GraphInfo; GRAPH_INFO PrevGraphInfo; GRAPH_INFO_ON_DISK GraphInfoOnDisk; diff --git a/src/PerfectHash/ExtractArg.c b/src/PerfectHash/ExtractArg.c index c501a832..5949df16 100644 --- a/src/PerfectHash/ExtractArg.c +++ b/src/PerfectHash/ExtractArg.c @@ -1040,6 +1040,14 @@ Return Value: ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(MaxSolveTimeInSeconds); + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(InitialPerFileConcurrency); + + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(MaxPerFileConcurrency); + + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER( + IncreaseConcurrencyAfterMilliseconds + ); + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(FunctionHookCallbackIgnoreRip); #define IS_VALUE_EQUAL(ValueName) \ diff --git a/src/PerfectHash/PerfectHashConstants.c b/src/PerfectHash/PerfectHashConstants.c index d2c9178e..ce8252c6 100644 --- a/src/PerfectHash/PerfectHashConstants.c +++ b/src/PerfectHash/PerfectHashConstants.c @@ -1136,6 +1136,8 @@ const PERFECT_HASH_CONTEXT_IOCP_VTBL PerfectHashContextIocpInterface = { (PPERFECT_HASH_CONTEXT_IOCP_LOCK_SERVER)&ComponentLockServer, &PerfectHashContextIocpSetMaximumConcurrency, &PerfectHashContextIocpGetMaximumConcurrency, + &PerfectHashContextIocpSetMaximumThreads, + &PerfectHashContextIocpGetMaximumThreads, &PerfectHashContextIocpSetNumaNodeMask, &PerfectHashContextIocpGetNumaNodeMask, &PerfectHashContextIocpSetBaseOutputDirectory, @@ -1154,7 +1156,7 @@ const PERFECT_HASH_CONTEXT_IOCP_VTBL PerfectHashContextIocpInterface = { NULL, #endif }; -VERIFY_VTBL_SIZE(PERFECT_HASH_CONTEXT_IOCP, 14); +VERIFY_VTBL_SIZE(PERFECT_HASH_CONTEXT_IOCP, 16); // // PerfectHashServer @@ -1168,18 +1170,24 @@ const PERFECT_HASH_SERVER_VTBL PerfectHashServerInterface = { (PPERFECT_HASH_SERVER_LOCK_SERVER)&ComponentLockServer, &PerfectHashServerSetMaximumConcurrency, &PerfectHashServerGetMaximumConcurrency, + &PerfectHashServerSetMaximumThreads, + &PerfectHashServerGetMaximumThreads, &PerfectHashServerSetNumaNodeMask, &PerfectHashServerGetNumaNodeMask, &PerfectHashServerSetEndpoint, &PerfectHashServerGetEndpoint, &PerfectHashServerSetLocalOnly, &PerfectHashServerGetLocalOnly, + &PerfectHashServerSetVerbose, + &PerfectHashServerGetVerbose, + &PerfectHashServerSetNoFileIo, + &PerfectHashServerGetNoFileIo, &PerfectHashServerStart, &PerfectHashServerStop, &PerfectHashServerWait, &PerfectHashServerSubmitRequest, }; -VERIFY_VTBL_SIZE(PERFECT_HASH_SERVER, 12); +VERIFY_VTBL_SIZE(PERFECT_HASH_SERVER, 18); // // PerfectHashClient diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index ddb0a19b..039e95b5 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -488,6 +488,9 @@ Return Value: Context->MinimumConcurrency = MaximumConcurrency; Context->MaximumConcurrency = MaximumConcurrency; + Context->InitialPerFileConcurrency = 1; + Context->MaxPerFileConcurrency = MaximumConcurrency; + Context->IncreaseConcurrencyAfterMilliseconds = 500; ThreadpoolConcurrency = GetMainThreadpoolConcurrency(MaximumConcurrency); diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index aab10538..29fc6e79 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -1035,6 +1035,10 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { PTP_TIMER SolveTimeout; ULONG MinimumConcurrency; ULONG MaximumConcurrency; + ULONG InitialPerFileConcurrency; + ULONG MaxPerFileConcurrency; + ULONG IncreaseConcurrencyAfterMilliseconds; + ULONG Padding10; union { ULONG CuConcurrency; ULONG NumberOfGpuThreads; diff --git a/src/PerfectHash/PerfectHashContextBulkCreate.c b/src/PerfectHash/PerfectHashContextBulkCreate.c index e6deacfc..60c9a064 100644 --- a/src/PerfectHash/PerfectHashContextBulkCreate.c +++ b/src/PerfectHash/PerfectHashContextBulkCreate.c @@ -1618,6 +1618,27 @@ Return Value: break; } + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractBulkCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + // // If we failed, clean up the table create parameters. If that fails, // report the error, then replace our return value error code with that diff --git a/src/PerfectHash/PerfectHashContextIocp.c b/src/PerfectHash/PerfectHashContextIocp.c index 174a3f46..c4d1b1ba 100644 --- a/src/PerfectHash/PerfectHashContextIocp.c +++ b/src/PerfectHash/PerfectHashContextIocp.c @@ -112,10 +112,8 @@ Return Value: Rtl = ContextIocp->Rtl; - ContextIocp->MaximumConcurrency = Rtl->CpuFeatures.LogicalProcessorCount; - if (ContextIocp->MaximumConcurrency == 0) { - ContextIocp->MaximumConcurrency = 1; - } + ContextIocp->IocpConcurrency = 0; + ContextIocp->MaxWorkerThreads = 0; ContextIocp->NumaNodeCount = Rtl->CpuFeatures.NumaNodeCount; if (ContextIocp->NumaNodeCount == 0) { @@ -624,40 +622,49 @@ PerfectHashContextIocpEnumerateNumaNodes( } // - // Distribute worker thread counts across nodes. + // Configure per-node IOCP concurrency and worker thread counts. // { ULONG Index; - ULONG TargetConcurrency; - ULONG Remaining; - - TargetConcurrency = ContextIocp->MaximumConcurrency; - if (TargetConcurrency == 0 || TargetConcurrency > TotalProcessors) { - TargetConcurrency = TotalProcessors; - } - - Remaining = TargetConcurrency; + ULONG TotalWorkerThreads = 0; + ULONG DefaultIocpConcurrency = ContextIocp->IocpConcurrency; + ULONG DefaultMaxThreads = ContextIocp->MaxWorkerThreads; for (Index = 0; Index < ContextIocp->NodeCount; Index++) { PPERFECT_HASH_IOCP_NODE Node = &Nodes[Index]; - ULONG Share = (Node->ProcessorCount * TargetConcurrency) / - TotalProcessors; - Node->WorkerThreadCount = Share; - Remaining -= Share; - } + ULONG IocpConcurrency; + ULONG MaxThreads; + + IocpConcurrency = DefaultIocpConcurrency; + if (IocpConcurrency == 0 || + IocpConcurrency > Node->ProcessorCount) { + IocpConcurrency = Node->ProcessorCount; + } - Index = 0; - while (Remaining) { - Nodes[Index].WorkerThreadCount++; - Remaining--; - Index++; - if (Index == ContextIocp->NodeCount) { - Index = 0; + if (IocpConcurrency == 0) { + IocpConcurrency = 1; } + + MaxThreads = DefaultMaxThreads; + if (MaxThreads == 0) { + if (DefaultIocpConcurrency != 0) { + MaxThreads = IocpConcurrency * 2; + } else { + MaxThreads = Node->ProcessorCount; + } + } + + if (MaxThreads == 0) { + MaxThreads = 1; + } + + Node->IocpConcurrency = IocpConcurrency; + Node->WorkerThreadCount = MaxThreads; + TotalWorkerThreads += MaxThreads; } - ContextIocp->TotalWorkerThreadCount = TargetConcurrency; + ContextIocp->TotalWorkerThreadCount = TotalWorkerThreads; } return S_OK; @@ -687,7 +694,7 @@ PerfectHashContextIocpCreateIoCompletionPorts( INVALID_HANDLE_VALUE, NULL, 0, - Node->ProcessorCount + Node->IocpConcurrency ); if (!Node->IoCompletionPort) { @@ -764,7 +771,7 @@ PerfectHashContextIocpCreateWorkerThreads( } PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY - PerfectHashContextIocpSetMaximumConcurrency; +PerfectHashContextIocpSetMaximumConcurrency; _Use_decl_annotations_ HRESULT @@ -785,7 +792,7 @@ PerfectHashContextIocpSetMaximumConcurrency( return PH_E_CONTEXT_LOCKED; } - ContextIocp->MaximumConcurrency = MaximumConcurrency; + ContextIocp->IocpConcurrency = MaximumConcurrency; ReleasePerfectHashContextIocpLockExclusive(ContextIocp); @@ -793,7 +800,7 @@ PerfectHashContextIocpSetMaximumConcurrency( } PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY - PerfectHashContextIocpGetMaximumConcurrency; +PerfectHashContextIocpGetMaximumConcurrency; _Use_decl_annotations_ HRESULT @@ -810,7 +817,58 @@ PerfectHashContextIocpGetMaximumConcurrency( return E_POINTER; } - *MaximumConcurrency = ContextIocp->MaximumConcurrency; + *MaximumConcurrency = ContextIocp->IocpConcurrency; + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS + PerfectHashContextIocpSetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpSetMaximumThreads( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + ULONG MaximumThreads + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (MaximumThreads == 0) { + return E_INVALIDARG; + } + + if (!TryAcquirePerfectHashContextIocpLockExclusive(ContextIocp)) { + return PH_E_CONTEXT_LOCKED; + } + + ContextIocp->MaxWorkerThreads = MaximumThreads; + + ReleasePerfectHashContextIocpLockExclusive(ContextIocp); + + return S_OK; +} + +PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS + PerfectHashContextIocpGetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashContextIocpGetMaximumThreads( + PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + PULONG MaximumThreads + ) +{ + if (!ARGUMENT_PRESENT(ContextIocp)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumThreads)) { + return E_POINTER; + } + + *MaximumThreads = ContextIocp->MaxWorkerThreads; return S_OK; } @@ -1083,14 +1141,23 @@ PerfectHashContextIocpInvokeLegacyContextArgvW( return Result; } - if (ContextIocp->MaximumConcurrency > 0) { - Result = Context->Vtbl->SetMaximumConcurrency( - Context, - ContextIocp->MaximumConcurrency - ); - if (FAILED(Result)) { - RELEASE(Context); - return Result; + { + ULONG Concurrency; + + Concurrency = ContextIocp->MaxWorkerThreads; + if (Concurrency == 0) { + Concurrency = ContextIocp->IocpConcurrency; + } + + if (Concurrency > 0) { + Result = Context->Vtbl->SetMaximumConcurrency( + Context, + Concurrency + ); + if (FAILED(Result)) { + RELEASE(Context); + return Result; + } } } diff --git a/src/PerfectHash/PerfectHashContextIocp.h b/src/PerfectHash/PerfectHashContextIocp.h index b86e78c0..c23d0478 100644 --- a/src/PerfectHash/PerfectHashContextIocp.h +++ b/src/PerfectHash/PerfectHashContextIocp.h @@ -92,6 +92,8 @@ typedef struct _PERFECT_HASH_IOCP_NODE { struct _PERFECT_HASH_CONTEXT_IOCP *ContextIocp; ULONG NodeId; ULONG ProcessorCount; + ULONG IocpConcurrency; + ULONG Padding1; #ifdef PH_WINDOWS GROUP_AFFINITY GroupAffinity; #else @@ -100,7 +102,7 @@ typedef struct _PERFECT_HASH_IOCP_NODE { HANDLE IoCompletionPort; HANDLE *WorkerThreads; ULONG WorkerThreadCount; - ULONG Padding1; + ULONG Padding2; } PERFECT_HASH_IOCP_NODE; typedef PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; @@ -112,11 +114,13 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT_IOCP { // Configuration. // - ULONG MaximumConcurrency; + ULONG IocpConcurrency; + ULONG MaxWorkerThreads; ULONG NumaNodeCount; - PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; ULONG Padding1; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; ULONG Padding2; + ULONG Padding3; // // Pointer to the base output directory, if set. @@ -132,7 +136,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT_IOCP { ULONG NodeCount; ULONG TotalWorkerThreadCount; ULONG IoCompletionPortCount; - ULONG Padding3; + ULONG Padding4; // // Completion dispatch callback (owned by higher-level components). @@ -223,6 +227,10 @@ extern PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_CONCURRENCY PerfectHashContextIocpSetMaximumConcurrency; extern PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_CONCURRENCY PerfectHashContextIocpGetMaximumConcurrency; +extern PERFECT_HASH_CONTEXT_IOCP_SET_MAXIMUM_THREADS + PerfectHashContextIocpSetMaximumThreads; +extern PERFECT_HASH_CONTEXT_IOCP_GET_MAXIMUM_THREADS + PerfectHashContextIocpGetMaximumThreads; extern PERFECT_HASH_CONTEXT_IOCP_SET_NUMA_NODE_MASK PerfectHashContextIocpSetNumaNodeMask; extern PERFECT_HASH_CONTEXT_IOCP_GET_NUMA_NODE_MASK diff --git a/src/PerfectHash/PerfectHashContextIocpArgs.c b/src/PerfectHash/PerfectHashContextIocpArgs.c index c52cc042..1897277e 100644 --- a/src/PerfectHash/PerfectHashContextIocpArgs.c +++ b/src/PerfectHash/PerfectHashContextIocpArgs.c @@ -383,6 +383,39 @@ Return Value: break; } + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractBulkCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + + if (SUCCEEDED(Result) && + *MaximumConcurrency == 0 && + ContextIocp->IocpConcurrency > 0) { + *MaximumConcurrency = ContextIocp->IocpConcurrency; + } + + if (SUCCEEDED(Result) && + ContextIocp->IocpConcurrency > 0 && + *MaximumConcurrency > ContextIocp->IocpConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + // // If we failed, clean up the table create parameters. If that fails, // report the error, then replace our return value error code with that @@ -729,6 +762,39 @@ Return Value: break; } + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractTableCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + + if (SUCCEEDED(Result) && + *MaximumConcurrency == 0 && + ContextIocp->IocpConcurrency > 0) { + *MaximumConcurrency = ContextIocp->IocpConcurrency; + } + + if (SUCCEEDED(Result) && + ContextIocp->IocpConcurrency > 0 && + *MaximumConcurrency > ContextIocp->IocpConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } + if (SUCCEEDED(Result)) { // diff --git a/src/PerfectHash/PerfectHashContextTableCreate.c b/src/PerfectHash/PerfectHashContextTableCreate.c index 46b45d2c..c3e80a9f 100644 --- a/src/PerfectHash/PerfectHashContextTableCreate.c +++ b/src/PerfectHash/PerfectHashContextTableCreate.c @@ -1370,6 +1370,27 @@ Return Value: break; } + if (SUCCEEDED(Result)) { + PPERFECT_HASH_TABLE_CREATE_PARAMETER Param = NULL; + HRESULT LookupResult; + + LookupResult = GetTableCreateParameterForId( + TableCreateParameters, + TableCreateParameterMaxPerFileConcurrencyId, + &Param + ); + if (FAILED(LookupResult)) { + PH_ERROR(ExtractTableCreateArgs_GetMaxPerFileConcurrency, LookupResult); + Result = LookupResult; + } else if (LookupResult == S_OK && Param) { + if (Param->AsULong == 0) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + } else { + *MaximumConcurrency = Param->AsULong; + } + } + } + if (SUCCEEDED(Result)) { // diff --git a/src/PerfectHash/PerfectHashFile.c b/src/PerfectHash/PerfectHashFile.c index 120bcd56..9409fde5 100644 --- a/src/PerfectHash/PerfectHashFile.c +++ b/src/PerfectHash/PerfectHashFile.c @@ -700,7 +700,19 @@ Return Value: EndOfFile.QuadPart = File->FileInfo.EndOfFile.QuadPart; - if (!FileCreateFlags.EndOfFileIsExtensionSizeIfFileExists) { + if (EndOfFile.QuadPart == 0) { + + // + // Existing file is empty; treat it like a new file and extend + // it to the requested size. This avoids invalid end-of-file + // failures when mapping zero-length streams. + // + + ASSERT(EndOfFilePointer->QuadPart > 0); + + EndOfFile.QuadPart = EndOfFilePointer->QuadPart; + + } else if (!FileCreateFlags.EndOfFileIsExtensionSizeIfFileExists) { DoTruncate = FALSE; diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c index 57a84aae..062725e4 100644 --- a/src/PerfectHash/PerfectHashServer.c +++ b/src/PerfectHash/PerfectHashServer.c @@ -321,7 +321,8 @@ Return Value: ContextIocp = Server->ContextIocp; - Server->MaximumConcurrency = ContextIocp->MaximumConcurrency; + Server->IocpConcurrency = ContextIocp->IocpConcurrency; + Server->MaxWorkerThreads = ContextIocp->MaxWorkerThreads; Server->NumaNodeMask = ContextIocp->NumaNodeMask; Server->NumaNodeCount = ContextIocp->NumaNodeCount; @@ -338,6 +339,8 @@ Return Value: Server->Endpoint = PerfectHashServerDefaultPipeName; Server->Flags.LocalOnly = TRUE; Server->Flags.EndpointAllocated = FALSE; + Server->Flags.Verbose = FALSE; + Server->Flags.NoFileIo = FALSE; Server->State.Initialized = TRUE; return S_OK; @@ -439,7 +442,7 @@ PerfectHashServerSetMaximumConcurrency( } } - Server->MaximumConcurrency = MaximumConcurrency; + Server->IocpConcurrency = MaximumConcurrency; return S_OK; } @@ -461,7 +464,63 @@ PerfectHashServerGetMaximumConcurrency( return E_POINTER; } - *MaximumConcurrency = Server->MaximumConcurrency; + *MaximumConcurrency = Server->IocpConcurrency; + return S_OK; +} + +PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS + PerfectHashServerSetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetMaximumThreads( + PPERFECT_HASH_SERVER Server, + ULONG MaximumThreads + ) +{ + HRESULT Result = E_UNEXPECTED; + + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (MaximumThreads == 0) { + return E_INVALIDARG; + } + + if (Server->ContextIocp) { + Result = Server->ContextIocp->Vtbl->SetMaximumThreads( + Server->ContextIocp, + MaximumThreads + ); + if (FAILED(Result)) { + return Result; + } + } + + Server->MaxWorkerThreads = MaximumThreads; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS + PerfectHashServerGetMaximumThreads; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetMaximumThreads( + PPERFECT_HASH_SERVER Server, + PULONG MaximumThreads + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(MaximumThreads)) { + return E_POINTER; + } + + *MaximumThreads = Server->MaxWorkerThreads; return S_OK; } @@ -643,6 +702,90 @@ PerfectHashServerGetLocalOnly( return S_OK; } +PERFECT_HASH_SERVER_SET_VERBOSE PerfectHashServerSetVerbose; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetVerbose( + PPERFECT_HASH_SERVER Server, + BOOLEAN Verbose + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.Verbose = Verbose ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_VERBOSE PerfectHashServerGetVerbose; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetVerbose( + PPERFECT_HASH_SERVER Server, + PBOOLEAN Verbose + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Verbose)) { + return E_POINTER; + } + + *Verbose = (Server->Flags.Verbose != 0); + return S_OK; +} + +PERFECT_HASH_SERVER_SET_NO_FILE_IO PerfectHashServerSetNoFileIo; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetNoFileIo( + PPERFECT_HASH_SERVER Server, + BOOLEAN NoFileIo + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.NoFileIo = NoFileIo ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_NO_FILE_IO PerfectHashServerGetNoFileIo; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetNoFileIo( + PPERFECT_HASH_SERVER Server, + PBOOLEAN NoFileIo + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(NoFileIo)) { + return E_POINTER; + } + + *NoFileIo = (Server->Flags.NoFileIo != 0); + return S_OK; +} + PERFECT_HASH_SERVER_START PerfectHashServerStart; _Use_decl_annotations_ @@ -2449,6 +2592,13 @@ PerfectHashServerDispatchBulkCreateDirectoryRequest( // TableCreateFlags.DisableCsvOutputFile = TRUE; + if (Server->Flags.NoFileIo) { + TableCreateFlags.NoFileIo = TRUE; + } + if (!Server->Flags.Verbose) { + TableCreateFlags.Silent = TRUE; + TableCreateFlags.Quiet = FALSE; + } Request = (PPERFECT_HASH_SERVER_BULK_REQUEST)( Allocator->Vtbl->Calloc(Allocator, 1, sizeof(*Request)) @@ -2797,6 +2947,14 @@ PerfectHashServerDispatchTableCreateRequest( } ParsedArgs = TRUE; + if (Server->Flags.NoFileIo) { + TableCreateFlags.NoFileIo = TRUE; + } + if (!Server->Flags.Verbose) { + TableCreateFlags.Silent = TRUE; + TableCreateFlags.Quiet = FALSE; + } + Result = PerfectHashContextIocpCreateTableContext(ContextIocp, &Context); if (FAILED(Result)) { diff --git a/src/PerfectHash/PerfectHashServer.h b/src/PerfectHash/PerfectHashServer.h index c231f090..38f314fc 100644 --- a/src/PerfectHash/PerfectHashServer.h +++ b/src/PerfectHash/PerfectHashServer.h @@ -43,8 +43,10 @@ typedef union _PERFECT_HASH_SERVER_FLAGS { ULONG LocalOnly:1; ULONG EndpointAllocated:1; + ULONG Verbose:1; + ULONG NoFileIo:1; - ULONG Unused:30; + ULONG Unused:28; }; LONG AsLong; ULONG AsULong; @@ -59,11 +61,13 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_SERVER { COMMON_COMPONENT_HEADER(PERFECT_HASH_SERVER); - ULONG MaximumConcurrency; + ULONG IocpConcurrency; + ULONG MaxWorkerThreads; ULONG NumaNodeCount; - PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; ULONG Padding1; + PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; ULONG Padding2; + ULONG Padding3; // // The underlying IOCP context used by the server. @@ -83,7 +87,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_SERVER { PPERFECT_HASH_SERVER_PIPE Pipes; ULONG PipeCount; - ULONG Padding3; + ULONG Padding4; HANDLE ShutdownEvent; HANDLE StartedEvent; @@ -119,6 +123,10 @@ extern PERFECT_HASH_SERVER_SET_MAXIMUM_CONCURRENCY PerfectHashServerSetMaximumConcurrency; extern PERFECT_HASH_SERVER_GET_MAXIMUM_CONCURRENCY PerfectHashServerGetMaximumConcurrency; +extern PERFECT_HASH_SERVER_SET_MAXIMUM_THREADS + PerfectHashServerSetMaximumThreads; +extern PERFECT_HASH_SERVER_GET_MAXIMUM_THREADS + PerfectHashServerGetMaximumThreads; extern PERFECT_HASH_SERVER_SET_NUMA_NODE_MASK PerfectHashServerSetNumaNodeMask; extern PERFECT_HASH_SERVER_GET_NUMA_NODE_MASK @@ -127,6 +135,10 @@ extern PERFECT_HASH_SERVER_SET_ENDPOINT PerfectHashServerSetEndpoint; extern PERFECT_HASH_SERVER_GET_ENDPOINT PerfectHashServerGetEndpoint; extern PERFECT_HASH_SERVER_SET_LOCAL_ONLY PerfectHashServerSetLocalOnly; extern PERFECT_HASH_SERVER_GET_LOCAL_ONLY PerfectHashServerGetLocalOnly; +extern PERFECT_HASH_SERVER_SET_VERBOSE PerfectHashServerSetVerbose; +extern PERFECT_HASH_SERVER_GET_VERBOSE PerfectHashServerGetVerbose; +extern PERFECT_HASH_SERVER_SET_NO_FILE_IO PerfectHashServerSetNoFileIo; +extern PERFECT_HASH_SERVER_GET_NO_FILE_IO PerfectHashServerGetNoFileIo; extern PERFECT_HASH_SERVER_START PerfectHashServerStart; extern PERFECT_HASH_SERVER_STOP PerfectHashServerStop; extern PERFECT_HASH_SERVER_WAIT PerfectHashServerWait; diff --git a/src/PerfectHash/PerfectHashTableCreate.c b/src/PerfectHash/PerfectHashTableCreate.c index 81d2f3e7..b598dee6 100644 --- a/src/PerfectHash/PerfectHashTableCreate.c +++ b/src/PerfectHash/PerfectHashTableCreate.c @@ -611,6 +611,9 @@ Return Value: BOOLEAN SawResizeLimit = FALSE; BOOLEAN SawInitialResizes = FALSE; BOOLEAN SawResizeThreshold = FALSE; + BOOLEAN SawInitialPerFileConcurrency = FALSE; + BOOLEAN SawMaxPerFileConcurrency = FALSE; + BOOLEAN SawIncreaseConcurrencyAfter = FALSE; PERFECT_HASH_TABLE_BEST_COVERAGE_TYPE_ID CoverageType; PPERFECT_HASH_CONTEXT Context; PPERFECT_HASH_TABLE_CREATE_PARAMETER Param; @@ -833,6 +836,21 @@ Return Value: ); break; + case TableCreateParameterInitialPerFileConcurrencyId: + Context->InitialPerFileConcurrency = Param->AsULong; + SawInitialPerFileConcurrency = TRUE; + break; + + case TableCreateParameterMaxPerFileConcurrencyId: + Context->MaxPerFileConcurrency = Param->AsULong; + SawMaxPerFileConcurrency = TRUE; + break; + + case TableCreateParameterIncreaseConcurrencyAfterMillisecondsId: + Context->IncreaseConcurrencyAfterMilliseconds = Param->AsULong; + SawIncreaseConcurrencyAfter = TRUE; + break; + case TableCreateParameterBestCoverageTargetValueId: // @@ -871,6 +889,36 @@ Return Value: DEFAULT_MIN_NUMBER_OF_KEYS_FOR_FIND_BEST_GRAPH; } + if (!SawInitialPerFileConcurrency) { + Context->InitialPerFileConcurrency = 1; + } + + if (!SawMaxPerFileConcurrency) { + if (Context->MaximumConcurrency > 0) { + Context->MaxPerFileConcurrency = Context->MaximumConcurrency; + } else { + Context->MaxPerFileConcurrency = 1; + } + } + + if (!SawIncreaseConcurrencyAfter) { + Context->IncreaseConcurrencyAfterMilliseconds = 500; + } + + if (Context->InitialPerFileConcurrency == 0 || + Context->MaxPerFileConcurrency == 0 || + Context->InitialPerFileConcurrency > Context->MaxPerFileConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + goto Error; + } + + if (!SkipThreadpoolInitialization(Context) && + Context->MaximumConcurrency > 0 && + Context->MaxPerFileConcurrency > Context->MaximumConcurrency) { + Result = PH_E_INVALID_MAXIMUM_CONCURRENCY; + goto Error; + } + // // If the Silent flag has been supplied, also set Quiet. // diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.c b/src/PerfectHashServerExe/PerfectHashServerExe.c index d772e48c..69e7edf8 100644 --- a/src/PerfectHashServerExe/PerfectHashServerExe.c +++ b/src/PerfectHashServerExe/PerfectHashServerExe.c @@ -45,13 +45,19 @@ typedef PVOID PMINIDUMP_CALLBACK_INFORMATION; typedef struct _PERFECT_HASH_SERVER_CLI_OPTIONS { UNICODE_STRING Endpoint; PERFECT_HASH_NUMA_NODE_MASK NumaNodeMask; - ULONG MaximumConcurrency; + ULONG IocpConcurrency; + ULONG MaxThreads; BOOLEAN EndpointPresent; - BOOLEAN MaximumConcurrencyPresent; + BOOLEAN IocpConcurrencyPresent; + BOOLEAN MaxThreadsPresent; BOOLEAN NumaNodeMaskPresent; BOOLEAN LocalOnly; BOOLEAN LocalOnlyPresent; - UCHAR Padding1[7]; + BOOLEAN Verbose; + BOOLEAN VerbosePresent; + BOOLEAN NoFileIo; + BOOLEAN NoFileIoPresent; + UCHAR Padding1[6]; } PERFECT_HASH_SERVER_CLI_OPTIONS; typedef PERFECT_HASH_SERVER_CLI_OPTIONS *PPERFECT_HASH_SERVER_CLI_OPTIONS; @@ -598,14 +604,19 @@ PrintUsage( VOID ) { - wprintf(L"Usage: PerfectHashServer [--MaxConcurrency=] " - L"[--Numa=All|] [--Endpoint=] " - L"[--AllowRemote|--LocalOnly]\n"); - wprintf(L" --MaxConcurrency= Total worker threads (default: all)\n"); + wprintf(L"Usage: PerfectHashServer [--IocpConcurrency=] " + L"[--MaxThreads=] [--Numa=All|] [--Endpoint=] " + L"[--AllowRemote|--LocalOnly] [--Verbose] [--NoFileIo]\n"); + wprintf(L" --IocpConcurrency= IOCP concurrency per NUMA node\n"); + wprintf(L" --MaxConcurrency= Alias for --IocpConcurrency\n"); + wprintf(L" --MaxThreads= Worker threads per NUMA node\n"); + wprintf(L" Default: IocpConcurrency * 2 when set\n"); wprintf(L" --Numa=All|0,1|0-3 NUMA node selection mask\n"); wprintf(L" --Endpoint= Named pipe endpoint\n"); wprintf(L" --AllowRemote Allow remote named pipe clients\n"); wprintf(L" --LocalOnly Reject remote named pipe clients\n"); + wprintf(L" --Verbose Enable per-request console output\n"); + wprintf(L" --NoFileIo Disable file I/O for requests\n"); } static @@ -705,16 +716,22 @@ ParseServerArgs( { ULONG Index; - Options->MaximumConcurrency = 0; + Options->IocpConcurrency = 0; + Options->MaxThreads = 0; Options->NumaNodeMask = PERFECT_HASH_NUMA_NODE_MASK_ALL; Options->Endpoint.Buffer = NULL; Options->Endpoint.Length = 0; Options->Endpoint.MaximumLength = 0; Options->EndpointPresent = FALSE; - Options->MaximumConcurrencyPresent = FALSE; + Options->IocpConcurrencyPresent = FALSE; + Options->MaxThreadsPresent = FALSE; Options->NumaNodeMaskPresent = FALSE; Options->LocalOnly = TRUE; Options->LocalOnlyPresent = FALSE; + Options->Verbose = FALSE; + Options->VerbosePresent = FALSE; + Options->NoFileIo = FALSE; + Options->NoFileIoPresent = FALSE; for (Index = 1; Index < NumberOfArguments; Index++) { PCWSTR Arg = ArgvW[Index]; @@ -735,14 +752,36 @@ ParseServerArgs( return S_FALSE; } + if (_wcsnicmp(Arg, L"IocpConcurrency=", 16) == 0) { + HRESULT Result; + Arg += 16; + Result = ParseUnsignedInteger(Arg, &Options->IocpConcurrency); + if (FAILED(Result)) { + return Result; + } + Options->IocpConcurrencyPresent = TRUE; + continue; + } + if (_wcsnicmp(Arg, L"MaxConcurrency=", 15) == 0) { HRESULT Result; Arg += 15; - Result = ParseUnsignedInteger(Arg, &Options->MaximumConcurrency); + Result = ParseUnsignedInteger(Arg, &Options->IocpConcurrency); if (FAILED(Result)) { return Result; } - Options->MaximumConcurrencyPresent = TRUE; + Options->IocpConcurrencyPresent = TRUE; + continue; + } + + if (_wcsnicmp(Arg, L"MaxThreads=", 11) == 0) { + HRESULT Result; + Arg += 11; + Result = ParseUnsignedInteger(Arg, &Options->MaxThreads); + if (FAILED(Result)) { + return Result; + } + Options->MaxThreadsPresent = TRUE; continue; } @@ -783,6 +822,18 @@ ParseServerArgs( continue; } + if (_wcsicmp(Arg, L"Verbose") == 0) { + Options->Verbose = TRUE; + Options->VerbosePresent = TRUE; + continue; + } + + if (_wcsicmp(Arg, L"NoFileIo") == 0) { + Options->NoFileIo = TRUE; + Options->NoFileIoPresent = TRUE; + continue; + } + return PH_E_INVALID_COMMANDLINE_ARG; } @@ -876,10 +927,33 @@ mainCRTStartup( goto Error; } - if (Options.MaximumConcurrencyPresent) { + if (Options.IocpConcurrencyPresent) { Result = Server->Vtbl->SetMaximumConcurrency( Server, - Options.MaximumConcurrency + Options.IocpConcurrency + ); + if (FAILED(Result)) { + goto Error; + } + + if (!Options.MaxThreadsPresent) { + ULONGLONG DefaultThreads; + + DefaultThreads = (ULONGLONG)Options.IocpConcurrency * 2; + if (DefaultThreads == 0 || DefaultThreads > ULONG_MAX) { + Result = E_INVALIDARG; + goto Error; + } + + Options.MaxThreads = (ULONG)DefaultThreads; + Options.MaxThreadsPresent = TRUE; + } + } + + if (Options.MaxThreadsPresent) { + Result = Server->Vtbl->SetMaximumThreads( + Server, + Options.MaxThreads ); if (FAILED(Result)) { goto Error; @@ -908,6 +982,20 @@ mainCRTStartup( } } + if (Options.VerbosePresent) { + Result = Server->Vtbl->SetVerbose(Server, Options.Verbose); + if (FAILED(Result)) { + goto Error; + } + } + + if (Options.NoFileIoPresent) { + Result = Server->Vtbl->SetNoFileIo(Server, Options.NoFileIo); + if (FAILED(Result)) { + goto Error; + } + } + GlobalServer = Server; SetConsoleCtrlHandler(PerfectHashServerConsoleCtrlHandler, TRUE); From edd15c3ce917c594a1e9034623e122e06413f9eb Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Thu, 22 Jan 2026 19:40:45 -0800 Subject: [PATCH 5/9] Update logs and ADS.md. --- ADS.md | 27 +++++++++++++++++++++++++++ IOCP-LOGS.md | 6 ++++++ 2 files changed, 33 insertions(+) create mode 100644 ADS.md diff --git a/ADS.md b/ADS.md new file mode 100644 index 00000000..0f841318 --- /dev/null +++ b/ADS.md @@ -0,0 +1,27 @@ +# ADS Notes + +Context +- Table-size ADS (keys:Algorithm_Hash_Mask.TableSize) is only used to seed + RequestedNumberOfTableElements for the current run. If missing, creation + still works; the requested size stays 0 and the solver proceeds normally. +- Table-info ADS (table.pht1:Info) is required for Table->Vtbl->Load() and + for downstream generators/tests that read Table->TableInfoOnDisk. + +Key locations +- Table-size load/save: `src/PerfectHash/PerfectHashKeysLoadTableSize.c` +- Table-info stream save: `src/PerfectHash/Chm01FileWorkTableInfoStream.c` +- Table-info load: `src/PerfectHash/PerfectHashTableLoad.c` + +Implications +- Dropping table-size ADS is low risk if we accept losing the "reuse previous + table size" hint. It is already optional from a correctness standpoint. +- Dropping table-info ADS breaks load-from-disk and any tools that read + TableInfoOnDisk (CSV outputs, generators, tests). A replacement on-disk + metadata format would be required first. + +Notes +- ReFS/ADS issues observed: zero-length TableSize streams caused + PH_E_INVALID_END_OF_FILE. Fixed by extending empty existing streams when + NoTruncate is set. +- If we want to avoid ADS entirely, table-size is a good first candidate to + move to a sidecar file (e.g., .TableSize) without affecting core table load. diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index 22922680..834a0237 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -130,3 +130,9 @@ - IOCP sys32-200 subset run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms) completed successfully. - IOCP `hard` run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms) crashed with `0xC0000005` (server exit code -1073741819); no minidump found. - Re-ran `hard` under `cdb` attach (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=4`, `--MaxThreads=8`, per-file ramp 1/4/500ms); client returned `0x2004000F`, shutdown succeeded, and no AV was captured (`build-win\\logs\\cdb-hard-attach-release.log`). +- IOCP full sys32 run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/4/500ms) completed in ~38.1s. +- OG BulkCreate full sys32 run (Release, `Mulshrolate4RX`, `--NoFileIo`, concurrency 32, full paths) completed in ~20.05s; no errors logged (`build-win\\logs\\og-sys32-conc32.log`). +- IOCP full sys32 run (Release, `Mulshrolate4RX`, `--NoFileIo`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/16/500ms) completed in ~38.1s. +- OG BulkCreate full sys32 run with file I/O (Release, `Mulshrolate4RX`, concurrency 32) completed in ~2m30s based on `G:\\Scratch\\sys32.og.fileio` directory timestamps (created 19:27:31, last write 19:30:01). +- IOCP full sys32 run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/4/500ms) completed in ~1m27s (`G:\\Scratch\\iocp-sys32-conc32-fileio.log`). +- IOCP full sys32 run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=60`, `--MaxThreads=240`, per-file ramp 1/4/500ms) completed in ~1m49s (`G:\\Scratch\\iocp-sys32-conc60-1-4-500-fileio.log`). From 2f6f3938b5865f7a4855e73ff1f93ca958922a63 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Fri, 23 Jan 2026 17:28:49 -0800 Subject: [PATCH 6/9] Checkpoint of not-crashing-it-seems Iocp progress. --- IOCP-LOGS.md | 31 + IOCP-TOOD.md | 34 +- include/PerfectHash.h | 10 +- src/PerfectHash/CMakeLists.txt | 2 + src/PerfectHash/Chm01Async.c | 37 +- src/PerfectHash/Chm01FileWork.c | 1148 ++++++++++++++++- src/PerfectHash/Chm01FileWorkCHeaderFile.c | 10 + .../Chm01FileWorkCppSourceUnityFile.c | 10 + .../Chm01FileWorkMakefileMainMkFile.c | 14 +- .../Chm01FileWorkTableInfoStream.c | 9 +- src/PerfectHash/Chm01FileWorkVSSolutionFile.c | 14 +- src/PerfectHash/Chm01Shared.c | 60 + src/PerfectHash/PerfectHash.vcxproj | 2 + src/PerfectHash/PerfectHash.vcxproj.filters | 6 + src/PerfectHash/PerfectHashAsync.c | 6 +- src/PerfectHash/PerfectHashContext.c | 66 +- src/PerfectHash/PerfectHashContext.h | 31 +- src/PerfectHash/PerfectHashContextIocp.c | 59 +- src/PerfectHash/PerfectHashContextIocp.h | 12 + src/PerfectHash/PerfectHashFile.c | 21 +- src/PerfectHash/PerfectHashFile.h | 12 + src/PerfectHash/PerfectHashIocpBufferPool.c | 208 +++ src/PerfectHash/PerfectHashIocpBufferPool.h | 159 +++ src/PerfectHash/PerfectHashServer.c | 2 + tests/CMakeLists.txt | 15 + tests/PerfectHashIocpBufferPoolTests.cpp | 114 ++ 26 files changed, 2044 insertions(+), 48 deletions(-) create mode 100644 src/PerfectHash/PerfectHashIocpBufferPool.c create mode 100644 src/PerfectHash/PerfectHashIocpBufferPool.h create mode 100644 tests/PerfectHashIocpBufferPoolTests.cpp diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index 834a0237..b95ee61c 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -136,3 +136,34 @@ - OG BulkCreate full sys32 run with file I/O (Release, `Mulshrolate4RX`, concurrency 32) completed in ~2m30s based on `G:\\Scratch\\sys32.og.fileio` directory timestamps (created 19:27:31, last write 19:30:01). - IOCP full sys32 run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=32`, `--MaxThreads=64`, per-file ramp 1/4/500ms) completed in ~1m27s (`G:\\Scratch\\iocp-sys32-conc32-fileio.log`). - IOCP full sys32 run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=60`, `--MaxThreads=240`, per-file ramp 1/4/500ms) completed in ~1m49s (`G:\\Scratch\\iocp-sys32-conc60-1-4-500-fileio.log`). +- Added IOCP buffer pool module (`PerfectHashIocpBufferPool.[ch]`) and gtest coverage for create/pop/push and empty-pop behavior. +- Fixed buffer-pool unit test AVs by providing test `RtlFillMemory`/`RtlZeroMemory` helpers (internal `ZeroMemory()` macros call `Rtl->RtlFillMemory`). +- Verified `ctest -C Debug -R PerfectHashIocpBufferPool` passes. +- Added per-node bucketed IOCP file-work buffer pools (bucketed by AlignUpPow2(keys) and indexed by file id); IOCP contexts now capture node pointers from the server for pool selection. +- Switched IOCP file saves to true overlapped writes: issue `WriteFile` with IOCP work item, associate output file handles with per-node ports, and complete saves via IOCP callback (set events, return buffers, record errors). +- IOCP sys32-200 subset run with file I/O (Release, `Mulshrolate4RX`, `--IocpConcurrency=4`, `--MaxThreads=8`) hung; only 7/200 output dirs created in `G:\\Scratch\\iocp-sys32-200-conc4-fileio-new`, client stuck waiting, server already exited. +- Attached cdb to `PerfectHashServer.exe` during sys32-200 file-I/O run; captured access violation (`0xC0000005`) in `PerfectHashFileTruncate()` → `PerfectHashFileCreate()` → `PrepareFileChm01()` → `FileWorkItemCallbackChm01()` on an IOCP worker thread, with other threads in `PrepareCHeaderFileChm01()`/`AppendCharBufferToCharBuffer()`; log at `build-win\\logs\\cdb-iocp-fileio-sys32-200-av.log`. + +## 2026-01-23 +- Ran IOCP bulk-create against `G:\\Scratch\\sys32-200` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8 --NoFileIo`); client exited with `0x2004000F` success after manual run. +- Captured server thread stacks post-run: main thread waiting in `PerfectHashServerWait`, IOCP worker threads blocked in `GetQueuedCompletionStatus`, two worker-factory threads idle; log at `build-win\\logs\\cdb-iocp-200-conc4-stacks2.log`. +- Manual `PerfectHashClient` runs require quoting the entire `--BulkCreateDirectory=...` argument; otherwise client reports an invalid argument. +- Ran IOCP bulk-create against `G:\\Scratch\\sys32-1000` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8 --NoFileIo`) and attached cdb ~1s in with `!runaway;~* kb`. Thread sample shows main thread waiting in `PerfectHashServerWait`, IOCP worker threads in `GetQueuedCompletionStatus`, and four worker-factory threads parked in `ZwWaitForWorkViaWorkerFactory` while work is active; log at `build-win\\logs\\cdb-iocp-1000-conc4-stacks.log`. +- Reviewed async/bulk accounting paths (server + CHM01 async + file work) and identified potential outstanding-count leaks on error/requeue paths; see analysis notes for proposed fixes. +- Fixed IOCP async accounting: requeue failures now complete work (decrement outstanding), and graph-work submit failures roll back counts and signal events when needed. +- Quick test (Release, `Mulshrolate4RX`, `G:\\Scratch\\sys32-200`, `--IocpConcurrency=4 --MaxThreads=8 --NoFileIo`) completed in ~1.01s, client exit `0x2004000F`, server exit 0. + +## 2026-01-24 +- Fixed IOCP buffer sizing to index EOF initializers by `FileId` (not `FileWorkId`) to avoid out-of-bounds reads during save-stage sizing; `SavePythonTestFileChm01` no longer fails with `PH_E_INVARIANT_CHECK_FAILED`. +- Rebuilt Release and reran IOCP file-I/O runs on `perfecthash-keys\\test1` and `perfecthash-keys\\hard` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`); both completed without errors. +- Skipped mapping in `PerfectHashFileExtend()` when `SkipMapping` is set (IOCP mode) to prevent `ERROR_USER_MAPPED_FILE` during close. +- Added IOCP one-off buffer fallback and buffer-size validation/cleanup so save stages can request larger payloads without failing; one-off pools are destroyed after use. +- Rebuilt Release and reran IOCP file-I/O on a 200-file sys32 subset (`D:\\Scratch\\sys32-200` → `D:\\Scratch\\iocp-sys32-200-fileio-4`, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`); run completed cleanly with no file-work errors. +- Timed IOCP file-I/O run on `G:\\Scratch\\sys32-1000` (1000 random sys32 keys) with `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8 --MaximumConcurrency=4`; completed in ~26.11s (`G:\\Scratch\\iocp-sys32-1000-fileio-timed2-20260123-120241`). +- Timed OG bulk-create file-I/O run on the same 1000-file set with `Mulshrolate4RX` and concurrency 4; completed in ~11.30s (`G:\\Scratch\\sys32-1000-og-fileio-timed2-20260123-120330`). +- Reset `File->NumberOfBytesWritten` before IOCP prepare callbacks and added IOCP-only bounds checks in `SaveCHeaderFileChm01`/`SaveCppSourceUnityFileChm01` to avoid stale offsets and AVs. +- `Chm01AcquireFileWorkBuffer()` now falls back to one-off buffers when the per-file pool is empty (instead of returning `E_OUTOFMEMORY`). +- IOCP file-I/O run on `G:\\Scratch\\sys32-repro-small` (Release, `Mulshrolate4RX`, `--IocpConcurrency=1 --MaxThreads=2`) completed successfully. +- IOCP file-I/O run on `D:\\src\\perfecthash-keys\\hard` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`) completed successfully. +- Added fail-fast `PH_RAISE` guards for oversized IOCP write requests and reset `NumberOfBytesWritten` for IOCP save-only file work items. +- Full IOCP sys32 file-I/O run completed (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `G:\\Scratch\\iocp-sys32-full` in ~289.6s; largest output file ~2.5MB, no anomalously large files observed. diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md index 64ae76e4..b5fdb686 100644 --- a/IOCP-TOOD.md +++ b/IOCP-TOOD.md @@ -1,5 +1,36 @@ # IOCP TODO +- Overlapped I/O plan (IOCP only; OG remains memory-mapped): + - Done: added `UseOverlappedIo` flag to `PERFECT_HASH_CONTEXT` (IOCP contexts set it). + - Done: added `PerfectHashFileCreateFlags.SkipMapping` so output files can be created with `FILE_FLAG_OVERLAPPED` and no mapping. + - Implement a per-NUMA buffer pool: + - Buckets keyed by `AlignUpPow2(NumberOfKeys)` (32 buckets max), then indexed by file id. + - Each bucket/file id has an `SLIST_HEADER` of buffers. + - New buffer header struct: `SLIST_ENTRY`, `ULONG Size`, `ULONG Bucket`, `ULONG FileId`, `USHORT NumaNode`, `USHORT Flags`, `PVOID Base`, `ULONG BytesCapacity`, `ULONG BytesWritten`. + - Base address for file work writes must point *after* the header. Account for header overhead in sizing; rely on existing EOF multipliers + trap-on-overrun. If traps occur, bump multipliers. + - Allocate buffers from NUMA node (or `RtlCreateMultipleBuffers()` if it supports NUMA), with guard pages enabled. + - Cap per-bucket buffers to `IocpConcurrency` (or `min(IocpConcurrency, FilesInBucket)`). + - Done (initial): per-file buffer pool backed by `RtlCreateMultipleBuffers()` + guard pages. + - Done (initial): per-NUMA bucketed pool indexing by bucket+file id (on-demand allocation). Still need directory-scan sizing and pre-allocation. + - Pre-size and allocate pools per bulk request: + - During directory scan, capture `NumberOfKeys` per file; compute bucket index and per-file output sizes via existing file work table logic. + - Compute total bytes for all buffers (sum of bucket capacity * per-buffer size) and allocate one large NUMA block; carve into buffer headers + payloads. + - Output file pipeline (IOCP): + - Pop a buffer for each file work save stage. + - Set `File->BaseAddress` to buffer base (after header) to reuse existing `OUTPUT_*` macros. + - Track `File->NumberOfBytesWritten` from `Output` - `BaseAddress`. + - Done: issue `WriteFile` with IOCP work item and handle completion in IOCP callback (no `GetOverlappedResult` waits). + - Done: `UnmapFileChm01()` no-ops in overlapped mode; `CloseFileChm01()` uses `File->NumberOfBytesWritten` (no mapping). + - Input file pipeline (IOCP): + - Replace memory-mapped keys file reads with overlapped `ReadFile` into a keys buffer (or reuse pool per key-size bucket). + - Parse keys from buffer; keep mapped path for OG. +- Add lightweight stats (debug-only): pool exhaustion count, max buffers in use, per-bucket hits; optional ETW later. +- Add unit tests for IOCP file-write completion handling (buffer return, event signal, error path). +- Audit async accounting for leaks: + - Ensure `PerfectHashAsyncIocpCompletionCallback` decrements outstanding if requeue fails. + - Add cleanup in `Chm01AsyncDispatchGraphWork` when `PerfectHashAsyncSubmit` fails (decrement `ActiveGraphs`/`RemainingSolverLoops`, free `GraphWork`). + - Add logging/asserts for `Job->ActiveGraphs`, `Context->RemainingSolverLoops`, and `Job->Async.Outstanding` to catch leaks. + - Get a sys32 `--NoFileIo` run to complete (Release, max concurrency); if it hangs, capture bulk-count logs and identify the stuck state. - Diagnose 1000-file sys32 subset hang with `--IocpConcurrency=32 --MaxThreads=64` (Release, `--NoFileIo`); identify which work items are stuck and why. - Re-run sys32 with file I/O to `G:\\Scratch` once `--NoFileIo` completes; record timing and correctness. @@ -11,7 +42,8 @@ - Exercise IOCP file work dispatch path (bulk create) to confirm outstanding event signaling and non-threadpool file work callbacks. - Identify root cause of remaining IOCP bulk-create failures (`E_UNEXPECTED` on `shell32-13803.keys` / `CoreUIComponents-7995.keys`) if they recur. - Investigate unexpected ~1500 threadpool worker threads (`ZwWaitForWorkViaWorkerFactory`) observed during IOCP server runs; identify source (RPC/TP APIs) and whether it contributes to hangs. -- Validate context file work skip fix for `SetEndOfFile`/`PerfectHashFileTruncate` 1224 failures; decide whether to generate context files once per bulk request. +- Re-run a larger file-I/O set to confirm the `PerfectHashFileExtend` skip-mapping fix and one-off buffer fallback eliminate `SetEndOfFile` 1224 and disk-full pre-size failures; decide whether context-file skip is still required. +- Decide whether to keep the one-off buffer fallback or implement a pool resize/rehash strategy for oversized file-work payloads. - Decide whether `PerfectHashClientExe` should wait on bulk-create tokens for `BulkCreate=` requests now that the server routes them through the bulk-directory path. - Choose a node-selection policy for single table-create server requests (round-robin vs node 0 vs affinity). - Add IOCP work item queueing/state to drive per-request pipelines. diff --git a/include/PerfectHash.h b/include/PerfectHash.h index 8ff7cea3..35417da1 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -1737,11 +1737,19 @@ typedef union _PERFECT_HASH_FILE_CREATE_FLAGS { ULONG EndOfFileIsExtensionSizeIfFileExists:1; + // + // When set, skip creating a memory-mapped view for this file. The + // caller is responsible for providing a suitable buffer and handling + // any I/O. + // + + ULONG SkipMapping:1; + // // Unused bits. // - ULONG Unused:29; + ULONG Unused:28; }; LONG AsLong; diff --git a/src/PerfectHash/CMakeLists.txt b/src/PerfectHash/CMakeLists.txt index ab08ab37..1702a151 100644 --- a/src/PerfectHash/CMakeLists.txt +++ b/src/PerfectHash/CMakeLists.txt @@ -27,6 +27,7 @@ set(Private_Header_Files "PerfectHashConstants.h" "PerfectHashContext.h" "PerfectHashContextIocp.h" + "PerfectHashIocpBufferPool.h" "PerfectHashAsync.h" "PerfectHashCu.h" "PerfectHashClient.h" @@ -209,6 +210,7 @@ set(Source_Files "PerfectHashContext.c" "PerfectHashContextIocp.c" "PerfectHashContextIocpArgs.c" + "PerfectHashIocpBufferPool.c" "PerfectHashContextTableCreate.c" "PerfectHashClient.c" "PerfectHashKeys.c" diff --git a/src/PerfectHash/Chm01Async.c b/src/PerfectHash/Chm01Async.c index 45df72d6..cfd1d1aa 100644 --- a/src/PerfectHash/Chm01Async.c +++ b/src/PerfectHash/Chm01Async.c @@ -544,6 +544,40 @@ Chm01AsyncDispatchGraphWork( Result = PerfectHashAsyncSubmit(&Job->Async, &GraphWork->Work); if (FAILED(Result)) { + if (!GraphWork->Work.Flags.InlineDispatch) { + if (InterlockedDecrement(&Context->RemainingSolverLoops) == 0) { + if (Context->FinishedCount == 0) { + if (!SetEvent(Context->FailedEvent)) { + SYS_ERROR(SetEvent); + } + } else { + if (!SetEvent(Context->SucceededEvent)) { + SYS_ERROR(SetEvent); + } + } + SetStopSolving(Context); + } + + if (InterlockedDecrement(&Job->ActiveGraphs) == 0) { + if (Job->GraphsCompleteEvent) { + if (!SetEvent(Job->GraphsCompleteEvent)) { + SYS_ERROR(SetEvent); + } + } + } + + if (Job->DispatchedGraphs > 0) { + Job->DispatchedGraphs--; + } + + if (Job->Allocator) { + Job->Allocator->Vtbl->FreePointer( + Job->Allocator, + (PVOID *)&GraphWork + ); + } + } + return Result; } } @@ -1796,7 +1830,8 @@ Chm01AsyncPumpIoCompletionPort( const ULONG AllowedFlags = ( PH_IOCP_WORK_FLAG_ASYNC | PH_IOCP_WORK_FLAG_FILE_WORK | - PH_IOCP_WORK_FLAG_PIPE + PH_IOCP_WORK_FLAG_PIPE | + PH_IOCP_WORK_FLAG_FILE_IO ); IoCompletionPort = Job->Async.IoCompletionPort; diff --git a/src/PerfectHash/Chm01FileWork.c b/src/PerfectHash/Chm01FileWork.c index 701a8c78..559374f4 100644 --- a/src/PerfectHash/Chm01FileWork.c +++ b/src/PerfectHash/Chm01FileWork.c @@ -21,6 +21,8 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashContextIocp.h" +#include "PerfectHashIocpBufferPool.h" // // File work callback array. @@ -40,6 +42,855 @@ FILE_WORK_CALLBACK_IMPL *FileCallbacks[] = { NULL }; +static +ULONG +Chm01GetFileWorkBufferBucketIndex( + _In_ PRTL Rtl, + _In_ ULONGLONG NumberOfKeys + ) +{ + ULONGLONG Rounded; + ULONG BucketIndex; + + if (!ARGUMENT_PRESENT(Rtl)) { + return 0; + } + + if (NumberOfKeys == 0) { + return 0; + } + + Rounded = Rtl->RoundUpPowerOfTwo64(NumberOfKeys); + if (Rounded == 0) { + return 0; + } + + BucketIndex = (ULONG)Rtl->TrailingZeros64(Rounded); + if (BucketIndex > PERFECT_HASH_IOCP_BUFFER_MAX_BUCKET_INDEX) { + BucketIndex = PERFECT_HASH_IOCP_BUFFER_MAX_BUCKET_INDEX; + } + + return BucketIndex; +} + +static +ULONG +Chm01GetFileWorkBufferBucketIndexForFile( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONG FileIndex + ) +{ + FILE_ID FileId; + ULONGLONG Count; + PRTL Rtl; + PCEOF_INIT Eof; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + + if (!ARGUMENT_PRESENT(Context)) { + return 0; + } + + Rtl = Context->Rtl; + if (!Rtl) { + return 0; + } + + FileId = (FILE_ID)(FileIndex + 1); + if (!IsValidFileId(FileId)) { + return 0; + } + + Table = Context->Table; + Keys = Table ? Table->Keys : NULL; + if (!Table || !Keys) { + return 0; + } + + Count = Keys->NumberOfKeys.QuadPart; + Eof = &EofInits[FileId]; + if (Eof->Type == EofInitTypeNumberOfTableElementsMultiplier && + Table->TableInfoOnDisk) { + Count = Table->TableInfoOnDisk->NumberOfTableElements.QuadPart; + } + + return Chm01GetFileWorkBufferBucketIndex(Rtl, Count); +} + +static +HRESULT +Chm01GetFileWorkBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONG FileIndex, + _In_ ULONG BucketIndex, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + ULONG Pages; + ULONG BufferCount; + ULONG PoolIndex; + ULONG PoolCount; + ULONGLONG RequiredSize; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_NODE Node; + + if (!ARGUMENT_PRESENT(Context)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + *PoolPointer = NULL; + + if (FileIndex >= NUMBER_OF_FILES) { + return E_INVALIDARG; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + Node = Context->IocpNode; + if (Node) { + if (!Node->FileWorkBufferPools) { + AcquireSRWLockExclusive(&Node->FileWorkBufferPoolLock); + if (!Node->FileWorkBufferPools) { + PoolCount = PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT * + NUMBER_OF_FILES; + if (PoolCount < PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT || + PoolCount < NUMBER_OF_FILES) { + ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); + return E_INVALIDARG; + } + + Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + PoolCount, + sizeof(*Pools) + ); + if (!Pools) { + ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); + return E_OUTOFMEMORY; + } + + Node->FileWorkBufferPools = Pools; + Node->FileWorkBufferPoolBucketCount = + PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT; + Node->FileWorkBufferPoolFileCount = NUMBER_OF_FILES; + Node->FileWorkBufferPoolCount = PoolCount; + Node->FileWorkBufferPoolPageSize = PAGE_SIZE; + } + ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); + } + + if (BucketIndex >= Node->FileWorkBufferPoolBucketCount) { + return E_INVALIDARG; + } + + PoolIndex = (BucketIndex * Node->FileWorkBufferPoolFileCount) + + FileIndex; + if (PoolIndex >= Node->FileWorkBufferPoolCount) { + return E_INVALIDARG; + } + + Pool = &Node->FileWorkBufferPools[PoolIndex]; + } else { + if (!Context->FileWorkBufferPools) { + AcquireSRWLockExclusive(&Context->Lock); + if (!Context->FileWorkBufferPools) { + Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + NUMBER_OF_FILES, + sizeof(*Pools) + ); + if (!Pools) { + ReleaseSRWLockExclusive(&Context->Lock); + return E_OUTOFMEMORY; + } + + Context->FileWorkBufferPools = Pools; + Context->FileWorkBufferPoolCount = NUMBER_OF_FILES; + Context->FileWorkBufferPoolPageSize = PAGE_SIZE; + } + ReleaseSRWLockExclusive(&Context->Lock); + } + + Pool = &Context->FileWorkBufferPools[FileIndex]; + } + + if (!Pool->BaseAddress) { + if (Node) { + AcquireSRWLockExclusive(&Node->FileWorkBufferPoolLock); + } else { + AcquireSRWLockExclusive(&Context->Lock); + } + if (!Pool->BaseAddress) { + if (Node && Node->IocpConcurrency != 0) { + BufferCount = Node->IocpConcurrency; + } else { + BufferCount = Context->MaximumConcurrency; + } + if (BufferCount == 0) { + BufferCount = 1; + } + + RequiredSize = RequiredPayloadBytes + + PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + Pages = (ULONG)BYTES_TO_PAGES(RequiredSize); + if (Pages == 0) { + Pages = 1; + } + + Result = PerfectHashIocpBufferPoolCreate( + Rtl, + &Context->ProcessHandle, + PAGE_SIZE, + BufferCount, + Pages, + NULL, + NULL, + Pool + ); + + if (FAILED(Result)) { + if (Node) { + ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); + } else { + ReleaseSRWLockExclusive(&Context->Lock); + } + return Result; + } + } + if (Node) { + ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); + } else { + ReleaseSRWLockExclusive(&Context->Lock); + } + } + + *PoolPointer = Pool; + return S_OK; +} + +static +HRESULT +Chm01CreateOneOffFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + HRESULT Result; + ULONG Pages; + ULONGLONG RequiredSize; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER Buffer; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + if (!ARGUMENT_PRESENT(PoolPointer) || !ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; + } + + *PoolPointer = NULL; + *BufferPointer = NULL; + + if (!ARGUMENT_PRESENT(Context)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + RequiredSize = RequiredPayloadBytes + + PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + Pages = (ULONG)BYTES_TO_PAGES(RequiredSize); + if (Pages == 0) { + Pages = 1; + } + + Pool = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*Pool) + ); + if (!Pool) { + return E_OUTOFMEMORY; + } + + Result = PerfectHashIocpBufferPoolCreate(Rtl, + &Context->ProcessHandle, + PAGE_SIZE, + 1, + Pages, + NULL, + NULL, + Pool); + if (FAILED(Result)) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + return Result; + } + + Pool->Flags |= PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_ONE_OFF; + Buffer = PerfectHashIocpBufferPoolPop(Pool); + if (!Buffer) { + PerfectHashIocpBufferPoolDestroy(Rtl, Pool); + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + return E_OUTOFMEMORY; + } + + *PoolPointer = Pool; + *BufferPointer = Buffer; + return S_OK; +} + +static +HRESULT +Chm01AcquireFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONG FileIndex, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + HRESULT Result; + ULONG Retry; + ULONG BucketIndex; + PPERFECT_HASH_IOCP_BUFFER Buffer; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + if (!ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; + } + + *BufferPointer = NULL; + + if (!ARGUMENT_PRESENT(Context) || !Context->Rtl) { + return E_POINTER; + } + + BucketIndex = Chm01GetFileWorkBufferBucketIndexForFile(Context, + FileIndex); + + Result = Chm01GetFileWorkBufferPool(Context, + FileIndex, + BucketIndex, + RequiredPayloadBytes, + &Pool); + if (FAILED(Result)) { + return Result; + } + + if (Pool->PayloadSizeInBytes < RequiredPayloadBytes) { + Result = Chm01CreateOneOffFileWorkBuffer(Context, + RequiredPayloadBytes, + &Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + goto AssignBuffer; + } + + Buffer = NULL; + for (Retry = 0; Retry < 1024; Retry++) { + Buffer = PerfectHashIocpBufferPoolPop(Pool); + if (Buffer) { + break; + } + if ((Retry & 0x3F) == 0) { + Sleep(0); + } else { + YieldProcessor(); + } + } + + if (!Buffer) { + Result = Chm01CreateOneOffFileWorkBuffer(Context, + RequiredPayloadBytes, + &Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + goto AssignBuffer; + } + +AssignBuffer: + Buffer->BucketIndex = BucketIndex; + Buffer->FileId = FileIndex; + if (Context && Context->IocpNode) { + Buffer->NumaNode = (USHORT)Context->IocpNode->NodeId; + } else { + Buffer->NumaNode = 0; + } + + *PoolPointer = Pool; + *BufferPointer = Buffer; + return S_OK; +} + +static +VOID +Chm01ClearFileWorkBuffer( + _In_ PPERFECT_HASH_FILE File + ) +{ + if (!ARGUMENT_PRESENT(File)) { + return; + } + + File->IocpBufferPool = NULL; + File->IocpBuffer = NULL; + File->BaseAddress = NULL; +} + +static +VOID +Chm01ReleaseIocpBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + PALLOCATOR Allocator; + PRTL Rtl; + + if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_ONE_OFF) { + if (!ARGUMENT_PRESENT(Context)) { + return; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return; + } + + PerfectHashIocpBufferPoolDestroy(Rtl, Pool); + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + return; + } + + PerfectHashIocpBufferPoolPush(Pool, Buffer); +} + +static +VOID +Chm01ReleaseFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PPERFECT_HASH_FILE File + ) +{ + if (!ARGUMENT_PRESENT(File)) { + return; + } + + if (File->IocpBufferPool && File->IocpBuffer) { + Chm01ReleaseIocpBuffer(Context, + File->IocpBufferPool, + File->IocpBuffer); + } + + Chm01ClearFileWorkBuffer(File); +} + +static +VOID +Chm01RequeueFileWorkItem( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PFILE_WORK_ITEM Item + ) +{ + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(Item)) { + return; + } + + InsertTailFileWork(Context, &Item->ListEntry); + PerfectHashContextSubmitFileWork(Context); +} + +static +HRESULT +Chm01EnsureFileWorkBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONG FileIndex, + _In_ ULONGLONG RequiredPayloadBytes, + _In_ PPERFECT_HASH_FILE File + ) +{ + HRESULT Result; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(File)) { + return E_POINTER; + } + + if (File->IocpBuffer) { + if (File->IocpBuffer->PayloadSize < RequiredPayloadBytes) { + Chm01ReleaseFileWorkBuffer(Context, File); + } else { + File->BaseAddress = PerfectHashIocpBufferPayload(File->IocpBuffer); + return S_OK; + } + } + + Result = Chm01AcquireFileWorkBuffer(Context, + FileIndex, + RequiredPayloadBytes, + &Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + File->IocpBufferPool = Pool; + File->IocpBuffer = Buffer; + File->BaseAddress = PerfectHashIocpBufferPayload(Buffer); + + return S_OK; +} + +static +ULONGLONG +Chm01AdjustRequiredPayloadBytes( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ FILE_ID FileId, + _In_ ULONGLONG RequiredPayloadBytes + ) +{ + ULONGLONG Adjusted; + ULONGLONG BucketCount; + PRTL Rtl; + PCEOF_INIT Eof; + PPERFECT_HASH_TABLE Table; + PPERFECT_HASH_KEYS Keys; + + if (!ARGUMENT_PRESENT(Context)) { + return RequiredPayloadBytes; + } + + Rtl = Context->Rtl; + Table = Context->Table; + Keys = Table ? Table->Keys : NULL; + + if (!Rtl || !Table || !Keys) { + return RequiredPayloadBytes; + } + + if (!IsValidFileId(FileId)) { + return RequiredPayloadBytes; + } + + Eof = &EofInits[FileId]; + switch (Eof->Type) { + case EofInitTypeNumberOfKeysMultiplier: + BucketCount = Rtl->RoundUpPowerOfTwo64(Keys->NumberOfKeys.QuadPart); + if (BucketCount == 0 || Eof->Multiplier == 0) { + return RequiredPayloadBytes; + } + if (BucketCount > ((ULONGLONG)-1) / Eof->Multiplier) { + return RequiredPayloadBytes; + } + Adjusted = BucketCount * Eof->Multiplier; + break; + + case EofInitTypeNumberOfTableElementsMultiplier: + if (!Table->TableInfoOnDisk) { + return RequiredPayloadBytes; + } + BucketCount = Rtl->RoundUpPowerOfTwo64( + Table->TableInfoOnDisk->NumberOfTableElements.QuadPart + ); + if (BucketCount == 0 || Eof->Multiplier == 0) { + return RequiredPayloadBytes; + } + if (BucketCount > ((ULONGLONG)-1) / Eof->Multiplier) { + return RequiredPayloadBytes; + } + Adjusted = BucketCount * Eof->Multiplier; + break; + + case EofInitTypeNull: + case EofInitTypeDefault: + case EofInitTypeAssignedSize: + case EofInitTypeFixed: + case EofInitTypeNumberOfPages: + case EofInitTypeInvalid: + default: + return RequiredPayloadBytes; + } + + if (Adjusted < Context->SystemAllocationGranularity) { + Adjusted = Context->SystemAllocationGranularity; + } + + if (Adjusted < RequiredPayloadBytes) { + Adjusted = RequiredPayloadBytes; + } + + return Adjusted; +} + +typedef struct _CHM01_FILE_IOCP_WRITE { + PERFECT_HASH_IOCP_WORK Iocp; + PPERFECT_HASH_CONTEXT Context; + PFILE_WORK_ITEM Item; + PPERFECT_HASH_FILE File; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + ULONG BytesToWrite; + ULONG Padding1; +} CHM01_FILE_IOCP_WRITE; +typedef CHM01_FILE_IOCP_WRITE *PCHM01_FILE_IOCP_WRITE; + +static +HRESULT +Chm01FileWriteIocpCompletionCallback( + _In_ PPERFECT_HASH_CONTEXT_IOCP ContextIocp, + _In_ ULONG_PTR CompletionKey, + _In_ LPOVERLAPPED Overlapped, + _In_ DWORD NumberOfBytesTransferred, + _In_ BOOL Success + ) +{ + ULONG EventIndex; + ULONG LastError; + HANDLE Event; + HRESULT Result; + PFILE_WORK_ITEM Item; + PPERFECT_HASH_CONTEXT Context; + PCHM01_FILE_IOCP_WRITE WorkItem; + + UNREFERENCED_PARAMETER(ContextIocp); + UNREFERENCED_PARAMETER(CompletionKey); + + WorkItem = (PCHM01_FILE_IOCP_WRITE)Overlapped; + if (!WorkItem) { + return E_POINTER; + } + + Context = WorkItem->Context; + Item = WorkItem->Item; + Event = NULL; + Result = S_OK; + LastError = ERROR_SUCCESS; + + if (!Success) { + LastError = GetLastError(); + SYS_ERROR(WriteFile_IocpCompletion); + Result = PH_E_SYSTEM_CALL_FAILED; + } else if (NumberOfBytesTransferred != WorkItem->BytesToWrite) { + LastError = ERROR_WRITE_FAULT; + SetLastError(LastError); + Result = PH_E_SYSTEM_CALL_FAILED; + } + + if (WorkItem->Buffer) { + WorkItem->Buffer->BytesWritten = NumberOfBytesTransferred; + } + + if (WorkItem->Pool && WorkItem->Buffer) { + Chm01ReleaseIocpBuffer(Context, WorkItem->Pool, WorkItem->Buffer); + } + + if (WorkItem->File) { + Chm01ClearFileWorkBuffer(WorkItem->File); + } + + if (Item) { + Item->LastResult = Result; + Item->LastError = (LONG)LastError; + if (FAILED(Result)) { + InterlockedIncrement(&Item->NumberOfErrors); + } + } + + if (Context && Item && !IsCloseFileWorkId(Item->FileWorkId)) { + EventIndex = FileWorkIdToEventIndex(Item->FileWorkId); + Event = *(&Context->FirstPreparedEvent + EventIndex); + if (Event) { + SetEvent(Event); + } + } + + if (Context && Context->Allocator) { + Context->Allocator->Vtbl->FreePointer( + Context->Allocator, + (PVOID *)&WorkItem + ); + } + + return S_OK; +} + +static +HRESULT +Chm01WriteFileFromBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ PFILE_WORK_ITEM Item, + _In_ PPERFECT_HASH_FILE File, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + BOOL Success; + DWORD BytesWritten; + DWORD BytesToWrite; + ULONG LastError; + HRESULT Result = S_OK; + LARGE_INTEGER BytesRemaining; + PVOID BaseAddress; + PALLOCATOR Allocator; + PCHM01_FILE_IOCP_WRITE WorkItem; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(File) || + !ARGUMENT_PRESENT(Item) || + !ARGUMENT_PRESENT(Pool) || + !ARGUMENT_PRESENT(Buffer)) { + return E_POINTER; + } + + BytesRemaining.QuadPart = File->NumberOfBytesWritten.QuadPart; + if (BytesRemaining.QuadPart <= 0) { + PerfectHashIocpBufferPoolPush(Pool, Buffer); + return S_OK; + } + +#ifdef PH_WINDOWS + if (BytesRemaining.QuadPart > (LONGLONG)Buffer->PayloadSize || + BytesRemaining.QuadPart >= (LONGLONG)(1ULL << 30)) { + BOOL LogLargeWrite; + WCHAR LogBuffer[512]; + int CharCount; + HANDLE LogHandle; + DWORD BytesWrittenLog; + + LogLargeWrite = (GetEnvironmentVariableW( + L"PH_LOG_LARGE_FILE_WRITE", NULL, 0) != 0); + + if (LogLargeWrite) { + CharCount = _snwprintf_s( + LogBuffer, + ARRAYSIZE(LogBuffer), + _TRUNCATE, + L"PH_LARGE_WRITE: FileId=%lu FileWorkId=%lu " + L"BytesRemaining=%lld PayloadSize=%llu " + L"EndOfFile=%lld Path=%wZ\n", + (ULONG)Item->FileId, + (ULONG)Item->FileWorkId, + BytesRemaining.QuadPart, + Buffer->PayloadSize, + File->FileInfo.EndOfFile.QuadPart, + (File->Path ? &File->Path->FullPath : &NullUnicodeString) + ); + + if (CharCount > 0) { + OutputDebugStringW(LogBuffer); + LogHandle = CreateFileW(L"PerfectHashLargeWrite.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (IsValidHandle(LogHandle)) { + WriteFile(LogHandle, + LogBuffer, + (DWORD)(CharCount * sizeof(WCHAR)), + &BytesWrittenLog, + NULL); + CloseHandle(LogHandle); + } + } + } + + PH_RAISE(PH_E_INVALID_END_OF_FILE); + } +#endif + + if (BytesRemaining.HighPart != 0) { + PH_RAISE(PH_E_INVALID_END_OF_FILE); + } + + BaseAddress = PerfectHashIocpBufferPayload(Buffer); + BytesToWrite = BytesRemaining.LowPart; + BytesWritten = 0; + + Allocator = Context->Allocator; + if (!Allocator) { + Result = E_UNEXPECTED; + goto End; + } + + WorkItem = (PCHM01_FILE_IOCP_WRITE)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*WorkItem) + ); + if (!WorkItem) { + Result = E_OUTOFMEMORY; + goto End; + } + + WorkItem->Iocp.Signature = PH_IOCP_WORK_SIGNATURE; + WorkItem->Iocp.Flags = PH_IOCP_WORK_FLAG_FILE_IO; + WorkItem->Iocp.CompletionCallback = + Chm01FileWriteIocpCompletionCallback; + WorkItem->Context = Context; + WorkItem->Item = Item; + WorkItem->File = File; + WorkItem->Pool = Pool; + WorkItem->Buffer = Buffer; + WorkItem->BytesToWrite = BytesToWrite; + + Success = WriteFile(File->FileHandle, + BaseAddress, + BytesToWrite, + &BytesWritten, + &WorkItem->Iocp.Overlapped); + + if (!Success) { + LastError = GetLastError(); + if (LastError != ERROR_IO_PENDING) { + SetLastError(LastError); + SYS_ERROR(WriteFile); + Result = PH_E_SYSTEM_CALL_FAILED; + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&WorkItem); + goto End; + } + } + + return S_FALSE; + +End: + + Chm01ReleaseIocpBuffer(Context, Pool, Buffer); + Chm01ClearFileWorkBuffer(File); + return Result; +} + // // Forward decls. // @@ -127,6 +978,7 @@ Return Value: PGRAPH_INFO Info; FILE_ID FileId; FILE_WORK_ID FileWorkId; + FILE_WORK_ID PrepareWorkId; CONTEXT_FILE_ID ContextFileId; PFILE_WORK_CALLBACK_IMPL Impl = NULL; PPERFECT_HASH_TABLE Table; @@ -140,6 +992,8 @@ Return Value: PPERFECT_HASH_CONTEXT Context; BOOLEAN IsContextFile = FALSE; BOOLEAN IsMakefileOrMainMkOrCMakeListsTextFile = FALSE; + BOOLEAN HasSaveCallback = FALSE; + BOOLEAN HasPrepareCallback = FALSE; // // Initialize aliases. @@ -603,19 +1457,147 @@ Return Value: PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); } +#ifdef PH_WINDOWS + if (EndOfFile.QuadPart >= (LONGLONG)(1ULL << 30)) { + BOOL LogLargeEof; + BOOL FailLargeEof; + ULONGLONG NumberOfKeys; + ULONGLONG LargeEofTableElements; + ULONGLONG AssignedSizeInBytes; + ULONG AllocationGranularity; + PCUNICODE_STRING Extension; + PCUNICODE_STRING Suffix; + WCHAR LogBuffer[512]; + int CharCount; + HANDLE LogHandle; + DWORD BytesWritten; + + LogLargeEof = (GetEnvironmentVariableW( + L"PH_LOG_LARGE_EOF", NULL, 0) != 0); + FailLargeEof = (GetEnvironmentVariableW( + L"PH_FAIL_LARGE_EOF", NULL, 0) != 0); + + if (LogLargeEof || FailLargeEof) { + NumberOfKeys = Keys ? Keys->NumberOfKeys.QuadPart : 0; + LargeEofTableElements = ( + TableInfo ? + TableInfo->NumberOfTableElements.QuadPart : + 0 + ); + AssignedSizeInBytes = ( + Info ? Info->AssignedSizeInBytes : 0 + ); + AllocationGranularity = ( + Context ? Context->SystemAllocationGranularity : 0 + ); + Extension = GetFileWorkItemExtension(FileWorkId); + Suffix = GetFileWorkItemSuffix(FileWorkId); + + CharCount = _snwprintf_s( + LogBuffer, + ARRAYSIZE(LogBuffer), + _TRUNCATE, + L"PH_LARGE_EOF: FileWorkId=%lu FileId=%lu " + L"EofType=%lu Multiplier=%llu EndOfFile=%lld " + L"Keys=%llu TableElements=%llu AssignedSize=%llu " + L"Granularity=%lu Suffix=%wZ Ext=%wZ Path=%wZ\n", + (ULONG)FileWorkId, + (ULONG)FileId, + (ULONG)Eof->Type, + (ULONGLONG)Eof->Multiplier, + EndOfFile.QuadPart, + NumberOfKeys, + LargeEofTableElements, + AssignedSizeInBytes, + AllocationGranularity, + Suffix ? Suffix : &NullUnicodeString, + Extension ? Extension : &NullUnicodeString, + Path ? &Path->FullPath : &NullUnicodeString + ); + + if (CharCount > 0) { + OutputDebugStringW(LogBuffer); + + LogHandle = CreateFileW(L"PerfectHashLargeEof.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (IsValidHandle(LogHandle)) { + WriteFile(LogHandle, + LogBuffer, + (DWORD)(CharCount * sizeof(WCHAR)), + &BytesWritten, + NULL); + CloseHandle(LogHandle); + } + } + } + + if (FailLargeEof) { + Result = PH_E_INVALID_END_OF_FILE; + PH_ERROR(PrepareFileChm01_LargeEof, Result); + goto End; + } + } +#endif + Result = PrepareFileChm01(Table, Item, Path, &EndOfFile, DependentEvent); + if (Result == S_FALSE) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } + if (FAILED(Result)) { PH_ERROR(PrepareFileChm01, Result); goto End; } + if (UseOverlappedIo(Context) && Impl) { + ULONGLONG BufferPayloadBytes; + + if (!*File) { + Result = E_UNEXPECTED; + goto End; + } + + BufferPayloadBytes = Chm01AdjustRequiredPayloadBytes( + Context, + FileId, + (ULONGLONG)EndOfFile.QuadPart + ); + + Result = Chm01EnsureFileWorkBuffer( + Context, + FileIndex, + BufferPayloadBytes, + *File + ); + if (FAILED(Result)) { + goto End; + } + + // + // Reset bytes written for IOCP buffers. Prepare routines must + // explicitly advance this as they generate output. + // + + (*File)->NumberOfBytesWritten.QuadPart = 0; + } + if (Impl) { Result = Impl(Context, Item); + if (Result == S_FALSE) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } if (FAILED(Result)) { // @@ -627,6 +1609,31 @@ Return Value: } } + if (UseOverlappedIo(Context) && Impl) { + FILE_WORK_ID SaveWorkId; + + SaveWorkId = FileWorkIdToDependentId(FileWorkId); + HasSaveCallback = (FileCallbacks[SaveWorkId] != NULL); + + if (!HasSaveCallback) { + Result = Chm01WriteFileFromBuffer( + Context, + Item, + *File, + (*File)->IocpBufferPool, + (*File)->IocpBuffer + ); + if (Result == S_FALSE) { + Chm01ClearFileWorkBuffer(*File); + return; + } + Chm01ClearFileWorkBuffer(*File); + if (FAILED(Result)) { + goto End; + } + } + } + } else if (IsSaveFileWorkId(FileWorkId)) { ULONG WaitResult; @@ -638,7 +1645,14 @@ Return Value: DependentEventIndex = FileWorkIdToDependentEventIndex(FileWorkId); DependentEvent = *(&Context->FirstPreparedEvent + DependentEventIndex); - WaitResult = WaitForSingleObject(DependentEvent, INFINITE); + WaitResult = WaitForSingleObject( + DependentEvent, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForSingleObject); Result = PH_E_SYSTEM_CALL_FAILED; @@ -659,8 +1673,57 @@ Return Value: goto End; } + PrepareWorkId = FileWorkIdToDependentId(FileWorkId); + HasPrepareCallback = (FileCallbacks[PrepareWorkId] != NULL); + + if (UseOverlappedIo(Context)) { + if (Impl || (*File)->IocpBuffer) { + ULONGLONG BufferPayloadBytes; + + BufferPayloadBytes = Chm01AdjustRequiredPayloadBytes( + Context, + FileId, + (ULONGLONG)(*File)->FileInfo.EndOfFile.QuadPart + ); + + Result = Chm01EnsureFileWorkBuffer( + Context, + FileIndex, + BufferPayloadBytes, + *File + ); + if (FAILED(Result)) { + goto End; + } + } + } + + if (UseOverlappedIo(Context) && Impl && !HasPrepareCallback) { + (*File)->NumberOfBytesWritten.QuadPart = 0; + } + if (Impl) { Result = Impl(Context, Item); + if (Result == S_FALSE) { + Chm01RequeueFileWorkItem(Context, Item); + return; + } + if (FAILED(Result)) { + goto End; + } + } + + if (UseOverlappedIo(Context) && (*File)->IocpBuffer) { + Result = Chm01WriteFileFromBuffer(Context, + Item, + *File, + (*File)->IocpBufferPool, + (*File)->IocpBuffer); + if (Result == S_FALSE) { + Chm01ClearFileWorkBuffer(*File); + return; + } + Chm01ClearFileWorkBuffer(*File); if (FAILED(Result)) { goto End; } @@ -673,15 +1736,17 @@ Return Value: // has to do when the close work items are submitted in parallel. // - Result = UnmapFileChm01(Table, Item); - if (FAILED(Result)) { + if (!UseOverlappedIo(Context)) { + Result = UnmapFileChm01(Table, Item); + if (FAILED(Result)) { - // - // Nothing needs doing here. The Result will bubble back up - // via the normal mechanisms. - // + // + // Nothing needs doing here. The Result will bubble back up + // via the normal mechanisms. + // - NOTHING; + NOTHING; + } } } else { @@ -725,6 +1790,15 @@ Return Value: RELEASE(Path); + if (UseOverlappedIo(Context) && + File && + *File && + (*File)->IocpBuffer) { + if (FAILED(Result) || IsCloseFileWorkId(FileWorkId)) { + Chm01ReleaseFileWorkBuffer(Context, *File); + } + } + // // If the item's UUID string buffer is non-NULL here, the downstream routine // did not successfully take ownership of it, and thus, we're responsbile @@ -821,6 +1895,21 @@ Return Value: HRESULT Result = S_OK; PPERFECT_HASH_FILE File = NULL; PPERFECT_HASH_DIRECTORY Directory = NULL; + PPERFECT_HASH_CONTEXT Context = NULL; + PERFECT_HASH_FILE_CREATE_FLAGS CreateFlags = { 0 }; + PPERFECT_HASH_FILE_CREATE_FLAGS CreateFlagsPointer = NULL; + + // + // Dereference the file pointer provided by the caller. If NULL, this + // is the first preparation request for the given file instance. Otherwise, + // a table resize event has occurred, which means a file rename needs to be + // scheduled (as we include the number of table elements in the file name), + // and the mapping size needs to be extended (as a larger table size means + // larger files are required to capture table data). + // + + File = *Item->FilePointer; + Context = Table->Context; // // If a dependent event has been provided, wait for this object to become @@ -829,8 +1918,14 @@ Return Value: if (IsValidHandle(DependentEvent)) { ULONG WaitResult; + DWORD Timeout; + + Timeout = (Context && UseOverlappedIo(Context)) ? 0 : INFINITE; - WaitResult = WaitForSingleObject(DependentEvent, INFINITE); + WaitResult = WaitForSingleObject(DependentEvent, Timeout); + if (WaitResult == WAIT_TIMEOUT && Timeout == 0) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForSingleObject); Result = PH_E_SYSTEM_CALL_FAILED; @@ -840,14 +1935,6 @@ Return Value: // // Dereference the file pointer provided by the caller. If NULL, this - // is the first preparation request for the given file instance. Otherwise, - // a table resize event has occurred, which means a file rename needs to be - // scheduled (as we include the number of table elements in the file name), - // and the mapping size needs to be extended (as a larger table size means - // larger files are required to capture table data). - // - - File = *Item->FilePointer; if (!File) { @@ -878,11 +1965,16 @@ Return Value: Directory = Table->OutputDirectory; } + if (Context && UseOverlappedIo(Context)) { + CreateFlags.SkipMapping = TRUE; + CreateFlagsPointer = &CreateFlags; + } + Result = File->Vtbl->Create(File, Path, EndOfFile, Directory, - NULL); + CreateFlagsPointer); if (FAILED(Result)) { PH_ERROR(PerfectHashFileCreate, Result); @@ -890,6 +1982,26 @@ Return Value: goto Error; } + if (Context && + UseOverlappedIo(Context) && + Context->FileWorkIoCompletionPort) { + HANDLE PortHandle; + + PortHandle = CreateIoCompletionPort( + File->FileHandle, + Context->FileWorkIoCompletionPort, + (ULONG_PTR)File, + 0 + ); + + if (!PortHandle) { + SYS_ERROR(CreateIoCompletionPort_FileWork); + Result = PH_E_SYSTEM_CALL_FAILED; + RELEASE(File); + goto Error; + } + } + // // Set the file ID and then update the file pointer. // diff --git a/src/PerfectHash/Chm01FileWorkCHeaderFile.c b/src/PerfectHash/Chm01FileWorkCHeaderFile.c index b4a0762e..e6089748 100644 --- a/src/PerfectHash/Chm01FileWorkCHeaderFile.c +++ b/src/PerfectHash/Chm01FileWorkCHeaderFile.c @@ -56,6 +56,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" extern const STRING CompiledPerfectHashTableRoutinesPreCSourceRawCString; extern const STRING CompiledPerfectHashTableRoutinesCSourceRawCString; @@ -335,6 +336,15 @@ SaveCHeaderFileChm01( ); Algo.MaximumLength = Algo.Length; + if (UseOverlappedIo(Context)) { + if (!File->IocpBuffer || + File->NumberOfBytesWritten.QuadPart <= 0 || + (ULONGLONG)File->NumberOfBytesWritten.QuadPart >= + File->IocpBuffer->PayloadSize) { + return PH_E_INVALID_END_OF_FILE; + } + } + // // Pick up the offset from where we left off. // diff --git a/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c b/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c index 6b15b745..7a3234de 100644 --- a/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c +++ b/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c @@ -61,6 +61,7 @@ Module Name: #include "stdafx.h" #include "CompiledPerfectHashTableBenchmarkIndexInline_CSource_RawCString.h" +#include "PerfectHashIocpBufferPool.h" extern const STRING no_sal2CHeaderRawCString; extern const STRING CompiledPerfectHashCHeaderRawCString; @@ -507,6 +508,15 @@ SaveCppSourceUnityFileChm01( ); Algo.MaximumLength = Algo.Length; + if (UseOverlappedIo(Context)) { + if (!File->IocpBuffer || + File->NumberOfBytesWritten.QuadPart <= 0 || + (ULONGLONG)File->NumberOfBytesWritten.QuadPart >= + File->IocpBuffer->PayloadSize) { + return PH_E_INVALID_END_OF_FILE; + } + } + // // Pick up the offset from where we left off. // diff --git a/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c b/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c index 297b4692..b1c56eca 100644 --- a/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c +++ b/src/PerfectHash/Chm01FileWorkMakefileMainMkFile.c @@ -63,10 +63,16 @@ PrepareMakefileMainMkFileChm01( ASSERT(ARRAYSIZE(PrepareEvents) == (SIZE_T)NumberOfEvents); - WaitResult = WaitForMultipleObjects(NumberOfEvents, - PrepareEvents, - WaitForAllEvents, - INFINITE); + WaitResult = WaitForMultipleObjects( + NumberOfEvents, + PrepareEvents, + WaitForAllEvents, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForMultipleObjects); diff --git a/src/PerfectHash/Chm01FileWorkTableInfoStream.c b/src/PerfectHash/Chm01FileWorkTableInfoStream.c index 06448ed7..90d67474 100644 --- a/src/PerfectHash/Chm01FileWorkTableInfoStream.c +++ b/src/PerfectHash/Chm01FileWorkTableInfoStream.c @@ -82,7 +82,14 @@ SaveTableInfoStreamChm01( // to save the timings for verification to the header. // - WaitResult = WaitForSingleObject(Context->VerifiedTableEvent, INFINITE); + WaitResult = WaitForSingleObject( + Context->VerifiedTableEvent, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForSingleObject); Result = PH_E_SYSTEM_CALL_FAILED; diff --git a/src/PerfectHash/Chm01FileWorkVSSolutionFile.c b/src/PerfectHash/Chm01FileWorkVSSolutionFile.c index c484db70..8bee86e9 100644 --- a/src/PerfectHash/Chm01FileWorkVSSolutionFile.c +++ b/src/PerfectHash/Chm01FileWorkVSSolutionFile.c @@ -123,10 +123,16 @@ PrepareVSSolutionFileChm01( ASSERT(ARRAYSIZE(PrepareEvents) == (SIZE_T)NumberOfEvents); - WaitResult = WaitForMultipleObjects(NumberOfEvents, - PrepareEvents, - WaitForAllEvents, - INFINITE); + WaitResult = WaitForMultipleObjects( + NumberOfEvents, + PrepareEvents, + WaitForAllEvents, + UseOverlappedIo(Context) ? 0 : INFINITE + ); + + if (WaitResult == WAIT_TIMEOUT && UseOverlappedIo(Context)) { + return S_FALSE; + } if (WaitResult != WAIT_OBJECT_0) { SYS_ERROR(WaitForMultipleObjects); diff --git a/src/PerfectHash/Chm01Shared.c b/src/PerfectHash/Chm01Shared.c index 488fe238..7ae3ab96 100644 --- a/src/PerfectHash/Chm01Shared.c +++ b/src/PerfectHash/Chm01Shared.c @@ -884,6 +884,66 @@ Return Value: CopyInline(&GraphInfoOnDisk->Dimensions, Dim, sizeof(*Dim)); +#ifdef PH_WINDOWS + if (GetEnvironmentVariableW(L"PH_LOG_GRAPH_INFO", NULL, 0) != 0) { + if (TableInfoOnDisk->NumberOfTableElements.QuadPart >= (1ULL << 20) || + AssignedSizeInBytes >= (1ULL << 30)) { + WCHAR LogBuffer[512]; + int CharCount; + HANDLE LogHandle; + DWORD BytesWritten; + PCUNICODE_STRING KeysPath = NULL; + + if (Table->Keys && + Table->Keys->File && + Table->Keys->File->Path && + Table->Keys->File->Path->FullPath.Length > 0) { + KeysPath = &Table->Keys->File->Path->FullPath; + } + + CharCount = _snwprintf_s( + LogBuffer, + ARRAYSIZE(LogBuffer), + _TRUNCATE, + L"PH_GRAPH_INFO: Keys=%llu TableElems=%llu " + L"AssignedBytes=%llu Vertices=%llu Edges=%llu " + L"HashSize=%lu IndexSize=%lu Requested=%llu " + L"HashFunc=%lu MaskFunc=%lu Path=%wZ\n", + (Table->Keys ? Table->Keys->NumberOfKeys.QuadPart : 0), + TableInfoOnDisk->NumberOfTableElements.QuadPart, + AssignedSizeInBytes, + NumberOfVertices.QuadPart, + NumberOfEdges.QuadPart, + (ULONG)Table->HashSize, + (ULONG)Table->IndexSize, + Table->RequestedNumberOfTableElements.QuadPart, + (ULONG)Table->HashFunctionId, + (ULONG)Table->MaskFunctionId, + KeysPath ? KeysPath : &NullUnicodeString + ); + + if (CharCount > 0) { + OutputDebugStringW(LogBuffer); + LogHandle = CreateFileW(L"PerfectHashGraphInfo.log", + FILE_APPEND_DATA, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (IsValidHandle(LogHandle)) { + WriteFile(LogHandle, + LogBuffer, + (DWORD)(CharCount * sizeof(WCHAR)), + &BytesWritten, + NULL); + CloseHandle(LogHandle); + } + } + } + } +#endif + // // Capture ratios. // diff --git a/src/PerfectHash/PerfectHash.vcxproj b/src/PerfectHash/PerfectHash.vcxproj index 9279a6cb..df270e9c 100644 --- a/src/PerfectHash/PerfectHash.vcxproj +++ b/src/PerfectHash/PerfectHash.vcxproj @@ -150,6 +150,7 @@ + @@ -255,6 +256,7 @@ + diff --git a/src/PerfectHash/PerfectHash.vcxproj.filters b/src/PerfectHash/PerfectHash.vcxproj.filters index e5677f1f..68b914a1 100644 --- a/src/PerfectHash/PerfectHash.vcxproj.filters +++ b/src/PerfectHash/PerfectHash.vcxproj.filters @@ -67,6 +67,9 @@ Source Files + + Source Files + Source Files @@ -414,6 +417,9 @@ Private Header Files + + Private Header Files + Private Header Files diff --git a/src/PerfectHash/PerfectHashAsync.c b/src/PerfectHash/PerfectHashAsync.c index 5db72d42..2a1d4978 100644 --- a/src/PerfectHash/PerfectHashAsync.c +++ b/src/PerfectHash/PerfectHashAsync.c @@ -95,7 +95,11 @@ PerfectHashAsyncIocpCompletionCallback( Result = Work->Step(Work); if (Result == S_FALSE) { - return PerfectHashAsyncRequeueWork(Work); + Result = PerfectHashAsyncRequeueWork(Work); + if (FAILED(Result)) { + return PerfectHashAsyncCompleteWork(Work, Result); + } + return S_OK; } return PerfectHashAsyncCompleteWork(Work, Result); diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index 039e95b5..06c7bce9 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -21,6 +21,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" #include "bsthreadpool.h" // @@ -154,12 +155,13 @@ Return Value: --*/ { + HRESULT Result = S_OK; PRTL Rtl; + PALLOCATOR Allocator; PACL Acl = NULL; BYTE Index; BYTE NumberOfEvents; BOOL Success; - HRESULT Result = S_OK; ULONG LastError; PHANDLE Event; PCHAR Buffer; @@ -172,7 +174,6 @@ Return Value: PSTRING ComputerName; DWORD ComputerNameLength; PTP_POOL Threadpool; - PALLOCATOR Allocator; ULARGE_INTEGER AllocSize; ULONG ThreadpoolConcurrency; ULARGE_INTEGER ObjectNameArraySize; @@ -891,12 +892,14 @@ Return Value: --*/ { + HRESULT Result; PRTL Rtl; + PALLOCATOR Allocator; BYTE Index; BYTE NumberOfEvents; - PALLOCATOR Allocator; + ULONG PoolIndex; PHANDLE Event; - HRESULT Result; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; // // Validate arguments. @@ -908,6 +911,7 @@ Return Value: Rtl = Context->Rtl; Allocator = Context->Allocator; + Allocator = Context->Allocator; if (!Rtl || !Allocator) { return; @@ -1061,6 +1065,28 @@ Return Value: #endif + if (Context->FileWorkBufferPools) { + for (PoolIndex = 0; + PoolIndex < Context->FileWorkBufferPoolCount; + PoolIndex++) { + + Pool = &Context->FileWorkBufferPools[PoolIndex]; + if (!Pool->BaseAddress) { + continue; + } + + Result = PerfectHashIocpBufferPoolDestroy(Rtl, Pool); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpBufferPoolDestroy, Result); + } + } + + Allocator->Vtbl->FreePointer(Allocator, + (PVOID *)&Context->FileWorkBufferPools); + Context->FileWorkBufferPoolCount = 0; + Context->FileWorkBufferPoolPageSize = 0; + } + if (Context->ObjectNames) { Allocator->Vtbl->FreePointer(Allocator, &Context->ObjectNames); @@ -1123,13 +1149,18 @@ Return Value: --*/ { + HRESULT Result; PRTL Rtl; + PALLOCATOR Allocator; + ULONG PoolIndex; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; if (!ARGUMENT_PRESENT(Context)) { return E_POINTER; } Rtl = Context->Rtl; + Allocator = Context->Allocator; Context->AlgorithmId = PerfectHashNullAlgorithmId; Context->HashFunctionId = PerfectHashNullHashFunctionId; @@ -1202,6 +1233,33 @@ Return Value: } #endif + if (Context->FileWorkBufferPools) { + for (PoolIndex = 0; + PoolIndex < Context->FileWorkBufferPoolCount; + PoolIndex++) { + + Pool = &Context->FileWorkBufferPools[PoolIndex]; + if (!Pool->BaseAddress) { + continue; + } + + Result = PerfectHashIocpBufferPoolDestroy(Rtl, Pool); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpBufferPoolDestroy, Result); + } + } + + if (Allocator) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Context->FileWorkBufferPools + ); + } + + Context->FileWorkBufferPoolCount = 0; + Context->FileWorkBufferPoolPageSize = 0; + } + Context->KeysSubset = NULL; Context->UserSeeds = NULL; Context->SeedMasks = NULL; diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index 29fc6e79..6b88a78a 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -21,6 +21,11 @@ Module Name: #include "bsthreadpool.h" #endif +typedef struct _PERFECT_HASH_IOCP_BUFFER_POOL PERFECT_HASH_IOCP_BUFFER_POOL; +typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; +struct _PERFECT_HASH_IOCP_NODE; +typedef struct _PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; + // // Algorithms are required to register a callback routine with the perfect hash // table context that matches the following signature. This routine will be @@ -275,11 +280,18 @@ typedef union _PERFECT_HASH_CONTEXT_FLAGS { ULONG SkipThreadpoolInitialization:1; + // + // When set, indicates file I/O should use overlapped buffers instead + // of memory-mapped views. Intended for IOCP-native contexts. + // + + ULONG UseOverlappedIo:1; + // // Unused bits. // - ULONG Unused:31; + ULONG Unused:30; }; LONG AsLong; ULONG AsULong; @@ -290,6 +302,15 @@ typedef PERFECT_HASH_CONTEXT_FLAGS *PPERFECT_HASH_CONTEXT_FLAGS; #define SkipThreadpoolInitialization(Context) \ ((Context)->Flags.SkipThreadpoolInitialization != FALSE) +#define UseOverlappedIo(Context) \ + ((Context)->Flags.UseOverlappedIo != FALSE) + +#define SetContextUseOverlappedIo(Context) \ + ((Context)->Flags.UseOverlappedIo = TRUE) + +#define ClearContextUseOverlappedIo(Context) \ + ((Context)->Flags.UseOverlappedIo = FALSE) + typedef struct _BEST_GRAPH_INFO { // @@ -1117,7 +1138,13 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { HANDLE FileWorkIoCompletionPort; HANDLE FileWorkOutstandingEvent; volatile LONG FileWorkOutstanding; - ULONG Padding7; + ULONG FileWorkOutstandingPadding; + PPERFECT_HASH_IOCP_NODE IocpNode; + ULONG IocpNodeIndex; + ULONG IocpNodePadding; + PPERFECT_HASH_IOCP_BUFFER_POOL FileWorkBufferPools; + ULONG FileWorkBufferPoolCount; + ULONG FileWorkBufferPoolPageSize; volatile LONG GraphRegisterSolvedTsxSuccess; ULONG Padding4; diff --git a/src/PerfectHash/PerfectHashContextIocp.c b/src/PerfectHash/PerfectHashContextIocp.c index c4d1b1ba..ad447527 100644 --- a/src/PerfectHash/PerfectHashContextIocp.c +++ b/src/PerfectHash/PerfectHashContextIocp.c @@ -16,6 +16,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" // // Forward decls for internal helpers. @@ -164,11 +165,16 @@ Return Value: --*/ { ULONG Index; + PRTL Rtl; + PALLOCATOR Allocator; if (!ARGUMENT_PRESENT(ContextIocp)) { return; } + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + if (ContextIocp->State.Running) { PerfectHashContextIocpStop(ContextIocp); } @@ -178,6 +184,42 @@ Return Value: PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; ULONG ThreadIndex; + if (Node->FileWorkBufferPools) { + ULONG PoolIndex; + + for (PoolIndex = 0; + PoolIndex < Node->FileWorkBufferPoolCount; + PoolIndex++) { + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + Pool = &Node->FileWorkBufferPools[PoolIndex]; + if (!Pool->BaseAddress) { + continue; + } + + if (Rtl) { + HRESULT Result; + + Result = PerfectHashIocpBufferPoolDestroy(Rtl, Pool); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpBufferPoolDestroy, Result); + } + } + } + + if (Allocator) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Node->FileWorkBufferPools + ); + } + + Node->FileWorkBufferPoolBucketCount = 0; + Node->FileWorkBufferPoolFileCount = 0; + Node->FileWorkBufferPoolCount = 0; + Node->FileWorkBufferPoolPageSize = 0; + } + if (Node->WorkerThreads) { for (ThreadIndex = 0; ThreadIndex < Node->WorkerThreadCount; @@ -186,9 +228,9 @@ Return Value: CloseHandle(Node->WorkerThreads[ThreadIndex]); } } - if (ContextIocp->Allocator) { - ContextIocp->Allocator->Vtbl->FreePointer( - ContextIocp->Allocator, + if (Allocator) { + Allocator->Vtbl->FreePointer( + Allocator, (PVOID *)&Node->WorkerThreads ); } @@ -198,9 +240,9 @@ Return Value: CloseHandle(Node->IoCompletionPort); } } - if (ContextIocp->Allocator) { - ContextIocp->Allocator->Vtbl->FreePointer( - ContextIocp->Allocator, + if (Allocator) { + Allocator->Vtbl->FreePointer( + Allocator, (PVOID *)&ContextIocp->Nodes ); } @@ -619,6 +661,7 @@ PerfectHashContextIocpEnumerateNumaNodes( Node->NodeId = NodeId; Node->ProcessorCount = ProcessorCount; Node->GroupAffinity = Affinity; + InitializeSRWLock(&Node->FileWorkBufferPoolLock); } // @@ -1093,6 +1136,10 @@ Return Value: &IID_PERFECT_HASH_CONTEXT, ContextPointer); + if (SUCCEEDED(Result) && *ContextPointer) { + SetContextUseOverlappedIo(*ContextPointer); + } + if (LocalTlsActive) { PerfectHashTlsClearContextIfActive(&LocalTlsContext); } else if (ActiveTls) { diff --git a/src/PerfectHash/PerfectHashContextIocp.h b/src/PerfectHash/PerfectHashContextIocp.h index c23d0478..3eebce1a 100644 --- a/src/PerfectHash/PerfectHashContextIocp.h +++ b/src/PerfectHash/PerfectHashContextIocp.h @@ -76,6 +76,7 @@ typedef PERFECT_HASH_IOCP_COMPLETION_CALLBACK #define PH_IOCP_WORK_FLAG_BULK 0x00000002 #define PH_IOCP_WORK_FLAG_ASYNC 0x00000004 #define PH_IOCP_WORK_FLAG_FILE_WORK 0x00000008 +#define PH_IOCP_WORK_FLAG_FILE_IO 0x00000010 typedef struct _PERFECT_HASH_IOCP_WORK { OVERLAPPED Overlapped; @@ -103,6 +104,17 @@ typedef struct _PERFECT_HASH_IOCP_NODE { HANDLE *WorkerThreads; ULONG WorkerThreadCount; ULONG Padding2; + + // + // IOCP file work buffer pools (per-node). + // + + SRWLOCK FileWorkBufferPoolLock; + PPERFECT_HASH_IOCP_BUFFER_POOL FileWorkBufferPools; + ULONG FileWorkBufferPoolBucketCount; + ULONG FileWorkBufferPoolFileCount; + ULONG FileWorkBufferPoolCount; + ULONG FileWorkBufferPoolPageSize; } PERFECT_HASH_IOCP_NODE; typedef PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; diff --git a/src/PerfectHash/PerfectHashFile.c b/src/PerfectHash/PerfectHashFile.c index 9409fde5..fd72d1b0 100644 --- a/src/PerfectHash/PerfectHashFile.c +++ b/src/PerfectHash/PerfectHashFile.c @@ -629,6 +629,7 @@ Return Value: } File->State.IsReadOnly = FALSE; + File->FileCreateFlags.AsULong = FileCreateFlags.AsULong; // // Add a reference to the source path. @@ -772,11 +773,12 @@ Return Value: // Map the file into memory. // - Result = File->Vtbl->Map(File); - - if (FAILED(Result)) { - PH_ERROR(PerfectHashFileMap, Result); - goto Error; + if (!FileCreateFlags.SkipMapping) { + Result = File->Vtbl->Map(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileMap, Result); + goto Error; + } } // @@ -1331,6 +1333,11 @@ Return Value: return PH_E_FILE_NOT_OPEN; } + if (File->FileCreateFlags.SkipMapping) { + SetFileUnmapped(File); + return S_OK; + } + if (IsFileUnmapped(File)) { return PH_E_FILE_ALREADY_UNMAPPED; } @@ -1804,6 +1811,10 @@ Return Value: File->NumberOfBytesWritten.QuadPart = 0; + if (File->FileCreateFlags.SkipMapping) { + goto End; + } + Result = File->Vtbl->Map(File); if (FAILED(Result)) { PH_ERROR(PerfectHashFileExtend, Result); diff --git a/src/PerfectHash/PerfectHashFile.h b/src/PerfectHash/PerfectHashFile.h index cdc3e9f7..7ab8120f 100644 --- a/src/PerfectHash/PerfectHashFile.h +++ b/src/PerfectHash/PerfectHashFile.h @@ -21,6 +21,11 @@ Module Name: #include "stdafx.h" +typedef struct _PERFECT_HASH_IOCP_BUFFER_POOL PERFECT_HASH_IOCP_BUFFER_POOL; +typedef struct _PERFECT_HASH_IOCP_BUFFER PERFECT_HASH_IOCP_BUFFER; +typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; +typedef PERFECT_HASH_IOCP_BUFFER *PPERFECT_HASH_IOCP_BUFFER; + // // Private vtbl methods. // @@ -317,6 +322,13 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_FILE { PVOID MappedAddress; + // + // Optional IOCP buffer state used when SkipMapping is active. + // + + PPERFECT_HASH_IOCP_BUFFER_POOL IocpBufferPool; + PPERFECT_HASH_IOCP_BUFFER IocpBuffer; + // // File info for the current file. Updated whenever any routine mutates // end of file or allocation size. diff --git a/src/PerfectHash/PerfectHashIocpBufferPool.c b/src/PerfectHash/PerfectHashIocpBufferPool.c new file mode 100644 index 00000000..7ff57d6a --- /dev/null +++ b/src/PerfectHash/PerfectHashIocpBufferPool.c @@ -0,0 +1,208 @@ +/*++ + +Copyright (c) 2024-2026 Trent Nelson + +Module Name: + + PerfectHashIocpBufferPool.c + +Abstract: + + This module implements IOCP buffer pool routines used by the IOCP backend + for overlapped file I/O. + +--*/ + +#include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" + +PERFECT_HASH_IOCP_BUFFER_POOL_CREATE PerfectHashIocpBufferPoolCreate; +PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY PerfectHashIocpBufferPoolDestroy; +PERFECT_HASH_IOCP_BUFFER_POOL_POP PerfectHashIocpBufferPoolPop; +PERFECT_HASH_IOCP_BUFFER_POOL_PUSH PerfectHashIocpBufferPoolPush; + +_Use_decl_annotations_ +HRESULT +PerfectHashIocpBufferPoolCreate( + PRTL Rtl, + PHANDLE ProcessHandle, + ULONG PageSize, + ULONG NumberOfBuffers, + ULONG NumberOfPagesPerBuffer, + PULONG AdditionalProtectionFlags, + PULONG AdditionalAllocationTypeFlags, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ) +{ + ULONG Index; + ULONG PayloadOffset; + PBYTE Base; + HRESULT Result; + ULONGLONG UsableBufferSizeInBytes; + ULONGLONG TotalBufferSizeInBytes; + ULONGLONG BufferStrideInBytes; + ULONGLONG PayloadSizeInBytes; + PPERFECT_HASH_IOCP_BUFFER Buffer; + PRTL_VTBL Vtbl; + HANDLE TargetProcessHandle; + PHANDLE TargetProcessHandlePointer; + + if (!ARGUMENT_PRESENT(Rtl)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Pool)) { + return E_POINTER; + } + + if (!NumberOfBuffers || !NumberOfPagesPerBuffer) { + return E_INVALIDARG; + } + + if (!PageSize) { + return E_INVALIDARG; + } + + Vtbl = Rtl->Vtbl; + + ZeroMemory(Pool, sizeof(*Pool)); + InitializeSListHead(&Pool->ListHead); + + TargetProcessHandle = NULL; + if (ARGUMENT_PRESENT(ProcessHandle)) { + TargetProcessHandlePointer = ProcessHandle; + } else { + TargetProcessHandlePointer = &TargetProcessHandle; + } + + Result = Vtbl->CreateMultipleBuffers(Rtl, + TargetProcessHandlePointer, + PageSize, + NumberOfBuffers, + NumberOfPagesPerBuffer, + AdditionalProtectionFlags, + AdditionalAllocationTypeFlags, + &UsableBufferSizeInBytes, + &TotalBufferSizeInBytes, + &Pool->BaseAddress); + + if (FAILED(Result)) { + return Result; + } + + Pool->ProcessHandle = *TargetProcessHandlePointer; + PayloadOffset = PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + + if ((ULONGLONG)PayloadOffset >= UsableBufferSizeInBytes) { + Vtbl->DestroyBuffer(Rtl, + TargetProcessHandle, + &Pool->BaseAddress, + TotalBufferSizeInBytes); + return E_FAIL; + } + + PayloadSizeInBytes = UsableBufferSizeInBytes - (ULONGLONG)PayloadOffset; + BufferStrideInBytes = UsableBufferSizeInBytes + (ULONGLONG)PageSize; + + Pool->SizeOfStruct = sizeof(*Pool); + Pool->PageSize = PageSize; + Pool->NumberOfBuffers = NumberOfBuffers; + Pool->NumberOfPagesPerBuffer = NumberOfPagesPerBuffer; + Pool->PayloadOffset = PayloadOffset; + Pool->UsableBufferSizeInBytes = UsableBufferSizeInBytes; + Pool->PayloadSizeInBytes = PayloadSizeInBytes; + Pool->BufferStrideInBytes = BufferStrideInBytes; + Pool->TotalAllocationSizeInBytes = TotalBufferSizeInBytes; + Pool->ProcessHandle = TargetProcessHandle; + + Base = (PBYTE)Pool->BaseAddress; + for (Index = 0; Index < NumberOfBuffers; Index++) { + Buffer = (PPERFECT_HASH_IOCP_BUFFER)Base; + Buffer->SizeOfStruct = sizeof(*Buffer); + Buffer->PayloadOffset = PayloadOffset; + Buffer->PayloadSize = PayloadSizeInBytes; + Buffer->BytesWritten = 0; + Buffer->Flags = 0; + Buffer->BucketIndex = 0; + Buffer->FileId = 0; + Buffer->NumaNode = 0; + InterlockedPushEntrySList(&Pool->ListHead, &Buffer->ListEntry); + Base += BufferStrideInBytes; + } + + return S_OK; +} + +_Use_decl_annotations_ +HRESULT +PerfectHashIocpBufferPoolDestroy( + PRTL Rtl, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ) +{ + HRESULT Result; + + if (!ARGUMENT_PRESENT(Rtl)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(Pool)) { + return E_POINTER; + } + + if (!Pool->BaseAddress) { + return S_OK; + } + + Result = Rtl->Vtbl->DestroyBuffer(Rtl, + Pool->ProcessHandle, + &Pool->BaseAddress, + Pool->TotalAllocationSizeInBytes); + if (FAILED(Result)) { + return Result; + } + + ZeroMemory(Pool, sizeof(*Pool)); + return S_OK; +} + +_Use_decl_annotations_ +PPERFECT_HASH_IOCP_BUFFER +PerfectHashIocpBufferPoolPop( + PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ) +{ + PSLIST_ENTRY Entry; + + if (!ARGUMENT_PRESENT(Pool)) { + return NULL; + } + + Entry = InterlockedPopEntrySList(&Pool->ListHead); + if (!Entry) { + return NULL; + } + + return CONTAINING_RECORD(Entry, PERFECT_HASH_IOCP_BUFFER, ListEntry); +} + +_Use_decl_annotations_ +VOID +PerfectHashIocpBufferPoolPush( + PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + if (!ARGUMENT_PRESENT(Pool)) { + return; + } + + if (!ARGUMENT_PRESENT(Buffer)) { + return; + } + + Buffer->BytesWritten = 0; + InterlockedPushEntrySList(&Pool->ListHead, &Buffer->ListEntry); +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashIocpBufferPool.h b/src/PerfectHash/PerfectHashIocpBufferPool.h new file mode 100644 index 00000000..fb8ceda4 --- /dev/null +++ b/src/PerfectHash/PerfectHashIocpBufferPool.h @@ -0,0 +1,159 @@ +/*++ + +Copyright (c) 2024-2026 Trent Nelson + +Module Name: + + PerfectHashIocpBufferPool.h + +Abstract: + + This is the private header file for IOCP buffer pool support. It defines + buffer header and pool structures used by the IOCP backend for overlapped + file I/O, along with helper routines for pool management. + +--*/ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifndef ALIGN_UP +#define ALIGN_UP(Address, Alignment) ( \ + (((ULONG_PTR)(Address)) + (Alignment) - 1) & \ + ~((ULONG_PTR)(Alignment) - 1) \ +) +#endif + +typedef struct _RTL RTL; +typedef RTL *PRTL; + +// +// IOCP buffer buckets keyed by AlignUpPow2(NumberOfKeys). +// + +#define PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT 32 +#define PERFECT_HASH_IOCP_BUFFER_MAX_BUCKET_INDEX \ + (PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT - 1) + +// +// Buffer pool flags. +// + +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_ONE_OFF 0x00000001 + +// +// IOCP buffer header used by IOCP file work. Payload begins at PayloadOffset. +// + +typedef struct DECLSPEC_ALIGN(16) _PERFECT_HASH_IOCP_BUFFER { + SLIST_ENTRY ListEntry; + ULONGLONG BytesWritten; + ULONGLONG PayloadSize; + ULONG SizeOfStruct; + ULONG PayloadOffset; + ULONG BucketIndex; + ULONG FileId; + USHORT NumaNode; + USHORT Flags; + ULONG Padding1; + ULONG Padding2; + ULONG Padding3; +} PERFECT_HASH_IOCP_BUFFER; +typedef PERFECT_HASH_IOCP_BUFFER *PPERFECT_HASH_IOCP_BUFFER; + +#define PERFECT_HASH_IOCP_BUFFER_HEADER_ALIGNMENT MEMORY_ALLOCATION_ALIGNMENT + +#define PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE ( \ + (ULONG)ALIGN_UP(sizeof(PERFECT_HASH_IOCP_BUFFER), \ + PERFECT_HASH_IOCP_BUFFER_HEADER_ALIGNMENT) \ +) + +FORCEINLINE +PVOID +PerfectHashIocpBufferPayload( + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + return (PVOID)((PCHAR)Buffer + Buffer->PayloadOffset); +} + +// +// IOCP buffer pool. +// + +typedef struct DECLSPEC_ALIGN(16) _PERFECT_HASH_IOCP_BUFFER_POOL { + ULONG SizeOfStruct; + ULONG Flags; + ULONG PageSize; + ULONG NumberOfBuffers; + ULONG NumberOfPagesPerBuffer; + ULONG PayloadOffset; + ULONGLONG UsableBufferSizeInBytes; + ULONGLONG PayloadSizeInBytes; + ULONGLONG BufferStrideInBytes; + ULONGLONG TotalAllocationSizeInBytes; + HANDLE ProcessHandle; + PVOID BaseAddress; + ULONGLONG ListHeadPadding; + SLIST_HEADER ListHead; +} PERFECT_HASH_IOCP_BUFFER_POOL; +typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_CREATE)( + _In_ PRTL Rtl, + _In_opt_ PHANDLE ProcessHandle, + _In_ ULONG PageSize, + _In_ ULONG NumberOfBuffers, + _In_ ULONG NumberOfPagesPerBuffer, + _In_opt_ PULONG AdditionalProtectionFlags, + _In_opt_ PULONG AdditionalAllocationTypeFlags, + _Out_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_CREATE + *PPERFECT_HASH_IOCP_BUFFER_POOL_CREATE; + +typedef +_Must_inspect_result_ +HRESULT +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY)( + _In_ PRTL Rtl, + _Inout_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY + *PPERFECT_HASH_IOCP_BUFFER_POOL_DESTROY; + +typedef +_Must_inspect_result_ +_Success_(return != NULL) +PPERFECT_HASH_IOCP_BUFFER +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_POP)( + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_POP + *PPERFECT_HASH_IOCP_BUFFER_POOL_POP; + +typedef +VOID +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_PUSH)( + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_PUSH + *PPERFECT_HASH_IOCP_BUFFER_POOL_PUSH; + +extern PERFECT_HASH_IOCP_BUFFER_POOL_CREATE PerfectHashIocpBufferPoolCreate; +extern PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY PerfectHashIocpBufferPoolDestroy; +extern PERFECT_HASH_IOCP_BUFFER_POOL_POP PerfectHashIocpBufferPoolPop; +extern PERFECT_HASH_IOCP_BUFFER_POOL_PUSH PerfectHashIocpBufferPoolPush; + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c index 062725e4..6a85b41a 100644 --- a/src/PerfectHash/PerfectHashServer.c +++ b/src/PerfectHash/PerfectHashServer.c @@ -2019,6 +2019,8 @@ PerfectHashServerBulkCreateWorkItemCallback( } if (WorkItem->Node && WorkItem->Node->IoCompletionPort) { + Context->IocpNode = WorkItem->Node; + Context->IocpNodeIndex = WorkItem->NodeIndex; Context->FileWorkIoCompletionPort = WorkItem->Node->IoCompletionPort; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 37133fe3..2803610c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,9 @@ endif() add_executable(perfecthash_unit_tests PerfectHashArgvTests.cpp + PerfectHashIocpBufferPoolTests.cpp + ${PERFECTHASH_SRC_DIR}/PerfectHash/PerfectHashIocpBufferPool.c + ${PERFECTHASH_SRC_DIR}/PerfectHash/RtlBuffers.c ) target_link_libraries(perfecthash_unit_tests @@ -26,6 +29,18 @@ target_link_libraries(perfecthash_unit_tests perfecthash_apply_common_definitions(perfecthash_unit_tests) +set_source_files_properties( + ${PERFECTHASH_SRC_DIR}/PerfectHash/PerfectHashIocpBufferPool.c + PROPERTIES + COMPILE_DEFINITIONS _PERFECT_HASH_INTERNAL_BUILD +) + +set_source_files_properties( + ${PERFECTHASH_SRC_DIR}/PerfectHash/RtlBuffers.c + PROPERTIES + COMPILE_DEFINITIONS _PERFECT_HASH_INTERNAL_BUILD +) + set_target_properties(perfecthash_unit_tests PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED YES diff --git a/tests/PerfectHashIocpBufferPoolTests.cpp b/tests/PerfectHashIocpBufferPoolTests.cpp new file mode 100644 index 00000000..11b6637f --- /dev/null +++ b/tests/PerfectHashIocpBufferPoolTests.cpp @@ -0,0 +1,114 @@ +#include + +#include + +#define _PERFECT_HASH_INTERNAL_BUILD +#include "PerfectHash/stdafx.h" +#include "PerfectHash/PerfectHashIocpBufferPool.h" + +namespace { + +void NTAPI +TestRtlFillMemory( + PVOID Destination, + ULONG_PTR Length, + BYTE Fill + ) +{ + std::memset(Destination, Fill, static_cast(Length)); +} + +void NTAPI +TestRtlZeroMemory( + PVOID Destination, + ULONG_PTR Length + ) +{ + std::memset(Destination, 0, static_cast(Length)); +} + +class PerfectHashIocpBufferPoolTests : public ::testing::Test { +protected: + void SetUp() override { + std::memset(&rtl_, 0, sizeof(rtl_)); + std::memset(&vtbl_, 0, sizeof(vtbl_)); + vtbl_.CreateMultipleBuffers = RtlCreateMultipleBuffers; + vtbl_.DestroyBuffer = RtlDestroyBuffer; + rtl_.Vtbl = &vtbl_; + rtl_.RtlFillMemory = TestRtlFillMemory; + rtl_.RtlZeroMemory = TestRtlZeroMemory; + } + + void TearDown() override { + rtl_.Vtbl = nullptr; + } + + RTL rtl_; + RTL_VTBL vtbl_; +}; + +TEST_F(PerfectHashIocpBufferPoolTests, CreatePopPush) { + PERFECT_HASH_IOCP_BUFFER_POOL pool; + std::memset(&pool, 0, sizeof(pool)); + EXPECT_EQ(reinterpret_cast(&pool) & 0xF, 0u); + EXPECT_EQ(reinterpret_cast(&pool.ListHead) & 0xF, 0u); + + HRESULT result = PerfectHashIocpBufferPoolCreate( + &rtl_, + nullptr, + 4096, + 2, + 1, + nullptr, + nullptr, + &pool); + + ASSERT_GE(result, 0); + EXPECT_GT(pool.PayloadSizeInBytes, 0u); + EXPECT_GT(pool.PayloadOffset, 0u); + EXPECT_EQ(pool.PayloadSizeInBytes + pool.PayloadOffset, + pool.UsableBufferSizeInBytes); + + PPERFECT_HASH_IOCP_BUFFER buffer = PerfectHashIocpBufferPoolPop(&pool); + ASSERT_NE(buffer, nullptr); + EXPECT_EQ(buffer->PayloadSize, pool.PayloadSizeInBytes); + EXPECT_EQ(buffer->PayloadOffset, pool.PayloadOffset); + + PVOID payload = PerfectHashIocpBufferPayload(buffer); + EXPECT_EQ(reinterpret_cast(payload) - + reinterpret_cast(buffer), + pool.PayloadOffset); + + PerfectHashIocpBufferPoolPush(&pool, buffer); + PerfectHashIocpBufferPoolDestroy(&rtl_, &pool); +} + +TEST_F(PerfectHashIocpBufferPoolTests, PopAllReturnsNull) { + PERFECT_HASH_IOCP_BUFFER_POOL pool; + std::memset(&pool, 0, sizeof(pool)); + EXPECT_EQ(reinterpret_cast(&pool) & 0xF, 0u); + EXPECT_EQ(reinterpret_cast(&pool.ListHead) & 0xF, 0u); + + HRESULT result = PerfectHashIocpBufferPoolCreate( + &rtl_, + nullptr, + 4096, + 1, + 1, + nullptr, + nullptr, + &pool); + + ASSERT_GE(result, 0); + + PPERFECT_HASH_IOCP_BUFFER first = PerfectHashIocpBufferPoolPop(&pool); + ASSERT_NE(first, nullptr); + EXPECT_EQ(PerfectHashIocpBufferPoolPop(&pool), nullptr); + + PerfectHashIocpBufferPoolPush(&pool, first); + EXPECT_NE(PerfectHashIocpBufferPoolPop(&pool), nullptr); + + PerfectHashIocpBufferPoolDestroy(&rtl_, &pool); +} + +} // namespace From c3e49b7ff219953c084862fbf247c639c776b9ca Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Fri, 23 Jan 2026 20:11:15 -0800 Subject: [PATCH 7/9] Checkpoint commit of buffer refactor. --- IOCP-LOGS.md | 26 + IOCP-TOOD.md | 22 +- src/PerfectHash/Chm01FileWork.c | 543 +++++++++----------- src/PerfectHash/PerfectHashContext.c | 185 +++++-- src/PerfectHash/PerfectHashContext.h | 4 +- src/PerfectHash/PerfectHashContextIocp.c | 212 +++++++- src/PerfectHash/PerfectHashContextIocp.h | 6 +- src/PerfectHash/PerfectHashIocpBufferPool.c | 276 +++++----- src/PerfectHash/PerfectHashIocpBufferPool.h | 179 +++++-- tests/PerfectHashIocpBufferPoolTests.cpp | 133 +++-- 10 files changed, 982 insertions(+), 604 deletions(-) diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index b95ee61c..2076db65 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -78,6 +78,14 @@ - IOCP bulk stress against `perfecthash-keys\\hard` (Debug) still times out in ~120s; server left running and only ~12/23 output dirs created, indicating completion/hang persists post-resize removal. - Attached `cdb` to `PerfectHashServer.exe` ~10s into a Debug IOCP bulk run and captured `~* kb` (`cdb-hard-backtrace.log`): main thread blocked in `PerfectHashServerWait`, IOCP worker threads blocked in `GetQueuedCompletionStatus` (33 waits on one IOCP handle, 24 on another), and ~1539 threads waiting in `ZwWaitForWorkViaWorkerFactory`, indicating no active IOCP completions at capture time and unexpected threadpool worker proliferation. - Added IOCP-native argument extraction helpers (`PerfectHashContextIocpArgs.c`) and wired into build files. + +## 2026-01-25 +- Refactored IOCP file-work buffers to size-class pools (4KB–16MB) plus dynamic oversize pools; buffers tracked via guarded lists for teardown. +- Updated IOCP context/node structs and rundown paths to free pooled buffers and oversize pool lists. +- Reworked CHM01 file-work buffer acquisition to use size-class/oversize pools (no one-off pools) and new acquire/release APIs. +- Updated IOCP buffer pool API to accept `PRTL` for initialization zeroing; removed legacy create/pop/push/destroy calls. +- Updated IOCP buffer pool unit tests and size-class helper coverage. +- Built Release `perfecthash_unit_tests` and ran `build-win\\bin\\Release\\perfecthash_unit_tests.exe` (all 8 tests pass). - Added TLS flag plumbing for threadpool-less context creation and IOCP helper to create contexts with `CreateContextWithoutThreadpool` set. - Updated server bulk/table create to avoid legacy context parsing and use IOCP-native arg extraction. - Added IOCP table-create dispatch in server (context creation, file-work IOCP wiring, hook/RNG init, TLS context) and removed threadpool priority application. @@ -167,3 +175,21 @@ - IOCP file-I/O run on `D:\\src\\perfecthash-keys\\hard` (Release, `Mulshrolate4RX`, `--IocpConcurrency=4 --MaxThreads=8`) completed successfully. - Added fail-fast `PH_RAISE` guards for oversized IOCP write requests and reset `NumberOfBytesWritten` for IOCP save-only file work items. - Full IOCP sys32 file-I/O run completed (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `G:\\Scratch\\iocp-sys32-full` in ~289.6s; largest output file ~2.5MB, no anomalously large files observed. +- OG BulkCreate sys32 file-I/O run completed (Release, `Mulshrolate4RX`, concurrency 32) to `G:\\Scratch\\sys32-og-full` in ~126.6s; largest output file ~21.3MB (CSV), no anomalously large files observed. +- IOCP sys32 file-I/O run with per-file ramp 1/16/500 completed (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `G:\\Scratch\\iocp-sys32-full-max16` in ~289.6s; largest output file ~2.5MB, no anomalously large files observed. +- Created a fresh 1000-file sys32 subset at `E:\\Scratch\\sys32-1000` for ramp tuning. +- OG BulkCreate run on `E:\\Scratch\\sys32-1000` completed in ~22.9s (Release, `Mulshrolate4RX`, concurrency 32) to `E:\\Scratch\\sys32-1000-og`. +- IOCP run on `E:\\Scratch\\sys32-1000` with ramp `Initial=2`, `Max=8`, `After=200ms` completed in ~21.1s (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `E:\\Scratch\\sys32-1000-iocp-2-8-200`. +- IOCP run on `E:\\Scratch\\sys32-1000` with ramp `Initial=1`, `Max=8`, `After=200ms` completed in ~21.1s (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `E:\\Scratch\\sys32-1000-iocp-1-8-200-2`. +- IOCP run on `E:\\Scratch\\sys32-1000` with ramp `Initial=1`, `Max=4`, `After=200ms` completed in ~20.1s (Release, `Mulshrolate4RX`, `--IocpConcurrency=32 --MaxThreads=64 --MaximumConcurrency=32`) to `E:\\Scratch\\sys32-1000-iocp-1-4-200`. +- IOCP run on `E:\\Scratch\\sys32-1000` with `--IocpConcurrency=64 --MaxThreads=196`, ramp `Initial=1`, `Max=4`, `After=100ms` hit `VirtualAllocEx` failures (error 1455) and `PH_E_OUTOFMEMORY` during save (`SaveCppSourceUnityFile`); run completed in ~21.1s but was not clean (`E:\\Scratch\\sys32-1000-iocp-1-4-100-64`). +- IOCP run on `E:\\Scratch\\sys32-1000` with `--IocpConcurrency=64 --MaxThreads=128 --MaximumConcurrency=32`, ramp `Initial=1`, `Max=2`, `After=100ms` still hit `VirtualAllocEx` failures (error 1455) and `PH_E_OUTOFMEMORY` during save (`SaveCppSourceUnityFile`); run completed in ~20.1s but was not clean (`E:\\Scratch\\sys32-1000-iocp-1-2-100-64`). + +## 2026-01-25 +- Planned IOCP buffer pool redesign based on `D:\\src\\lookaside` NUMA lookaside pattern: + - Replace per-file/bucket pools with per-NUMA size-class pools keyed by payload size (power-of-two). + - Size classes initially 4KB–16MB; oversize allocations use dynamic pools keyed by next power-of-two. + - Pools allocate on demand; buffers are reusable via SLIST free lists and tracked via `GUARDED_LIST` for rundown. + - One-off allocations become reusable buffers owned by a pool; guarded list enables safe teardown. + - Add optional guard-page allocation flag (default on) to choose between guard-page buffers vs raw chunks. + - Add basic pool stats/diagnostics later (depth, alloc failures); ETW deferred. diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md index b5fdb686..9f159440 100644 --- a/IOCP-TOOD.md +++ b/IOCP-TOOD.md @@ -3,18 +3,16 @@ - Overlapped I/O plan (IOCP only; OG remains memory-mapped): - Done: added `UseOverlappedIo` flag to `PERFECT_HASH_CONTEXT` (IOCP contexts set it). - Done: added `PerfectHashFileCreateFlags.SkipMapping` so output files can be created with `FILE_FLAG_OVERLAPPED` and no mapping. - - Implement a per-NUMA buffer pool: - - Buckets keyed by `AlignUpPow2(NumberOfKeys)` (32 buckets max), then indexed by file id. - - Each bucket/file id has an `SLIST_HEADER` of buffers. - - New buffer header struct: `SLIST_ENTRY`, `ULONG Size`, `ULONG Bucket`, `ULONG FileId`, `USHORT NumaNode`, `USHORT Flags`, `PVOID Base`, `ULONG BytesCapacity`, `ULONG BytesWritten`. - - Base address for file work writes must point *after* the header. Account for header overhead in sizing; rely on existing EOF multipliers + trap-on-overrun. If traps occur, bump multipliers. - - Allocate buffers from NUMA node (or `RtlCreateMultipleBuffers()` if it supports NUMA), with guard pages enabled. - - Cap per-bucket buffers to `IocpConcurrency` (or `min(IocpConcurrency, FilesInBucket)`). - - Done (initial): per-file buffer pool backed by `RtlCreateMultipleBuffers()` + guard pages. - - Done (initial): per-NUMA bucketed pool indexing by bucket+file id (on-demand allocation). Still need directory-scan sizing and pre-allocation. - - Pre-size and allocate pools per bulk request: - - During directory scan, capture `NumberOfKeys` per file; compute bucket index and per-file output sizes via existing file work table logic. - - Compute total bytes for all buffers (sum of bucket capacity * per-buffer size) and allocate one large NUMA block; carve into buffer headers + payloads. +- Done: rebuild IOCP file-work buffers as NUMA size-class pools (lookaside-style). + - Pools keyed by payload size class (power-of-two); 4KB–16MB classes. + - Dynamic oversize pools keyed by next power-of-two above max size class. + - Each pool has an `SLIST_HEADER` free list and a `GUARDED_LIST` of all buffers for rundown. + - New buffer header: `SLIST_ENTRY` + `LIST_ENTRY` + `OwnerPool` + `PayloadSize` + + `AllocationSize` + `PayloadOffset` + `BytesWritten` + flags. + - Base address for file work writes points *after* the header; sizing still uses EOF multipliers + trap-on-overrun. + - Allocate on demand; buffers pushed to pool free list and guarded list for teardown. + - One-off allocations removed; buffers reused via size-class or oversize pools. + - TODO: add a guard-page allocation flag to toggle `RtlCreateBuffer()` vs raw allocation. - Output file pipeline (IOCP): - Pop a buffer for each file work save stage. - Set `File->BaseAddress` to buffer base (after header) to reuse existing `OUTPUT_*` macros. diff --git a/src/PerfectHash/Chm01FileWork.c b/src/PerfectHash/Chm01FileWork.c index 559374f4..2a0491bb 100644 --- a/src/PerfectHash/Chm01FileWork.c +++ b/src/PerfectHash/Chm01FileWork.c @@ -43,270 +43,183 @@ FILE_WORK_CALLBACK_IMPL *FileCallbacks[] = { }; static -ULONG -Chm01GetFileWorkBufferBucketIndex( - _In_ PRTL Rtl, - _In_ ULONGLONG NumberOfKeys - ) -{ - ULONGLONG Rounded; - ULONG BucketIndex; - - if (!ARGUMENT_PRESENT(Rtl)) { - return 0; - } - - if (NumberOfKeys == 0) { - return 0; - } - - Rounded = Rtl->RoundUpPowerOfTwo64(NumberOfKeys); - if (Rounded == 0) { - return 0; - } - - BucketIndex = (ULONG)Rtl->TrailingZeros64(Rounded); - if (BucketIndex > PERFECT_HASH_IOCP_BUFFER_MAX_BUCKET_INDEX) { - BucketIndex = PERFECT_HASH_IOCP_BUFFER_MAX_BUCKET_INDEX; - } - - return BucketIndex; -} - -static -ULONG -Chm01GetFileWorkBufferBucketIndexForFile( +VOID +Chm01GetFileWorkBufferPoolState( _In_ PPERFECT_HASH_CONTEXT Context, - _In_ ULONG FileIndex + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL **PoolsPointer, + _Out_ PULONG *PoolCountPointer, + _Outptr_ PGUARDED_LIST *BufferListPointer, + _Outptr_ PLIST_ENTRY *OversizeListPointer, + _Outptr_ PULONG *OversizeCountPointer, + _Outptr_ PSRWLOCK *PoolLockPointer, + _Out_ PUSHORT NumaNodePointer ) { - FILE_ID FileId; - ULONGLONG Count; - PRTL Rtl; - PCEOF_INIT Eof; - PPERFECT_HASH_TABLE Table; - PPERFECT_HASH_KEYS Keys; - - if (!ARGUMENT_PRESENT(Context)) { - return 0; - } - - Rtl = Context->Rtl; - if (!Rtl) { - return 0; - } - - FileId = (FILE_ID)(FileIndex + 1); - if (!IsValidFileId(FileId)) { - return 0; - } + PPERFECT_HASH_IOCP_NODE Node; - Table = Context->Table; - Keys = Table ? Table->Keys : NULL; - if (!Table || !Keys) { - return 0; + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer) || + !ARGUMENT_PRESENT(BufferListPointer) || + !ARGUMENT_PRESENT(OversizeListPointer) || + !ARGUMENT_PRESENT(OversizeCountPointer) || + !ARGUMENT_PRESENT(PoolLockPointer) || + !ARGUMENT_PRESENT(NumaNodePointer)) { + return; } - Count = Keys->NumberOfKeys.QuadPart; - Eof = &EofInits[FileId]; - if (Eof->Type == EofInitTypeNumberOfTableElementsMultiplier && - Table->TableInfoOnDisk) { - Count = Table->TableInfoOnDisk->NumberOfTableElements.QuadPart; + Node = Context->IocpNode; + if (Node) { + *PoolsPointer = &Node->FileWorkBufferPools; + *PoolCountPointer = &Node->FileWorkBufferPoolCount; + *BufferListPointer = Node->FileWorkBufferList; + *OversizeListPointer = &Node->FileWorkOversizePools; + *OversizeCountPointer = &Node->FileWorkOversizePoolCount; + *PoolLockPointer = &Node->FileWorkBufferPoolLock; + *NumaNodePointer = (USHORT)Node->NodeId; + } else { + *PoolsPointer = &Context->FileWorkBufferPools; + *PoolCountPointer = &Context->FileWorkBufferPoolCount; + *BufferListPointer = Context->FileWorkBufferList; + *OversizeListPointer = &Context->FileWorkOversizePools; + *OversizeCountPointer = &Context->FileWorkOversizePoolCount; + *PoolLockPointer = &Context->Lock; + *NumaNodePointer = 0; } - - return Chm01GetFileWorkBufferBucketIndex(Rtl, Count); } static HRESULT -Chm01GetFileWorkBufferPool( +Chm01EnsureFileWorkBufferPools( _In_ PPERFECT_HASH_CONTEXT Context, - _In_ ULONG FileIndex, - _In_ ULONG BucketIndex, - _In_ ULONGLONG RequiredPayloadBytes, - _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolsPointer, + _Out_ PULONG PoolCountPointer, + _Outptr_ PGUARDED_LIST *BufferListPointer, + _Out_ PUSHORT NumaNodePointer ) { HRESULT Result; - ULONG Pages; - ULONG BufferCount; + ULONG Flags; ULONG PoolIndex; ULONG PoolCount; - ULONGLONG RequiredSize; + ULONGLONG PayloadSize; PALLOCATOR Allocator; PRTL Rtl; PPERFECT_HASH_IOCP_BUFFER_POOL Pools; - PPERFECT_HASH_IOCP_BUFFER_POOL Pool; - PPERFECT_HASH_IOCP_NODE Node; - - if (!ARGUMENT_PRESENT(Context)) { - return E_POINTER; - } + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; - if (!ARGUMENT_PRESENT(PoolPointer)) { + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer) || + !ARGUMENT_PRESENT(BufferListPointer) || + !ARGUMENT_PRESENT(NumaNodePointer)) { return E_POINTER; } - *PoolPointer = NULL; - - if (FileIndex >= NUMBER_OF_FILES) { - return E_INVALIDARG; - } - Allocator = Context->Allocator; Rtl = Context->Rtl; - if (!Allocator || !Rtl) { return E_UNEXPECTED; } - Node = Context->IocpNode; - if (Node) { - if (!Node->FileWorkBufferPools) { - AcquireSRWLockExclusive(&Node->FileWorkBufferPoolLock); - if (!Node->FileWorkBufferPools) { - PoolCount = PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT * - NUMBER_OF_FILES; - if (PoolCount < PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT || - PoolCount < NUMBER_OF_FILES) { - ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); - return E_INVALIDARG; - } - - Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( - Allocator, - PoolCount, - sizeof(*Pools) - ); - if (!Pools) { - ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); - return E_OUTOFMEMORY; - } - - Node->FileWorkBufferPools = Pools; - Node->FileWorkBufferPoolBucketCount = - PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT; - Node->FileWorkBufferPoolFileCount = NUMBER_OF_FILES; - Node->FileWorkBufferPoolCount = PoolCount; - Node->FileWorkBufferPoolPageSize = PAGE_SIZE; - } - ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); - } - - if (BucketIndex >= Node->FileWorkBufferPoolBucketCount) { - return E_INVALIDARG; - } - - PoolIndex = (BucketIndex * Node->FileWorkBufferPoolFileCount) + - FileIndex; - if (PoolIndex >= Node->FileWorkBufferPoolCount) { - return E_INVALIDARG; - } - - Pool = &Node->FileWorkBufferPools[PoolIndex]; - } else { - if (!Context->FileWorkBufferPools) { - AcquireSRWLockExclusive(&Context->Lock); - if (!Context->FileWorkBufferPools) { - Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( - Allocator, - NUMBER_OF_FILES, - sizeof(*Pools) - ); - if (!Pools) { - ReleaseSRWLockExclusive(&Context->Lock); - return E_OUTOFMEMORY; - } - - Context->FileWorkBufferPools = Pools; - Context->FileWorkBufferPoolCount = NUMBER_OF_FILES; - Context->FileWorkBufferPoolPageSize = PAGE_SIZE; - } - ReleaseSRWLockExclusive(&Context->Lock); - } + Chm01GetFileWorkBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); - Pool = &Context->FileWorkBufferPools[FileIndex]; + if (!PoolArrayPointer || !PoolCountField || !BufferList) { + return E_UNEXPECTED; } - if (!Pool->BaseAddress) { - if (Node) { - AcquireSRWLockExclusive(&Node->FileWorkBufferPoolLock); - } else { - AcquireSRWLockExclusive(&Context->Lock); - } - if (!Pool->BaseAddress) { - if (Node && Node->IocpConcurrency != 0) { - BufferCount = Node->IocpConcurrency; - } else { - BufferCount = Context->MaximumConcurrency; - } - if (BufferCount == 0) { - BufferCount = 1; + Pools = *PoolArrayPointer; + if (!Pools) { + AcquireSRWLockExclusive(PoolLock); + Pools = *PoolArrayPointer; + if (!Pools) { + PoolCount = PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT; + Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + PoolCount, + sizeof(*Pools) + ); + if (!Pools) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; } - RequiredSize = RequiredPayloadBytes + - PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; - Pages = (ULONG)BYTES_TO_PAGES(RequiredSize); - if (Pages == 0) { - Pages = 1; - } + Flags = 0; + for (PoolIndex = 0; PoolIndex < PoolCount; PoolIndex++) { + PayloadSize = + PerfectHashIocpBufferGetPayloadSizeFromClassIndex( + (LONG)PoolIndex + ); - Result = PerfectHashIocpBufferPoolCreate( - Rtl, - &Context->ProcessHandle, - PAGE_SIZE, - BufferCount, - Pages, - NULL, - NULL, - Pool - ); + Result = PerfectHashIocpBufferPoolInitialize( + Rtl, + &Pools[PoolIndex], + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList + ); - if (FAILED(Result)) { - if (Node) { - ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); - } else { - ReleaseSRWLockExclusive(&Context->Lock); + if (FAILED(Result)) { + ReleaseSRWLockExclusive(PoolLock); + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Pools + ); + return Result; } - return Result; } + + *PoolArrayPointer = Pools; + *PoolCountField = PoolCount; } - if (Node) { - ReleaseSRWLockExclusive(&Node->FileWorkBufferPoolLock); - } else { - ReleaseSRWLockExclusive(&Context->Lock); - } + ReleaseSRWLockExclusive(PoolLock); } - *PoolPointer = Pool; + *PoolsPointer = Pools; + *PoolCountPointer = *PoolCountField; + *BufferListPointer = BufferList; + *NumaNodePointer = NumaNode; return S_OK; } static HRESULT -Chm01CreateOneOffFileWorkBuffer( +Chm01GetOversizeFileWorkBufferPool( _In_ PPERFECT_HASH_CONTEXT Context, - _In_ ULONGLONG RequiredPayloadBytes, - _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer, - _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + _In_ ULONGLONG PayloadSize, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer ) { HRESULT Result; - ULONG Pages; - ULONGLONG RequiredSize; + ULONG Flags; PALLOCATOR Allocator; PRTL Rtl; - PPERFECT_HASH_IOCP_BUFFER Buffer; PPERFECT_HASH_IOCP_BUFFER_POOL Pool; - - if (!ARGUMENT_PRESENT(PoolPointer) || !ARGUMENT_PRESENT(BufferPointer)) { - return E_POINTER; - } - - *PoolPointer = NULL; - *BufferPointer = NULL; - - if (!ARGUMENT_PRESENT(Context)) { + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PLIST_ENTRY Entry; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { return E_POINTER; } @@ -316,11 +229,33 @@ Chm01CreateOneOffFileWorkBuffer( return E_UNEXPECTED; } - RequiredSize = RequiredPayloadBytes + - PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; - Pages = (ULONG)BYTES_TO_PAGES(RequiredSize); - if (Pages == 0) { - Pages = 1; + Chm01GetFileWorkBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!OversizeList || !OversizeCount || !PoolLock) { + return E_UNEXPECTED; + } + + AcquireSRWLockExclusive(PoolLock); + + Entry = OversizeList->Flink; + while (Entry != OversizeList) { + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + if (Pool->PayloadSize == PayloadSize) { + ReleaseSRWLockExclusive(PoolLock); + *PoolPointer = Pool; + return S_OK; + } + + Entry = Entry->Flink; } Pool = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( @@ -329,35 +264,89 @@ Chm01CreateOneOffFileWorkBuffer( sizeof(*Pool) ); if (!Pool) { + ReleaseSRWLockExclusive(PoolLock); return E_OUTOFMEMORY; } - Result = PerfectHashIocpBufferPoolCreate(Rtl, - &Context->ProcessHandle, - PAGE_SIZE, - 1, - Pages, - NULL, - NULL, - Pool); + Flags = PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE; + Result = PerfectHashIocpBufferPoolInitialize(Rtl, + Pool, + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList); + if (FAILED(Result)) { Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + ReleaseSRWLockExclusive(PoolLock); return Result; } - Pool->Flags |= PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_ONE_OFF; - Buffer = PerfectHashIocpBufferPoolPop(Pool); - if (!Buffer) { - PerfectHashIocpBufferPoolDestroy(Rtl, Pool); - Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); - return E_OUTOFMEMORY; - } + InsertTailList(OversizeList, &Pool->ListEntry); + (*OversizeCount)++; + ReleaseSRWLockExclusive(PoolLock); *PoolPointer = Pool; - *BufferPointer = Buffer; return S_OK; } +static +HRESULT +Chm01GetFileWorkBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + LONG ClassIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + PGUARDED_LIST BufferList; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Rtl = Context->Rtl; + if (!Rtl) { + return E_UNEXPECTED; + } + + ClassIndex = PerfectHashIocpBufferGetClassIndex(Rtl, + RequiredPayloadBytes); + if (ClassIndex >= 0) { + Result = Chm01EnsureFileWorkBufferPools(Context, + &Pools, + &PoolCount, + &BufferList, + &NumaNode); + if (FAILED(Result)) { + return Result; + } + + if ((ULONG)ClassIndex >= PoolCount) { + return E_INVALIDARG; + } + + *PoolPointer = &Pools[ClassIndex]; + return S_OK; + } + + PayloadSize = Rtl->RoundUpPowerOfTwo64(RequiredPayloadBytes); + if (PayloadSize == 0) { + return E_INVALIDARG; + } + + return Chm01GetOversizeFileWorkBufferPool(Context, + PayloadSize, + PoolPointer); +} + static HRESULT Chm01AcquireFileWorkBuffer( @@ -369,75 +358,48 @@ Chm01AcquireFileWorkBuffer( ) { HRESULT Result; - ULONG Retry; - ULONG BucketIndex; + PALLOCATOR Allocator; + PRTL Rtl; PPERFECT_HASH_IOCP_BUFFER Buffer; PPERFECT_HASH_IOCP_BUFFER_POOL Pool; - if (!ARGUMENT_PRESENT(BufferPointer)) { + UNREFERENCED_PARAMETER(FileIndex); + + if (!ARGUMENT_PRESENT(BufferPointer) || !ARGUMENT_PRESENT(PoolPointer)) { return E_POINTER; } *BufferPointer = NULL; + *PoolPointer = NULL; - if (!ARGUMENT_PRESENT(Context) || !Context->Rtl) { + if (!ARGUMENT_PRESENT(Context)) { return E_POINTER; } - BucketIndex = Chm01GetFileWorkBufferBucketIndexForFile(Context, - FileIndex); + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } Result = Chm01GetFileWorkBufferPool(Context, - FileIndex, - BucketIndex, RequiredPayloadBytes, &Pool); if (FAILED(Result)) { return Result; } - if (Pool->PayloadSizeInBytes < RequiredPayloadBytes) { - Result = Chm01CreateOneOffFileWorkBuffer(Context, - RequiredPayloadBytes, - &Pool, - &Buffer); - if (FAILED(Result)) { - return Result; - } - goto AssignBuffer; - } - - Buffer = NULL; - for (Retry = 0; Retry < 1024; Retry++) { - Buffer = PerfectHashIocpBufferPoolPop(Pool); - if (Buffer) { - break; - } - if ((Retry & 0x3F) == 0) { - Sleep(0); - } else { - YieldProcessor(); - } - } - - if (!Buffer) { - Result = Chm01CreateOneOffFileWorkBuffer(Context, - RequiredPayloadBytes, - &Pool, - &Buffer); - if (FAILED(Result)) { - return Result; - } - goto AssignBuffer; + Result = PerfectHashIocpBufferPoolAcquire(Rtl, + Allocator, + Pool, + &Buffer); + if (FAILED(Result)) { + return Result; } -AssignBuffer: - Buffer->BucketIndex = BucketIndex; - Buffer->FileId = FileIndex; - if (Context && Context->IocpNode) { - Buffer->NumaNode = (USHORT)Context->IocpNode->NodeId; - } else { - Buffer->NumaNode = 0; + if (Buffer->PayloadSize < RequiredPayloadBytes) { + PerfectHashIocpBufferPoolRelease(Pool, Buffer); + return E_FAIL; } *PoolPointer = Pool; @@ -468,30 +430,13 @@ Chm01ReleaseIocpBuffer( _In_ PPERFECT_HASH_IOCP_BUFFER Buffer ) { - PALLOCATOR Allocator; - PRTL Rtl; + UNREFERENCED_PARAMETER(Context); if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(Buffer)) { return; } - if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_ONE_OFF) { - if (!ARGUMENT_PRESENT(Context)) { - return; - } - - Allocator = Context->Allocator; - Rtl = Context->Rtl; - if (!Allocator || !Rtl) { - return; - } - - PerfectHashIocpBufferPoolDestroy(Rtl, Pool); - Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); - return; - } - - PerfectHashIocpBufferPoolPush(Pool, Buffer); + PerfectHashIocpBufferPoolRelease(Pool, Buffer); } static @@ -774,7 +719,7 @@ Chm01WriteFileFromBuffer( BytesRemaining.QuadPart = File->NumberOfBytesWritten.QuadPart; if (BytesRemaining.QuadPart <= 0) { - PerfectHashIocpBufferPoolPush(Pool, Buffer); + PerfectHashIocpBufferPoolRelease(Pool, Buffer); return S_OK; } diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index 06c7bce9..ff5f9bbb 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -125,6 +125,110 @@ Return Value: return MaximumConcurrency + 2; } +static +VOID +PerfectHashIocpFreeBuffer( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + BOOL Success; + HRESULT Result; + HANDLE ProcessHandle; + PVOID BaseAddress; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + ProcessHandle = NULL; + if (Buffer->OwnerPool) { + ProcessHandle = Buffer->OwnerPool->ProcessHandle; + } + + if (!ProcessHandle) { + ProcessHandle = GetCurrentProcess(); + } + + if (Buffer->Flags & PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES) { + BaseAddress = Buffer; + Result = Rtl->Vtbl->DestroyBuffer(Rtl, + ProcessHandle, + &BaseAddress, + Buffer->AllocationSize); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpFreeBuffer_DestroyBuffer, Result); + } + return; + } + + Success = VirtualFreeEx(ProcessHandle, Buffer, 0, MEM_RELEASE); + if (!Success) { + SYS_ERROR(VirtualFreeEx); + } +} + +static +VOID +PerfectHashIocpRundownBufferList( + _In_ PRTL Rtl, + _In_ PGUARDED_LIST BufferList + ) +{ + BOOLEAN NotEmpty; + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(BufferList)) { + return; + } + + while (TRUE) { + Entry = NULL; + NotEmpty = BufferList->Vtbl->RemoveHeadEx(BufferList, &Entry); + if (!NotEmpty) { + break; + } + + Buffer = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER, + ListEntry); + + PerfectHashIocpFreeBuffer(Rtl, Buffer); + } + + BufferList->Vtbl->Reset(BufferList); +} + +static +VOID +PerfectHashIocpRundownOversizePools( + _In_ PALLOCATOR Allocator, + _Inout_ PLIST_ENTRY ListHead, + _Inout_opt_ PULONG PoolCount + ) +{ + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + if (!ARGUMENT_PRESENT(Allocator) || !ARGUMENT_PRESENT(ListHead)) { + return; + } + + while (!IsListEmpty(ListHead)) { + Entry = RemoveHeadList(ListHead); + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + } + + InitializeListHead(ListHead); + if (PoolCount) { + *PoolCount = 0; + } +} + // // Main context creation routine. @@ -249,6 +353,18 @@ Return Value: goto Error; } + Result = Context->Vtbl->CreateInstance(Context, + NULL, + &IID_PERFECT_HASH_GUARDED_LIST, + &Context->FileWorkBufferList); + + if (FAILED(Result)) { + goto Error; + } + + InitializeListHead(&Context->FileWorkOversizePools); + Context->FileWorkOversizePoolCount = 0; + // // Initialize aliases. // @@ -897,9 +1013,7 @@ Return Value: PALLOCATOR Allocator; BYTE Index; BYTE NumberOfEvents; - ULONG PoolIndex; PHANDLE Event; - PPERFECT_HASH_IOCP_BUFFER_POOL Pool; // // Validate arguments. @@ -1065,28 +1179,27 @@ Return Value: #endif - if (Context->FileWorkBufferPools) { - for (PoolIndex = 0; - PoolIndex < Context->FileWorkBufferPoolCount; - PoolIndex++) { - - Pool = &Context->FileWorkBufferPools[PoolIndex]; - if (!Pool->BaseAddress) { - continue; - } + if (Context->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, Context->FileWorkBufferList); + } - Result = PerfectHashIocpBufferPoolDestroy(Rtl, Pool); - if (FAILED(Result)) { - PH_ERROR(PerfectHashIocpBufferPoolDestroy, Result); - } + if (Allocator) { + if (Context->FileWorkBufferPools) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Context->FileWorkBufferPools + ); } - Allocator->Vtbl->FreePointer(Allocator, - (PVOID *)&Context->FileWorkBufferPools); - Context->FileWorkBufferPoolCount = 0; - Context->FileWorkBufferPoolPageSize = 0; + PerfectHashIocpRundownOversizePools( + Allocator, + &Context->FileWorkOversizePools, + &Context->FileWorkOversizePoolCount + ); } + Context->FileWorkBufferPoolCount = 0; + if (Context->ObjectNames) { Allocator->Vtbl->FreePointer(Allocator, &Context->ObjectNames); @@ -1110,6 +1223,7 @@ Return Value: RELEASE(Context->MainWorkList); RELEASE(Context->FileWorkList); + RELEASE(Context->FileWorkBufferList); RELEASE(Context->FinishedWorkList); RELEASE(Context->BulkCreateCsvFile); RELEASE(Context->BaseOutputDirectory); @@ -1149,11 +1263,8 @@ Return Value: --*/ { - HRESULT Result; PRTL Rtl; PALLOCATOR Allocator; - ULONG PoolIndex; - PPERFECT_HASH_IOCP_BUFFER_POOL Pool; if (!ARGUMENT_PRESENT(Context)) { return E_POINTER; @@ -1233,33 +1344,27 @@ Return Value: } #endif - if (Context->FileWorkBufferPools) { - for (PoolIndex = 0; - PoolIndex < Context->FileWorkBufferPoolCount; - PoolIndex++) { - - Pool = &Context->FileWorkBufferPools[PoolIndex]; - if (!Pool->BaseAddress) { - continue; - } - - Result = PerfectHashIocpBufferPoolDestroy(Rtl, Pool); - if (FAILED(Result)) { - PH_ERROR(PerfectHashIocpBufferPoolDestroy, Result); - } - } + if (Context->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, Context->FileWorkBufferList); + } - if (Allocator) { + if (Allocator) { + if (Context->FileWorkBufferPools) { Allocator->Vtbl->FreePointer( Allocator, (PVOID *)&Context->FileWorkBufferPools ); } - Context->FileWorkBufferPoolCount = 0; - Context->FileWorkBufferPoolPageSize = 0; + PerfectHashIocpRundownOversizePools( + Allocator, + &Context->FileWorkOversizePools, + &Context->FileWorkOversizePoolCount + ); } + Context->FileWorkBufferPoolCount = 0; + Context->KeysSubset = NULL; Context->UserSeeds = NULL; Context->SeedMasks = NULL; diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index 6b88a78a..e0878a68 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -1144,7 +1144,9 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { ULONG IocpNodePadding; PPERFECT_HASH_IOCP_BUFFER_POOL FileWorkBufferPools; ULONG FileWorkBufferPoolCount; - ULONG FileWorkBufferPoolPageSize; + ULONG FileWorkOversizePoolCount; + LIST_ENTRY FileWorkOversizePools; + PGUARDED_LIST FileWorkBufferList; volatile LONG GraphRegisterSolvedTsxSuccess; ULONG Padding4; diff --git a/src/PerfectHash/PerfectHashContextIocp.c b/src/PerfectHash/PerfectHashContextIocp.c index ad447527..db97e3da 100644 --- a/src/PerfectHash/PerfectHashContextIocp.c +++ b/src/PerfectHash/PerfectHashContextIocp.c @@ -46,6 +46,28 @@ PerfectHashIocpApplyNodeAffinity( _In_ PPERFECT_HASH_IOCP_NODE Node ); +static +VOID +PerfectHashIocpFreeBuffer( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ); + +static +VOID +PerfectHashIocpRundownBufferList( + _In_ PRTL Rtl, + _In_ PGUARDED_LIST BufferList + ); + +static +VOID +PerfectHashIocpRundownOversizePools( + _In_ PALLOCATOR Allocator, + _Inout_ PLIST_ENTRY ListHead, + _Inout_opt_ PULONG PoolCount + ); + static #ifdef PH_WINDOWS DWORD @@ -60,6 +82,110 @@ PerfectHashIocpWorkerThreadProc( ); #endif +static +VOID +PerfectHashIocpFreeBuffer( + _In_ PRTL Rtl, + _In_ PPERFECT_HASH_IOCP_BUFFER Buffer + ) +{ + BOOL Success; + HRESULT Result; + HANDLE ProcessHandle; + PVOID BaseAddress; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(Buffer)) { + return; + } + + ProcessHandle = NULL; + if (Buffer->OwnerPool) { + ProcessHandle = Buffer->OwnerPool->ProcessHandle; + } + + if (!ProcessHandle) { + ProcessHandle = GetCurrentProcess(); + } + + if (Buffer->Flags & PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES) { + BaseAddress = Buffer; + Result = Rtl->Vtbl->DestroyBuffer(Rtl, + ProcessHandle, + &BaseAddress, + Buffer->AllocationSize); + if (FAILED(Result)) { + PH_ERROR(PerfectHashIocpFreeBuffer_DestroyBuffer, Result); + } + return; + } + + Success = VirtualFreeEx(ProcessHandle, Buffer, 0, MEM_RELEASE); + if (!Success) { + SYS_ERROR(VirtualFreeEx); + } +} + +static +VOID +PerfectHashIocpRundownBufferList( + _In_ PRTL Rtl, + _In_ PGUARDED_LIST BufferList + ) +{ + BOOLEAN NotEmpty; + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(BufferList)) { + return; + } + + while (TRUE) { + Entry = NULL; + NotEmpty = BufferList->Vtbl->RemoveHeadEx(BufferList, &Entry); + if (!NotEmpty) { + break; + } + + Buffer = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER, + ListEntry); + + PerfectHashIocpFreeBuffer(Rtl, Buffer); + } + + BufferList->Vtbl->Reset(BufferList); +} + +static +VOID +PerfectHashIocpRundownOversizePools( + _In_ PALLOCATOR Allocator, + _Inout_ PLIST_ENTRY ListHead, + _Inout_opt_ PULONG PoolCount + ) +{ + PLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + + if (!ARGUMENT_PRESENT(Allocator) || !ARGUMENT_PRESENT(ListHead)) { + return; + } + + while (!IsListEmpty(ListHead)) { + Entry = RemoveHeadList(ListHead); + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + } + + InitializeListHead(ListHead); + if (PoolCount) { + *PoolCount = 0; + } +} + PERFECT_HASH_CONTEXT_IOCP_INITIALIZE PerfectHashContextIocpInitialize; _Use_decl_annotations_ @@ -184,42 +310,30 @@ Return Value: PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[Index]; ULONG ThreadIndex; - if (Node->FileWorkBufferPools) { - ULONG PoolIndex; - - for (PoolIndex = 0; - PoolIndex < Node->FileWorkBufferPoolCount; - PoolIndex++) { - PPERFECT_HASH_IOCP_BUFFER_POOL Pool; - - Pool = &Node->FileWorkBufferPools[PoolIndex]; - if (!Pool->BaseAddress) { - continue; - } - - if (Rtl) { - HRESULT Result; - - Result = PerfectHashIocpBufferPoolDestroy(Rtl, Pool); - if (FAILED(Result)) { - PH_ERROR(PerfectHashIocpBufferPoolDestroy, Result); - } - } - } + if (Node->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, + Node->FileWorkBufferList); + } - if (Allocator) { + if (Allocator) { + if (Node->FileWorkBufferPools) { Allocator->Vtbl->FreePointer( Allocator, (PVOID *)&Node->FileWorkBufferPools ); } - Node->FileWorkBufferPoolBucketCount = 0; - Node->FileWorkBufferPoolFileCount = 0; - Node->FileWorkBufferPoolCount = 0; - Node->FileWorkBufferPoolPageSize = 0; + PerfectHashIocpRundownOversizePools( + Allocator, + &Node->FileWorkOversizePools, + &Node->FileWorkOversizePoolCount + ); } + Node->FileWorkBufferPoolCount = 0; + + RELEASE(Node->FileWorkBufferList); + if (Node->WorkerThreads) { for (ThreadIndex = 0; ThreadIndex < Node->WorkerThreadCount; @@ -374,6 +488,8 @@ Return Value: #else ULONG NodeIndex; ULONG ThreadIndex; + PRTL Rtl; + PALLOCATOR Allocator; if (!ARGUMENT_PRESENT(ContextIocp)) { return E_POINTER; @@ -383,6 +499,9 @@ Return Value: return S_FALSE; } + Rtl = ContextIocp->Rtl; + Allocator = ContextIocp->Allocator; + ContextIocp->State.Stopping = TRUE; if (ContextIocp->ShutdownEvent) { SetEvent(ContextIocp->ShutdownEvent); @@ -417,6 +536,30 @@ Return Value: for (NodeIndex = 0; NodeIndex < ContextIocp->NodeCount; NodeIndex++) { PPERFECT_HASH_IOCP_NODE Node = &ContextIocp->Nodes[NodeIndex]; + if (Node->FileWorkBufferList && Rtl) { + PerfectHashIocpRundownBufferList(Rtl, + Node->FileWorkBufferList); + } + + if (Allocator) { + if (Node->FileWorkBufferPools) { + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Node->FileWorkBufferPools + ); + } + + PerfectHashIocpRundownOversizePools( + Allocator, + &Node->FileWorkOversizePools, + &Node->FileWorkOversizePoolCount + ); + } + + Node->FileWorkBufferPoolCount = 0; + + RELEASE(Node->FileWorkBufferList); + if (Node->WorkerThreads) { for (ThreadIndex = 0; ThreadIndex < Node->WorkerThreadCount; @@ -552,6 +695,7 @@ PerfectHashContextIocpEnumerateNumaNodes( { #ifdef PH_WINDOWS BOOL Success; + HRESULT Result; USHORT NodeId; ULONG HighestNode; USHORT HighestNodeShort; @@ -662,6 +806,20 @@ PerfectHashContextIocpEnumerateNumaNodes( Node->ProcessorCount = ProcessorCount; Node->GroupAffinity = Affinity; InitializeSRWLock(&Node->FileWorkBufferPoolLock); + InitializeListHead(&Node->FileWorkOversizePools); + Node->FileWorkBufferPoolCount = 0; + Node->FileWorkOversizePoolCount = 0; + + Result = ContextIocp->Vtbl->CreateInstance( + ContextIocp, + NULL, + &IID_PERFECT_HASH_GUARDED_LIST, + &Node->FileWorkBufferList + ); + + if (FAILED(Result)) { + return Result; + } } // diff --git a/src/PerfectHash/PerfectHashContextIocp.h b/src/PerfectHash/PerfectHashContextIocp.h index 3eebce1a..fb783a1e 100644 --- a/src/PerfectHash/PerfectHashContextIocp.h +++ b/src/PerfectHash/PerfectHashContextIocp.h @@ -111,10 +111,10 @@ typedef struct _PERFECT_HASH_IOCP_NODE { SRWLOCK FileWorkBufferPoolLock; PPERFECT_HASH_IOCP_BUFFER_POOL FileWorkBufferPools; - ULONG FileWorkBufferPoolBucketCount; - ULONG FileWorkBufferPoolFileCount; ULONG FileWorkBufferPoolCount; - ULONG FileWorkBufferPoolPageSize; + ULONG FileWorkOversizePoolCount; + LIST_ENTRY FileWorkOversizePools; + PGUARDED_LIST FileWorkBufferList; } PERFECT_HASH_IOCP_NODE; typedef PERFECT_HASH_IOCP_NODE *PPERFECT_HASH_IOCP_NODE; diff --git a/src/PerfectHash/PerfectHashIocpBufferPool.c b/src/PerfectHash/PerfectHashIocpBufferPool.c index 7ff57d6a..b5f9757b 100644 --- a/src/PerfectHash/PerfectHashIocpBufferPool.c +++ b/src/PerfectHash/PerfectHashIocpBufferPool.c @@ -9,200 +9,232 @@ Module Name: Abstract: This module implements IOCP buffer pool routines used by the IOCP backend - for overlapped file I/O. + for overlapped file I/O. Buffers are allocated on demand from size-class + pools and returned to SLIST free lists for reuse. --*/ #include "stdafx.h" #include "PerfectHashIocpBufferPool.h" -PERFECT_HASH_IOCP_BUFFER_POOL_CREATE PerfectHashIocpBufferPoolCreate; -PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY PerfectHashIocpBufferPoolDestroy; -PERFECT_HASH_IOCP_BUFFER_POOL_POP PerfectHashIocpBufferPoolPop; -PERFECT_HASH_IOCP_BUFFER_POOL_PUSH PerfectHashIocpBufferPoolPush; +PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE PerfectHashIocpBufferPoolInitialize; +PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE PerfectHashIocpBufferPoolAcquire; +PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE PerfectHashIocpBufferPoolRelease; +PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN PerfectHashIocpBufferPoolRundown; -_Use_decl_annotations_ +static +_Must_inspect_result_ HRESULT -PerfectHashIocpBufferPoolCreate( - PRTL Rtl, - PHANDLE ProcessHandle, - ULONG PageSize, - ULONG NumberOfBuffers, - ULONG NumberOfPagesPerBuffer, - PULONG AdditionalProtectionFlags, - PULONG AdditionalAllocationTypeFlags, - PPERFECT_HASH_IOCP_BUFFER_POOL Pool +PerfectHashIocpBufferPoolAllocate( + _In_ PRTL Rtl, + _In_opt_ PALLOCATOR Allocator, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer ) { - ULONG Index; - ULONG PayloadOffset; - PBYTE Base; + ULONG Pages; HRESULT Result; - ULONGLONG UsableBufferSizeInBytes; - ULONGLONG TotalBufferSizeInBytes; - ULONGLONG BufferStrideInBytes; - ULONGLONG PayloadSizeInBytes; + HANDLE ProcessHandle; + PVOID BaseAddress; + ULONGLONG AllocationSize; + ULONGLONG UsableBufferSize; + ULONGLONG PayloadSize; PPERFECT_HASH_IOCP_BUFFER Buffer; - PRTL_VTBL Vtbl; - HANDLE TargetProcessHandle; - PHANDLE TargetProcessHandlePointer; - if (!ARGUMENT_PRESENT(Rtl)) { - return E_POINTER; - } + UNREFERENCED_PARAMETER(Allocator); - if (!ARGUMENT_PRESENT(Pool)) { + if (!ARGUMENT_PRESENT(Rtl) || + !ARGUMENT_PRESENT(Pool) || + !ARGUMENT_PRESENT(BufferPointer)) { return E_POINTER; } - if (!NumberOfBuffers || !NumberOfPagesPerBuffer) { - return E_INVALIDARG; + *BufferPointer = NULL; + BaseAddress = NULL; + UsableBufferSize = 0; + PayloadSize = Pool->PayloadSize; + + ProcessHandle = Pool->ProcessHandle; + if (!ProcessHandle) { + ProcessHandle = GetCurrentProcess(); } - if (!PageSize) { + AllocationSize = Pool->AllocationSize; + if (AllocationSize == 0) { return E_INVALIDARG; } - Vtbl = Rtl->Vtbl; - - ZeroMemory(Pool, sizeof(*Pool)); - InitializeSListHead(&Pool->ListHead); + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES) { + Pages = (ULONG)BYTES_TO_PAGES(AllocationSize); + if (Pages == 0) { + Pages = 1; + } + + Result = Rtl->Vtbl->CreateBuffer(Rtl, + &ProcessHandle, + Pages, + NULL, + &UsableBufferSize, + &BaseAddress); + if (FAILED(Result)) { + return Result; + } + + AllocationSize = ( + ((ULONGLONG)Pages + 1) * (ULONGLONG)PAGE_SIZE + ); + if (UsableBufferSize <= PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE) { + Rtl->Vtbl->DestroyBuffer(Rtl, + ProcessHandle, + &BaseAddress, + AllocationSize); + return E_FAIL; + } + + PayloadSize = UsableBufferSize - + (ULONGLONG)PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; - TargetProcessHandle = NULL; - if (ARGUMENT_PRESENT(ProcessHandle)) { - TargetProcessHandlePointer = ProcessHandle; } else { - TargetProcessHandlePointer = &TargetProcessHandle; - } - - Result = Vtbl->CreateMultipleBuffers(Rtl, - TargetProcessHandlePointer, - PageSize, - NumberOfBuffers, - NumberOfPagesPerBuffer, - AdditionalProtectionFlags, - AdditionalAllocationTypeFlags, - &UsableBufferSizeInBytes, - &TotalBufferSizeInBytes, - &Pool->BaseAddress); - - if (FAILED(Result)) { - return Result; - } - Pool->ProcessHandle = *TargetProcessHandlePointer; - PayloadOffset = PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + BaseAddress = VirtualAllocEx(ProcessHandle, + NULL, + AllocationSize, + MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE); + if (!BaseAddress) { + SYS_ERROR(VirtualAllocEx); + return E_OUTOFMEMORY; + } + } + + Buffer = (PPERFECT_HASH_IOCP_BUFFER)BaseAddress; + Rtl->RtlZeroMemory(Buffer, PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE); + Buffer->SizeOfStruct = sizeof(*Buffer); + Buffer->PayloadOffset = PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE; + Buffer->PayloadSize = PayloadSize; + Buffer->AllocationSize = AllocationSize; + Buffer->BytesWritten = 0; + Buffer->OwnerPool = Pool; + Buffer->Flags = 0; - if ((ULONGLONG)PayloadOffset >= UsableBufferSizeInBytes) { - Vtbl->DestroyBuffer(Rtl, - TargetProcessHandle, - &Pool->BaseAddress, - TotalBufferSizeInBytes); - return E_FAIL; + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES) { + Buffer->Flags |= PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES; } - PayloadSizeInBytes = UsableBufferSizeInBytes - (ULONGLONG)PayloadOffset; - BufferStrideInBytes = UsableBufferSizeInBytes + (ULONGLONG)PageSize; + if (Pool->Flags & PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE) { + Buffer->Flags |= PERFECT_HASH_IOCP_BUFFER_FLAG_OVERSIZE; + } - Pool->SizeOfStruct = sizeof(*Pool); - Pool->PageSize = PageSize; - Pool->NumberOfBuffers = NumberOfBuffers; - Pool->NumberOfPagesPerBuffer = NumberOfPagesPerBuffer; - Pool->PayloadOffset = PayloadOffset; - Pool->UsableBufferSizeInBytes = UsableBufferSizeInBytes; - Pool->PayloadSizeInBytes = PayloadSizeInBytes; - Pool->BufferStrideInBytes = BufferStrideInBytes; - Pool->TotalAllocationSizeInBytes = TotalBufferSizeInBytes; - Pool->ProcessHandle = TargetProcessHandle; - - Base = (PBYTE)Pool->BaseAddress; - for (Index = 0; Index < NumberOfBuffers; Index++) { - Buffer = (PPERFECT_HASH_IOCP_BUFFER)Base; - Buffer->SizeOfStruct = sizeof(*Buffer); - Buffer->PayloadOffset = PayloadOffset; - Buffer->PayloadSize = PayloadSizeInBytes; - Buffer->BytesWritten = 0; - Buffer->Flags = 0; - Buffer->BucketIndex = 0; - Buffer->FileId = 0; - Buffer->NumaNode = 0; - InterlockedPushEntrySList(&Pool->ListHead, &Buffer->ListEntry); - Base += BufferStrideInBytes; + if (Pool->BufferList) { + Pool->BufferList->Vtbl->InsertTail(Pool->BufferList, + &Buffer->ListEntry); } + *BufferPointer = Buffer; return S_OK; } _Use_decl_annotations_ HRESULT -PerfectHashIocpBufferPoolDestroy( +PerfectHashIocpBufferPoolInitialize( PRTL Rtl, - PPERFECT_HASH_IOCP_BUFFER_POOL Pool + PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + ULONGLONG PayloadSize, + USHORT NumaNode, + ULONG Flags, + HANDLE ProcessHandle, + PGUARDED_LIST BufferList ) { - HRESULT Result; - - if (!ARGUMENT_PRESENT(Rtl)) { + if (!ARGUMENT_PRESENT(Rtl) || !ARGUMENT_PRESENT(Pool)) { return E_POINTER; } - if (!ARGUMENT_PRESENT(Pool)) { - return E_POINTER; + if (!PayloadSize) { + return E_INVALIDARG; } - if (!Pool->BaseAddress) { - return S_OK; - } + Rtl->RtlZeroMemory(Pool, sizeof(*Pool)); + InitializeSListHead(&Pool->FreeList); + InitializeListHead(&Pool->ListEntry); - Result = Rtl->Vtbl->DestroyBuffer(Rtl, - Pool->ProcessHandle, - &Pool->BaseAddress, - Pool->TotalAllocationSizeInBytes); - if (FAILED(Result)) { - return Result; - } + Pool->SizeOfStruct = sizeof(*Pool); + Pool->PayloadSize = PayloadSize; + Pool->AllocationSize = PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE + PayloadSize; + Pool->Flags = Flags | PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_INITIALIZED; + Pool->NumaNode = NumaNode; + Pool->ProcessHandle = ProcessHandle; + Pool->BufferList = BufferList; - ZeroMemory(Pool, sizeof(*Pool)); return S_OK; } _Use_decl_annotations_ -PPERFECT_HASH_IOCP_BUFFER -PerfectHashIocpBufferPoolPop( - PPERFECT_HASH_IOCP_BUFFER_POOL Pool +HRESULT +PerfectHashIocpBufferPoolAcquire( + PRTL Rtl, + PALLOCATOR Allocator, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + PPERFECT_HASH_IOCP_BUFFER *BufferPointer ) { PSLIST_ENTRY Entry; + PPERFECT_HASH_IOCP_BUFFER Buffer; - if (!ARGUMENT_PRESENT(Pool)) { - return NULL; + if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; } - Entry = InterlockedPopEntrySList(&Pool->ListHead); - if (!Entry) { - return NULL; + *BufferPointer = NULL; + + Entry = InterlockedPopEntrySList(&Pool->FreeList); + if (Entry) { + Buffer = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER, + FreeListEntry); + *BufferPointer = Buffer; + return S_OK; } - return CONTAINING_RECORD(Entry, PERFECT_HASH_IOCP_BUFFER, ListEntry); + return PerfectHashIocpBufferPoolAllocate(Rtl, + Allocator, + Pool, + BufferPointer); } _Use_decl_annotations_ VOID -PerfectHashIocpBufferPoolPush( +PerfectHashIocpBufferPoolRelease( PPERFECT_HASH_IOCP_BUFFER_POOL Pool, PPERFECT_HASH_IOCP_BUFFER Buffer ) { - if (!ARGUMENT_PRESENT(Pool)) { + if (!ARGUMENT_PRESENT(Pool) || !ARGUMENT_PRESENT(Buffer)) { return; } - if (!ARGUMENT_PRESENT(Buffer)) { + Buffer->BytesWritten = 0; + InterlockedPushEntrySList(&Pool->FreeList, &Buffer->FreeListEntry); +} + +_Use_decl_annotations_ +VOID +PerfectHashIocpBufferPoolRundown( + PRTL Rtl, + PALLOCATOR Allocator, + PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ) +{ + UNREFERENCED_PARAMETER(Rtl); + UNREFERENCED_PARAMETER(Allocator); + + if (!ARGUMENT_PRESENT(Pool)) { return; } - Buffer->BytesWritten = 0; - InterlockedPushEntrySList(&Pool->ListHead, &Buffer->ListEntry); + InitializeSListHead(&Pool->FreeList); + InitializeListHead(&Pool->ListEntry); + ZeroMemory(Pool, sizeof(*Pool)); } // vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashIocpBufferPool.h b/src/PerfectHash/PerfectHashIocpBufferPool.h index fb8ceda4..6a17f87d 100644 --- a/src/PerfectHash/PerfectHashIocpBufferPool.h +++ b/src/PerfectHash/PerfectHashIocpBufferPool.h @@ -33,36 +33,54 @@ typedef struct _RTL RTL; typedef RTL *PRTL; // -// IOCP buffer buckets keyed by AlignUpPow2(NumberOfKeys). +// IOCP buffer size-class configuration (payload sizes in bytes). // -#define PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT 32 -#define PERFECT_HASH_IOCP_BUFFER_MAX_BUCKET_INDEX \ - (PERFECT_HASH_IOCP_BUFFER_BUCKET_COUNT - 1) +#define PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT 12 // 4KB +#define PERFECT_HASH_IOCP_BUFFER_MAX_SHIFT 24 // 16MB + +#define PERFECT_HASH_IOCP_BUFFER_MIN_SIZE \ + (1ULL << PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT) + +#define PERFECT_HASH_IOCP_BUFFER_MAX_SIZE \ + (1ULL << PERFECT_HASH_IOCP_BUFFER_MAX_SHIFT) + +#define PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT ( \ + PERFECT_HASH_IOCP_BUFFER_MAX_SHIFT - \ + PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT + \ + 1 \ +) + +// +// Buffer flags. +// + +#define PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES 0x00000001 +#define PERFECT_HASH_IOCP_BUFFER_FLAG_OVERSIZE 0x00000002 // -// Buffer pool flags. +// Pool flags. // -#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_ONE_OFF 0x00000001 +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_INITIALIZED 0x00000001 +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES 0x00000002 +#define PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE 0x00000004 // // IOCP buffer header used by IOCP file work. Payload begins at PayloadOffset. // typedef struct DECLSPEC_ALIGN(16) _PERFECT_HASH_IOCP_BUFFER { - SLIST_ENTRY ListEntry; + SLIST_ENTRY FreeListEntry; + LIST_ENTRY ListEntry; + struct _PERFECT_HASH_IOCP_BUFFER_POOL *OwnerPool; ULONGLONG BytesWritten; ULONGLONG PayloadSize; + ULONGLONG AllocationSize; ULONG SizeOfStruct; ULONG PayloadOffset; - ULONG BucketIndex; - ULONG FileId; - USHORT NumaNode; - USHORT Flags; + ULONG Flags; ULONG Padding1; - ULONG Padding2; - ULONG Padding3; } PERFECT_HASH_IOCP_BUFFER; typedef PERFECT_HASH_IOCP_BUFFER *PPERFECT_HASH_IOCP_BUFFER; @@ -82,77 +100,126 @@ PerfectHashIocpBufferPayload( return (PVOID)((PCHAR)Buffer + Buffer->PayloadOffset); } +FORCEINLINE +LONG +PerfectHashIocpBufferGetClassIndex( + _In_ PRTL Rtl, + _In_ ULONGLONG PayloadBytes + ) +{ + ULONGLONG Rounded; + ULONG TrailingZeros; + + if (!ARGUMENT_PRESENT(Rtl)) { + return -1; + } + + if (PayloadBytes <= PERFECT_HASH_IOCP_BUFFER_MIN_SIZE) { + return 0; + } + + Rounded = Rtl->RoundUpPowerOfTwo64(PayloadBytes); + if (Rounded == 0) { + return -1; + } + + if (Rounded > PERFECT_HASH_IOCP_BUFFER_MAX_SIZE) { + return -1; + } + + TrailingZeros = (ULONG)Rtl->TrailingZeros64(Rounded); + return (LONG)TrailingZeros - PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT; +} + +FORCEINLINE +ULONGLONG +PerfectHashIocpBufferGetPayloadSizeFromClassIndex( + _In_ LONG ClassIndex + ) +{ + ULONGLONG Shift; + + if (ClassIndex < 0) { + return 0; + } + + Shift = (ULONGLONG)ClassIndex + PERFECT_HASH_IOCP_BUFFER_MIN_SHIFT; + return (1ULL << Shift); +} + // // IOCP buffer pool. // typedef struct DECLSPEC_ALIGN(16) _PERFECT_HASH_IOCP_BUFFER_POOL { + LIST_ENTRY ListEntry; + SLIST_HEADER FreeList; + PGUARDED_LIST BufferList; + HANDLE ProcessHandle; + ULONGLONG PayloadSize; + ULONGLONG AllocationSize; ULONG SizeOfStruct; ULONG Flags; - ULONG PageSize; - ULONG NumberOfBuffers; - ULONG NumberOfPagesPerBuffer; - ULONG PayloadOffset; - ULONGLONG UsableBufferSizeInBytes; - ULONGLONG PayloadSizeInBytes; - ULONGLONG BufferStrideInBytes; - ULONGLONG TotalAllocationSizeInBytes; - HANDLE ProcessHandle; - PVOID BaseAddress; - ULONGLONG ListHeadPadding; - SLIST_HEADER ListHead; + ULONG Padding1; + USHORT NumaNode; + USHORT Padding2; } PERFECT_HASH_IOCP_BUFFER_POOL; typedef PERFECT_HASH_IOCP_BUFFER_POOL *PPERFECT_HASH_IOCP_BUFFER_POOL; typedef _Must_inspect_result_ HRESULT -(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_CREATE)( +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE)( _In_ PRTL Rtl, - _In_opt_ PHANDLE ProcessHandle, - _In_ ULONG PageSize, - _In_ ULONG NumberOfBuffers, - _In_ ULONG NumberOfPagesPerBuffer, - _In_opt_ PULONG AdditionalProtectionFlags, - _In_opt_ PULONG AdditionalAllocationTypeFlags, - _Out_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _In_ ULONGLONG PayloadSize, + _In_ USHORT NumaNode, + _In_ ULONG Flags, + _In_opt_ HANDLE ProcessHandle, + _In_opt_ PGUARDED_LIST BufferList ); -typedef PERFECT_HASH_IOCP_BUFFER_POOL_CREATE - *PPERFECT_HASH_IOCP_BUFFER_POOL_CREATE; +typedef PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE + *PPERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE; typedef _Must_inspect_result_ HRESULT -(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY)( +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE)( _In_ PRTL Rtl, - _Inout_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool - ); -typedef PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY - *PPERFECT_HASH_IOCP_BUFFER_POOL_DESTROY; - -typedef -_Must_inspect_result_ -_Success_(return != NULL) -PPERFECT_HASH_IOCP_BUFFER -(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_POP)( - _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + _In_opt_ PALLOCATOR Allocator, + _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer ); -typedef PERFECT_HASH_IOCP_BUFFER_POOL_POP - *PPERFECT_HASH_IOCP_BUFFER_POOL_POP; +typedef PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE + *PPERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE; typedef VOID -(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_PUSH)( +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE)( _In_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool, _In_ PPERFECT_HASH_IOCP_BUFFER Buffer ); -typedef PERFECT_HASH_IOCP_BUFFER_POOL_PUSH - *PPERFECT_HASH_IOCP_BUFFER_POOL_PUSH; +typedef PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE + *PPERFECT_HASH_IOCP_BUFFER_POOL_RELEASE; -extern PERFECT_HASH_IOCP_BUFFER_POOL_CREATE PerfectHashIocpBufferPoolCreate; -extern PERFECT_HASH_IOCP_BUFFER_POOL_DESTROY PerfectHashIocpBufferPoolDestroy; -extern PERFECT_HASH_IOCP_BUFFER_POOL_POP PerfectHashIocpBufferPoolPop; -extern PERFECT_HASH_IOCP_BUFFER_POOL_PUSH PerfectHashIocpBufferPoolPush; +typedef +VOID +(NTAPI PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN)( + _In_ PRTL Rtl, + _In_opt_ PALLOCATOR Allocator, + _Inout_ PPERFECT_HASH_IOCP_BUFFER_POOL Pool + ); +typedef PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN + *PPERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN; + +extern PERFECT_HASH_IOCP_BUFFER_POOL_INITIALIZE + PerfectHashIocpBufferPoolInitialize; +extern PERFECT_HASH_IOCP_BUFFER_POOL_ACQUIRE + PerfectHashIocpBufferPoolAcquire; +extern PERFECT_HASH_IOCP_BUFFER_POOL_RELEASE + PerfectHashIocpBufferPoolRelease; +extern PERFECT_HASH_IOCP_BUFFER_POOL_RUNDOWN + PerfectHashIocpBufferPoolRundown; #ifdef __cplusplus } // extern "C" diff --git a/tests/PerfectHashIocpBufferPoolTests.cpp b/tests/PerfectHashIocpBufferPoolTests.cpp index 11b6637f..1d63e594 100644 --- a/tests/PerfectHashIocpBufferPoolTests.cpp +++ b/tests/PerfectHashIocpBufferPoolTests.cpp @@ -27,16 +27,58 @@ TestRtlZeroMemory( std::memset(Destination, 0, static_cast(Length)); } +ULONGLONG +TestRoundUpPowerOfTwo64( + ULONGLONG Value + ) +{ + if (Value == 0) { + return 0; + } + + Value--; + Value |= Value >> 1; + Value |= Value >> 2; + Value |= Value >> 4; + Value |= Value >> 8; + Value |= Value >> 16; + Value |= Value >> 32; + Value++; + + return Value; +} + +ULONGLONG +TestTrailingZeros64( + ULONGLONG Value + ) +{ + ULONGLONG Count = 0; + + if (Value == 0) { + return 64; + } + + while ((Value & 1) == 0) { + Count++; + Value >>= 1; + } + + return Count; +} + class PerfectHashIocpBufferPoolTests : public ::testing::Test { protected: void SetUp() override { std::memset(&rtl_, 0, sizeof(rtl_)); std::memset(&vtbl_, 0, sizeof(vtbl_)); - vtbl_.CreateMultipleBuffers = RtlCreateMultipleBuffers; + vtbl_.CreateBuffer = RtlCreateBuffer; vtbl_.DestroyBuffer = RtlDestroyBuffer; rtl_.Vtbl = &vtbl_; rtl_.RtlFillMemory = TestRtlFillMemory; rtl_.RtlZeroMemory = TestRtlZeroMemory; + rtl_.RoundUpPowerOfTwo64 = TestRoundUpPowerOfTwo64; + rtl_.TrailingZeros64 = TestTrailingZeros64; } void TearDown() override { @@ -47,68 +89,71 @@ class PerfectHashIocpBufferPoolTests : public ::testing::Test { RTL_VTBL vtbl_; }; -TEST_F(PerfectHashIocpBufferPoolTests, CreatePopPush) { +TEST_F(PerfectHashIocpBufferPoolTests, InitializeAcquireRelease) { PERFECT_HASH_IOCP_BUFFER_POOL pool; std::memset(&pool, 0, sizeof(pool)); EXPECT_EQ(reinterpret_cast(&pool) & 0xF, 0u); - EXPECT_EQ(reinterpret_cast(&pool.ListHead) & 0xF, 0u); - HRESULT result = PerfectHashIocpBufferPoolCreate( + HRESULT result = PerfectHashIocpBufferPoolInitialize( &rtl_, - nullptr, + &pool, 4096, - 2, - 1, + 0, + 0, nullptr, - nullptr, - &pool); + nullptr); ASSERT_GE(result, 0); - EXPECT_GT(pool.PayloadSizeInBytes, 0u); - EXPECT_GT(pool.PayloadOffset, 0u); - EXPECT_EQ(pool.PayloadSizeInBytes + pool.PayloadOffset, - pool.UsableBufferSizeInBytes); - - PPERFECT_HASH_IOCP_BUFFER buffer = PerfectHashIocpBufferPoolPop(&pool); + EXPECT_GT(pool.PayloadSize, 0u); + EXPECT_GT(pool.AllocationSize, 0u); + + PPERFECT_HASH_IOCP_BUFFER buffer = nullptr; + result = PerfectHashIocpBufferPoolAcquire(&rtl_, + nullptr, + &pool, + &buffer); + ASSERT_GE(result, 0); ASSERT_NE(buffer, nullptr); - EXPECT_EQ(buffer->PayloadSize, pool.PayloadSizeInBytes); - EXPECT_EQ(buffer->PayloadOffset, pool.PayloadOffset); + EXPECT_EQ(buffer->PayloadSize, pool.PayloadSize); + EXPECT_EQ(buffer->PayloadOffset, PERFECT_HASH_IOCP_BUFFER_HEADER_SIZE); PVOID payload = PerfectHashIocpBufferPayload(buffer); EXPECT_EQ(reinterpret_cast(payload) - reinterpret_cast(buffer), - pool.PayloadOffset); - - PerfectHashIocpBufferPoolPush(&pool, buffer); - PerfectHashIocpBufferPoolDestroy(&rtl_, &pool); -} - -TEST_F(PerfectHashIocpBufferPoolTests, PopAllReturnsNull) { - PERFECT_HASH_IOCP_BUFFER_POOL pool; - std::memset(&pool, 0, sizeof(pool)); - EXPECT_EQ(reinterpret_cast(&pool) & 0xF, 0u); - EXPECT_EQ(reinterpret_cast(&pool.ListHead) & 0xF, 0u); + buffer->PayloadOffset); - HRESULT result = PerfectHashIocpBufferPoolCreate( - &rtl_, - nullptr, - 4096, - 1, - 1, - nullptr, - nullptr, - &pool); + PerfectHashIocpBufferPoolRelease(&pool, buffer); + PPERFECT_HASH_IOCP_BUFFER second = nullptr; + result = PerfectHashIocpBufferPoolAcquire(&rtl_, + nullptr, + &pool, + &second); ASSERT_GE(result, 0); + ASSERT_NE(second, nullptr); + EXPECT_EQ(second, buffer); + PerfectHashIocpBufferPoolRelease(&pool, second); - PPERFECT_HASH_IOCP_BUFFER first = PerfectHashIocpBufferPoolPop(&pool); - ASSERT_NE(first, nullptr); - EXPECT_EQ(PerfectHashIocpBufferPoolPop(&pool), nullptr); - - PerfectHashIocpBufferPoolPush(&pool, first); - EXPECT_NE(PerfectHashIocpBufferPoolPop(&pool), nullptr); + PerfectHashIocpBufferPoolRundown(&rtl_, nullptr, &pool); +} - PerfectHashIocpBufferPoolDestroy(&rtl_, &pool); +TEST_F(PerfectHashIocpBufferPoolTests, SizeClassHelpers) { + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 0), 0); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 1), 0); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 4096), 0); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 4097), 1); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 8192), 1); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex( + &rtl_, + PERFECT_HASH_IOCP_BUFFER_MAX_SIZE), + PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT - 1); + EXPECT_EQ(PerfectHashIocpBufferGetClassIndex( + &rtl_, + PERFECT_HASH_IOCP_BUFFER_MAX_SIZE + 1), + -1); + + EXPECT_EQ(PerfectHashIocpBufferGetPayloadSizeFromClassIndex(0), 4096u); + EXPECT_EQ(PerfectHashIocpBufferGetPayloadSizeFromClassIndex(1), 8192u); } } // namespace From 8dd6949fcfed6587f992d5a0cd1a6036901c89b5 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Fri, 23 Jan 2026 21:29:35 -0800 Subject: [PATCH 8/9] Checkpoint commit of Iocp refactor progress. --- IOCP-LOGS.md | 5 + IOCP-TOOD.md | 6 +- include/PerfectHash.h | 40 +- src/PerfectHash/Chm01FileWork.c | 7 +- src/PerfectHash/PerfectHashConstants.c | 4 +- src/PerfectHash/PerfectHashContext.h | 17 +- .../PerfectHashContextBulkCreate.c | 4 + .../PerfectHashContextTableCreate.c | 4 + src/PerfectHash/PerfectHashFile.c | 495 +++++++++++++++++- src/PerfectHash/PerfectHashKeysLoad.c | 4 + src/PerfectHash/PerfectHashServer.c | 54 ++ src/PerfectHash/PerfectHashServer.h | 7 +- .../PerfectHashServerExe.c | 26 +- tests/PerfectHashIocpBufferPoolTests.cpp | 28 + 14 files changed, 683 insertions(+), 18 deletions(-) diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index 2076db65..aaec4171 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -193,3 +193,8 @@ - One-off allocations become reusable buffers owned by a pool; guarded list enables safe teardown. - Add optional guard-page allocation flag (default on) to choose between guard-page buffers vs raw chunks. - Add basic pool stats/diagnostics later (depth, alloc failures); ETW deferred. + +## 2026-01-26 +- Added IOCP buffer guard-page toggle: new server flag + CLI (`--IocpBufferGuardPages`), context flag propagation, and pool init wiring. +- Implemented IOCP keys-load path: `PERFECT_HASH_KEYS_LOAD_FLAGS.UseOverlappedIo`, `PERFECT_HASH_FILE_LOAD_FLAGS.SkipMapping`, overlapped `ReadFile` into pooled buffers, and buffer release on file close. +- Added gtest coverage for guard-page buffer pool flag propagation. diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md index 9f159440..2cb70fa4 100644 --- a/IOCP-TOOD.md +++ b/IOCP-TOOD.md @@ -12,7 +12,7 @@ - Base address for file work writes points *after* the header; sizing still uses EOF multipliers + trap-on-overrun. - Allocate on demand; buffers pushed to pool free list and guarded list for teardown. - One-off allocations removed; buffers reused via size-class or oversize pools. - - TODO: add a guard-page allocation flag to toggle `RtlCreateBuffer()` vs raw allocation. + - Done: add a guard-page allocation flag to toggle `RtlCreateBuffer()` vs raw allocation. - Output file pipeline (IOCP): - Pop a buffer for each file work save stage. - Set `File->BaseAddress` to buffer base (after header) to reuse existing `OUTPUT_*` macros. @@ -20,8 +20,8 @@ - Done: issue `WriteFile` with IOCP work item and handle completion in IOCP callback (no `GetOverlappedResult` waits). - Done: `UnmapFileChm01()` no-ops in overlapped mode; `CloseFileChm01()` uses `File->NumberOfBytesWritten` (no mapping). - Input file pipeline (IOCP): - - Replace memory-mapped keys file reads with overlapped `ReadFile` into a keys buffer (or reuse pool per key-size bucket). - - Parse keys from buffer; keep mapped path for OG. + - Done: replace memory-mapped keys file reads with overlapped `ReadFile` into a keys buffer (reuse size-class pool). + - Done: parse keys from buffer; keep mapped path for OG. - Add lightweight stats (debug-only): pool exhaustion count, max buffers in use, per-bucket hits; optional ETW later. - Add unit tests for IOCP file-write completion handling (buffer return, event signal, error path). - Audit async accounting for leaks: diff --git a/include/PerfectHash.h b/include/PerfectHash.h index 35417da1..fd3befc5 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -1650,11 +1650,18 @@ typedef union _PERFECT_HASH_FILE_LOAD_FLAGS { ULONG TryLargePagesForFileData:1; + // + // When set, skip mapping the file into memory. The caller is + // responsible for providing a suitable buffer and handling any I/O. + // + + ULONG SkipMapping:1; + // // Unused bits. // - ULONG Unused:31; + ULONG Unused:30; }; LONG AsLong; @@ -1992,6 +1999,13 @@ typedef union _PERFECT_HASH_KEYS_LOAD_FLAGS { ULONG TryLargePagesForKeysData:1; + // + // When set, keys are loaded using overlapped I/O into a buffer instead + // of memory-mapped views. Intended for IOCP-native contexts. + // + + ULONG UseOverlappedIo:1; + // // When set, skips the verification of keys during loading. // Specifically, skips enumerating all keys and verifying that the @@ -2033,7 +2047,7 @@ typedef union _PERFECT_HASH_KEYS_LOAD_FLAGS { // Unused bits. // - ULONG Unused:28; + ULONG Unused:27; }; LONG AsLong; @@ -4726,6 +4740,24 @@ HRESULT typedef PERFECT_HASH_SERVER_GET_NO_FILE_IO *PPERFECT_HASH_SERVER_GET_NO_FILE_IO; +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES)( + _In_ PPERFECT_HASH_SERVER Server, + _In_ BOOLEAN GuardPages + ); +typedef PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + *PPERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES; + +typedef +HRESULT +(STDAPICALLTYPE PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES)( + _In_ PPERFECT_HASH_SERVER Server, + _Out_ PBOOLEAN GuardPages + ); +typedef PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + *PPERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES; + typedef HRESULT (STDAPICALLTYPE PERFECT_HASH_SERVER_START)( @@ -4773,6 +4805,10 @@ typedef struct _PERFECT_HASH_SERVER_VTBL { PPERFECT_HASH_SERVER_GET_VERBOSE GetVerbose; PPERFECT_HASH_SERVER_SET_NO_FILE_IO SetNoFileIo; PPERFECT_HASH_SERVER_GET_NO_FILE_IO GetNoFileIo; + PPERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + SetIocpBufferGuardPages; + PPERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + GetIocpBufferGuardPages; PPERFECT_HASH_SERVER_START Start; PPERFECT_HASH_SERVER_STOP Stop; PPERFECT_HASH_SERVER_WAIT Wait; diff --git a/src/PerfectHash/Chm01FileWork.c b/src/PerfectHash/Chm01FileWork.c index 2a0491bb..9d4a6e97 100644 --- a/src/PerfectHash/Chm01FileWork.c +++ b/src/PerfectHash/Chm01FileWork.c @@ -157,7 +157,9 @@ Chm01EnsureFileWorkBufferPools( return E_OUTOFMEMORY; } - Flags = 0; + Flags = UseIocpBufferGuardPages(Context) ? + PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES : + 0; for (PoolIndex = 0; PoolIndex < PoolCount; PoolIndex++) { PayloadSize = PerfectHashIocpBufferGetPayloadSizeFromClassIndex( @@ -269,6 +271,9 @@ Chm01GetOversizeFileWorkBufferPool( } Flags = PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE; + if (UseIocpBufferGuardPages(Context)) { + Flags |= PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES; + } Result = PerfectHashIocpBufferPoolInitialize(Rtl, Pool, PayloadSize, diff --git a/src/PerfectHash/PerfectHashConstants.c b/src/PerfectHash/PerfectHashConstants.c index ce8252c6..b180e94d 100644 --- a/src/PerfectHash/PerfectHashConstants.c +++ b/src/PerfectHash/PerfectHashConstants.c @@ -1182,12 +1182,14 @@ const PERFECT_HASH_SERVER_VTBL PerfectHashServerInterface = { &PerfectHashServerGetVerbose, &PerfectHashServerSetNoFileIo, &PerfectHashServerGetNoFileIo, + &PerfectHashServerSetIocpBufferGuardPages, + &PerfectHashServerGetIocpBufferGuardPages, &PerfectHashServerStart, &PerfectHashServerStop, &PerfectHashServerWait, &PerfectHashServerSubmitRequest, }; -VERIFY_VTBL_SIZE(PERFECT_HASH_SERVER, 18); +VERIFY_VTBL_SIZE(PERFECT_HASH_SERVER, 20); // // PerfectHashClient diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index e0878a68..647819d2 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -287,11 +287,17 @@ typedef union _PERFECT_HASH_CONTEXT_FLAGS { ULONG UseOverlappedIo:1; + // + // When set, IOCP file I/O buffers use guard pages. + // + + ULONG UseIocpBufferGuardPages:1; + // // Unused bits. // - ULONG Unused:30; + ULONG Unused:29; }; LONG AsLong; ULONG AsULong; @@ -311,6 +317,15 @@ typedef PERFECT_HASH_CONTEXT_FLAGS *PPERFECT_HASH_CONTEXT_FLAGS; #define ClearContextUseOverlappedIo(Context) \ ((Context)->Flags.UseOverlappedIo = FALSE) +#define UseIocpBufferGuardPages(Context) \ + ((Context)->Flags.UseIocpBufferGuardPages != FALSE) + +#define SetContextUseIocpBufferGuardPages(Context) \ + ((Context)->Flags.UseIocpBufferGuardPages = TRUE) + +#define ClearContextUseIocpBufferGuardPages(Context) \ + ((Context)->Flags.UseIocpBufferGuardPages = FALSE) + typedef struct _BEST_GRAPH_INFO { // diff --git a/src/PerfectHash/PerfectHashContextBulkCreate.c b/src/PerfectHash/PerfectHashContextBulkCreate.c index 60c9a064..79419c62 100644 --- a/src/PerfectHash/PerfectHashContextBulkCreate.c +++ b/src/PerfectHash/PerfectHashContextBulkCreate.c @@ -206,6 +206,10 @@ Return Value: VALIDATE_FLAGS(KeysLoad, KEYS_LOAD, ULong); VALIDATE_FLAGS(TableCompile, TABLE_COMPILE, ULong); + if (UseOverlappedIo(Context)) { + KeysLoadFlags.UseOverlappedIo = TRUE; + } + // // IsValidTableCreateFlags() returns a more specific error code than the // other validation routines invoked above (which would be converted into diff --git a/src/PerfectHash/PerfectHashContextTableCreate.c b/src/PerfectHash/PerfectHashContextTableCreate.c index c3e80a9f..628c7420 100644 --- a/src/PerfectHash/PerfectHashContextTableCreate.c +++ b/src/PerfectHash/PerfectHashContextTableCreate.c @@ -244,6 +244,10 @@ Return Value: VALIDATE_FLAGS(KeysLoad, KEYS_LOAD, ULong); VALIDATE_FLAGS(TableCompile, TABLE_COMPILE, ULong); + if (UseOverlappedIo(Context)) { + KeysLoadFlags.UseOverlappedIo = TRUE; + } + // // IsValidTableCreateFlags() returns a more specific error code than the // other validation routines invoked above (which would be converted into diff --git a/src/PerfectHash/PerfectHashFile.c b/src/PerfectHash/PerfectHashFile.c index fd72d1b0..ee4e8d0e 100644 --- a/src/PerfectHash/PerfectHashFile.c +++ b/src/PerfectHash/PerfectHashFile.c @@ -15,6 +15,7 @@ Module Name: --*/ #include "stdafx.h" +#include "PerfectHashIocpBufferPool.h" // // Helper inline method for updating the file's FILE_INFO structure. @@ -41,6 +42,457 @@ PerfectHashFileUpdateFileInfo( return S_OK; } +static +VOID +PerfectHashFileGetIocpBufferPoolState( + _In_ PPERFECT_HASH_CONTEXT Context, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL **PoolsPointer, + _Out_ PULONG *PoolCountPointer, + _Outptr_ PGUARDED_LIST *BufferListPointer, + _Outptr_ PLIST_ENTRY *OversizeListPointer, + _Outptr_ PULONG *OversizeCountPointer, + _Outptr_ PSRWLOCK *PoolLockPointer, + _Out_ PUSHORT NumaNodePointer + ) +{ + PPERFECT_HASH_IOCP_NODE Node; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer) || + !ARGUMENT_PRESENT(BufferListPointer) || + !ARGUMENT_PRESENT(OversizeListPointer) || + !ARGUMENT_PRESENT(OversizeCountPointer) || + !ARGUMENT_PRESENT(PoolLockPointer) || + !ARGUMENT_PRESENT(NumaNodePointer)) { + return; + } + + Node = Context->IocpNode; + if (Node) { + *PoolsPointer = &Node->FileWorkBufferPools; + *PoolCountPointer = &Node->FileWorkBufferPoolCount; + *BufferListPointer = Node->FileWorkBufferList; + *OversizeListPointer = &Node->FileWorkOversizePools; + *OversizeCountPointer = &Node->FileWorkOversizePoolCount; + *PoolLockPointer = &Node->FileWorkBufferPoolLock; + *NumaNodePointer = (USHORT)Node->NodeId; + } else { + *PoolsPointer = &Context->FileWorkBufferPools; + *PoolCountPointer = &Context->FileWorkBufferPoolCount; + *BufferListPointer = Context->FileWorkBufferList; + *OversizeListPointer = &Context->FileWorkOversizePools; + *OversizeCountPointer = &Context->FileWorkOversizePoolCount; + *PoolLockPointer = &Context->Lock; + *NumaNodePointer = 0; + } +} + +static +HRESULT +PerfectHashFileEnsureIocpBufferPools( + _In_ PPERFECT_HASH_CONTEXT Context, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolsPointer, + _Out_ PULONG PoolCountPointer + ) +{ + HRESULT Result; + ULONG Flags; + ULONG PoolIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolsPointer) || + !ARGUMENT_PRESENT(PoolCountPointer)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PerfectHashFileGetIocpBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!PoolArrayPointer || !PoolCountField || !BufferList) { + return E_UNEXPECTED; + } + + Pools = *PoolArrayPointer; + if (!Pools) { + AcquireSRWLockExclusive(PoolLock); + Pools = *PoolArrayPointer; + if (!Pools) { + PoolCount = PERFECT_HASH_IOCP_BUFFER_CLASS_COUNT; + Pools = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + PoolCount, + sizeof(*Pools) + ); + if (!Pools) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; + } + + Flags = UseIocpBufferGuardPages(Context) ? + PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES : + 0; + for (PoolIndex = 0; PoolIndex < PoolCount; PoolIndex++) { + PayloadSize = + PerfectHashIocpBufferGetPayloadSizeFromClassIndex( + (LONG)PoolIndex + ); + + Result = PerfectHashIocpBufferPoolInitialize( + Rtl, + &Pools[PoolIndex], + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList + ); + + if (FAILED(Result)) { + ReleaseSRWLockExclusive(PoolLock); + Allocator->Vtbl->FreePointer( + Allocator, + (PVOID *)&Pools + ); + return Result; + } + } + + *PoolArrayPointer = Pools; + *PoolCountField = PoolCount; + } + ReleaseSRWLockExclusive(PoolLock); + } + + *PoolsPointer = Pools; + *PoolCountPointer = *PoolCountField; + return S_OK; +} + +static +HRESULT +PerfectHashFileGetOversizeIocpBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG PayloadSize, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + ULONG Flags; + PALLOCATOR Allocator; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER_POOL *PoolArrayPointer; + PULONG PoolCountField; + PGUARDED_LIST BufferList; + PLIST_ENTRY OversizeList; + PLIST_ENTRY Entry; + PULONG OversizeCount; + PSRWLOCK PoolLock; + USHORT NumaNode; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Allocator = Context->Allocator; + Rtl = Context->Rtl; + if (!Allocator || !Rtl) { + return E_UNEXPECTED; + } + + PerfectHashFileGetIocpBufferPoolState(Context, + &PoolArrayPointer, + &PoolCountField, + &BufferList, + &OversizeList, + &OversizeCount, + &PoolLock, + &NumaNode); + + if (!OversizeList || !OversizeCount || !PoolLock) { + return E_UNEXPECTED; + } + + AcquireSRWLockExclusive(PoolLock); + + Entry = OversizeList->Flink; + while (Entry != OversizeList) { + Pool = CONTAINING_RECORD(Entry, + PERFECT_HASH_IOCP_BUFFER_POOL, + ListEntry); + if (Pool->PayloadSize == PayloadSize) { + ReleaseSRWLockExclusive(PoolLock); + *PoolPointer = Pool; + return S_OK; + } + + Entry = Entry->Flink; + } + + Pool = (PPERFECT_HASH_IOCP_BUFFER_POOL)Allocator->Vtbl->Calloc( + Allocator, + 1, + sizeof(*Pool) + ); + if (!Pool) { + ReleaseSRWLockExclusive(PoolLock); + return E_OUTOFMEMORY; + } + + Flags = PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_OVERSIZE; + if (UseIocpBufferGuardPages(Context)) { + Flags |= PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES; + } + + Result = PerfectHashIocpBufferPoolInitialize(Rtl, + Pool, + PayloadSize, + NumaNode, + Flags, + Context->ProcessHandle, + BufferList); + + if (FAILED(Result)) { + Allocator->Vtbl->FreePointer(Allocator, (PVOID *)&Pool); + ReleaseSRWLockExclusive(PoolLock); + return Result; + } + + InsertTailList(OversizeList, &Pool->ListEntry); + (*OversizeCount)++; + ReleaseSRWLockExclusive(PoolLock); + + *PoolPointer = Pool; + return S_OK; +} + +static +HRESULT +PerfectHashFileGetIocpBufferPool( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer + ) +{ + HRESULT Result; + LONG ClassIndex; + ULONG PoolCount; + ULONGLONG PayloadSize; + PRTL Rtl; + PPERFECT_HASH_IOCP_BUFFER_POOL Pools; + + if (!ARGUMENT_PRESENT(Context) || !ARGUMENT_PRESENT(PoolPointer)) { + return E_POINTER; + } + + Rtl = Context->Rtl; + if (!Rtl) { + return E_UNEXPECTED; + } + + ClassIndex = PerfectHashIocpBufferGetClassIndex(Rtl, + RequiredPayloadBytes); + if (ClassIndex >= 0) { + Result = PerfectHashFileEnsureIocpBufferPools(Context, + &Pools, + &PoolCount); + if (FAILED(Result)) { + return Result; + } + + if ((ULONG)ClassIndex >= PoolCount) { + return E_INVALIDARG; + } + + *PoolPointer = &Pools[ClassIndex]; + return S_OK; + } + + PayloadSize = Rtl->RoundUpPowerOfTwo64(RequiredPayloadBytes); + if (PayloadSize == 0) { + return E_INVALIDARG; + } + + return PerfectHashFileGetOversizeIocpBufferPool(Context, + PayloadSize, + PoolPointer); +} + +static +HRESULT +PerfectHashFileAcquireIocpBuffer( + _In_ PPERFECT_HASH_CONTEXT Context, + _In_ ULONGLONG RequiredPayloadBytes, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER_POOL *PoolPointer, + _Outptr_ PPERFECT_HASH_IOCP_BUFFER *BufferPointer + ) +{ + HRESULT Result; + PRTL Rtl; + PALLOCATOR Allocator; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(Context) || + !ARGUMENT_PRESENT(PoolPointer) || + !ARGUMENT_PRESENT(BufferPointer)) { + return E_POINTER; + } + + Rtl = Context->Rtl; + Allocator = Context->Allocator; + if (!Rtl || !Allocator) { + return E_UNEXPECTED; + } + + Result = PerfectHashFileGetIocpBufferPool(Context, + RequiredPayloadBytes, + &Pool); + if (FAILED(Result)) { + return Result; + } + + Result = PerfectHashIocpBufferPoolAcquire(Rtl, + Allocator, + Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + *PoolPointer = Pool; + *BufferPointer = Buffer; + return S_OK; +} + +static +HRESULT +PerfectHashFileReadOverlapped( + _In_ PPERFECT_HASH_FILE File, + _In_ ULONGLONG FileSize + ) +{ + HRESULT Result; + BOOL Success; + DWORD BytesRead; + ULONG LastError; + ULONGLONG Offset; + ULONGLONG Remaining; + PBYTE Dest; + PPERFECT_HASH_CONTEXT Context; + PPERFECT_HASH_TLS_CONTEXT TlsContext; + PPERFECT_HASH_IOCP_BUFFER_POOL Pool; + PPERFECT_HASH_IOCP_BUFFER Buffer; + + if (!ARGUMENT_PRESENT(File)) { + return E_POINTER; + } + + if (FileSize == 0) { + return PH_E_FILE_EMPTY; + } + + TlsContext = PerfectHashTlsGetContext(); + if (!TlsContext || !TlsContext->Context) { + return PH_E_NO_TLS_CONTEXT_SET; + } + + Context = TlsContext->Context; + Result = PerfectHashFileAcquireIocpBuffer(Context, + FileSize, + &Pool, + &Buffer); + if (FAILED(Result)) { + return Result; + } + + File->IocpBufferPool = Pool; + File->IocpBuffer = Buffer; + File->BaseAddress = PerfectHashIocpBufferPayload(Buffer); + + Offset = 0; + Remaining = FileSize; + Dest = (PBYTE)File->BaseAddress; + + while (Remaining) { + DWORD Chunk; + OVERLAPPED Overlapped; + + Chunk = (Remaining > MAXDWORD) ? MAXDWORD : (DWORD)Remaining; + ZeroStructInline(Overlapped); + Overlapped.Offset = (DWORD)Offset; + Overlapped.OffsetHigh = (DWORD)(Offset >> 32); + + BytesRead = 0; + Success = ReadFile(File->FileHandle, + Dest, + Chunk, + &BytesRead, + &Overlapped); + if (!Success) { + LastError = GetLastError(); + if (LastError == ERROR_IO_PENDING) { + Success = GetOverlappedResult(File->FileHandle, + &Overlapped, + &BytesRead, + TRUE); + if (!Success) { + SYS_ERROR(GetOverlappedResult); + Result = PH_E_SYSTEM_CALL_FAILED; + break; + } + } else { + SYS_ERROR(ReadFile); + Result = PH_E_SYSTEM_CALL_FAILED; + break; + } + } + + if (BytesRead != Chunk) { + Result = PH_E_INVARIANT_CHECK_FAILED; + break; + } + + Remaining -= BytesRead; + Offset += BytesRead; + Dest += BytesRead; + } + + if (FAILED(Result)) { + PerfectHashIocpBufferPoolRelease(Pool, Buffer); + File->IocpBufferPool = NULL; + File->IocpBuffer = NULL; + File->BaseAddress = NULL; + } + + return Result; +} + PERFECT_HASH_FILE_INITIALIZE PerfectHashFileInitialize; @@ -356,6 +808,8 @@ Return Value: } File->State.IsReadOnly = TRUE; + File->FileCreateFlags.AsULong = 0; + File->FileCreateFlags.SkipMapping = FileLoadFlags.SkipMapping; // // Add a reference to the source path. @@ -420,15 +874,34 @@ Return Value: EndOfFilePointer->QuadPart = File->FileInfo.EndOfFile.QuadPart; } - // - // Map the file into memory. - // + if (FileLoadFlags.SkipMapping) { - Result = File->Vtbl->Map(File); + // + // Load the file data into an overlapped buffer. + // - if (FAILED(Result)) { - PH_ERROR(PerfectHashFileMap, Result); - goto Error; + Result = PerfectHashFileReadOverlapped( + File, + (ULONGLONG)File->FileInfo.EndOfFile.QuadPart + ); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileReadOverlapped, Result); + goto Error; + } + + } else { + + // + // Map the file into memory. + // + + Result = File->Vtbl->Map(File); + + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileMap, Result); + goto Error; + } } // @@ -929,6 +1402,14 @@ Return Value: } } + if (File->IocpBufferPool && File->IocpBuffer) { + PerfectHashIocpBufferPoolRelease(File->IocpBufferPool, + File->IocpBuffer); + File->IocpBufferPool = NULL; + File->IocpBuffer = NULL; + File->BaseAddress = NULL; + } + // // If the end of file is non-zero, truncate the file. Otherwise, if it's // zero, treat this as an indication that the file should be deleted. diff --git a/src/PerfectHash/PerfectHashKeysLoad.c b/src/PerfectHash/PerfectHashKeysLoad.c index df9b9e4d..5aca7bfd 100644 --- a/src/PerfectHash/PerfectHashKeysLoad.c +++ b/src/PerfectHash/PerfectHashKeysLoad.c @@ -231,6 +231,10 @@ Return Value: KeysLoadFlags.TryLargePagesForKeysData ); + if (KeysLoadFlags.UseOverlappedIo) { + FileLoadFlags.SkipMapping = TRUE; + } + Result = File->Vtbl->Load(File, Path, &EndOfFile, &FileLoadFlags); if (FAILED(Result)) { diff --git a/src/PerfectHash/PerfectHashServer.c b/src/PerfectHash/PerfectHashServer.c index 6a85b41a..e27e4872 100644 --- a/src/PerfectHash/PerfectHashServer.c +++ b/src/PerfectHash/PerfectHashServer.c @@ -341,6 +341,7 @@ Return Value: Server->Flags.EndpointAllocated = FALSE; Server->Flags.Verbose = FALSE; Server->Flags.NoFileIo = FALSE; + Server->Flags.IocpBufferGuardPages = FALSE; Server->State.Initialized = TRUE; return S_OK; @@ -786,6 +787,50 @@ PerfectHashServerGetNoFileIo( return S_OK; } +PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerSetIocpBufferGuardPages; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerSetIocpBufferGuardPages( + PPERFECT_HASH_SERVER Server, + BOOLEAN GuardPages + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (Server->State.Running) { + return E_UNEXPECTED; + } + + Server->Flags.IocpBufferGuardPages = GuardPages ? 1 : 0; + return S_OK; +} + +PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerGetIocpBufferGuardPages; + +_Use_decl_annotations_ +HRESULT +PerfectHashServerGetIocpBufferGuardPages( + PPERFECT_HASH_SERVER Server, + PBOOLEAN GuardPages + ) +{ + if (!ARGUMENT_PRESENT(Server)) { + return E_POINTER; + } + + if (!ARGUMENT_PRESENT(GuardPages)) { + return E_POINTER; + } + + *GuardPages = (Server->Flags.IocpBufferGuardPages != 0); + return S_OK; +} + PERFECT_HASH_SERVER_START PerfectHashServerStart; _Use_decl_annotations_ @@ -2014,6 +2059,11 @@ PerfectHashServerBulkCreateWorkItemCallback( goto Complete; } + if (Request && Request->Server && + Request->Server->Flags.IocpBufferGuardPages) { + SetContextUseIocpBufferGuardPages(Context); + } + if (Request->CommandLineBuffer) { Context->CommandLineW = Request->CommandLineBuffer; } @@ -2963,6 +3013,10 @@ PerfectHashServerDispatchTableCreateRequest( goto Error; } + if (Server->Flags.IocpBufferGuardPages) { + SetContextUseIocpBufferGuardPages(Context); + } + if (CommandLine) { CommandLineChars = (ULONG)wcslen(CommandLine); if (CommandLineChars > ((ULONG_MAX / sizeof(WCHAR)) - 1)) { diff --git a/src/PerfectHash/PerfectHashServer.h b/src/PerfectHash/PerfectHashServer.h index 38f314fc..d5ec26ad 100644 --- a/src/PerfectHash/PerfectHashServer.h +++ b/src/PerfectHash/PerfectHashServer.h @@ -45,8 +45,9 @@ typedef union _PERFECT_HASH_SERVER_FLAGS { ULONG EndpointAllocated:1; ULONG Verbose:1; ULONG NoFileIo:1; + ULONG IocpBufferGuardPages:1; - ULONG Unused:28; + ULONG Unused:27; }; LONG AsLong; ULONG AsULong; @@ -139,6 +140,10 @@ extern PERFECT_HASH_SERVER_SET_VERBOSE PerfectHashServerSetVerbose; extern PERFECT_HASH_SERVER_GET_VERBOSE PerfectHashServerGetVerbose; extern PERFECT_HASH_SERVER_SET_NO_FILE_IO PerfectHashServerSetNoFileIo; extern PERFECT_HASH_SERVER_GET_NO_FILE_IO PerfectHashServerGetNoFileIo; +extern PERFECT_HASH_SERVER_SET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerSetIocpBufferGuardPages; +extern PERFECT_HASH_SERVER_GET_IOCP_BUFFER_GUARD_PAGES + PerfectHashServerGetIocpBufferGuardPages; extern PERFECT_HASH_SERVER_START PerfectHashServerStart; extern PERFECT_HASH_SERVER_STOP PerfectHashServerStop; extern PERFECT_HASH_SERVER_WAIT PerfectHashServerWait; diff --git a/src/PerfectHashServerExe/PerfectHashServerExe.c b/src/PerfectHashServerExe/PerfectHashServerExe.c index 69e7edf8..fc7e9c08 100644 --- a/src/PerfectHashServerExe/PerfectHashServerExe.c +++ b/src/PerfectHashServerExe/PerfectHashServerExe.c @@ -57,7 +57,9 @@ typedef struct _PERFECT_HASH_SERVER_CLI_OPTIONS { BOOLEAN VerbosePresent; BOOLEAN NoFileIo; BOOLEAN NoFileIoPresent; - UCHAR Padding1[6]; + BOOLEAN IocpBufferGuardPages; + BOOLEAN IocpBufferGuardPagesPresent; + UCHAR Padding1[4]; } PERFECT_HASH_SERVER_CLI_OPTIONS; typedef PERFECT_HASH_SERVER_CLI_OPTIONS *PPERFECT_HASH_SERVER_CLI_OPTIONS; @@ -606,7 +608,8 @@ PrintUsage( { wprintf(L"Usage: PerfectHashServer [--IocpConcurrency=] " L"[--MaxThreads=] [--Numa=All|] [--Endpoint=] " - L"[--AllowRemote|--LocalOnly] [--Verbose] [--NoFileIo]\n"); + L"[--AllowRemote|--LocalOnly] [--Verbose] [--NoFileIo] " + L"[--IocpBufferGuardPages]\n"); wprintf(L" --IocpConcurrency= IOCP concurrency per NUMA node\n"); wprintf(L" --MaxConcurrency= Alias for --IocpConcurrency\n"); wprintf(L" --MaxThreads= Worker threads per NUMA node\n"); @@ -617,6 +620,7 @@ PrintUsage( wprintf(L" --LocalOnly Reject remote named pipe clients\n"); wprintf(L" --Verbose Enable per-request console output\n"); wprintf(L" --NoFileIo Disable file I/O for requests\n"); + wprintf(L" --IocpBufferGuardPages Enable guard pages for IOCP buffers\n"); } static @@ -732,6 +736,8 @@ ParseServerArgs( Options->VerbosePresent = FALSE; Options->NoFileIo = FALSE; Options->NoFileIoPresent = FALSE; + Options->IocpBufferGuardPages = FALSE; + Options->IocpBufferGuardPagesPresent = FALSE; for (Index = 1; Index < NumberOfArguments; Index++) { PCWSTR Arg = ArgvW[Index]; @@ -834,6 +840,12 @@ ParseServerArgs( continue; } + if (_wcsicmp(Arg, L"IocpBufferGuardPages") == 0) { + Options->IocpBufferGuardPages = TRUE; + Options->IocpBufferGuardPagesPresent = TRUE; + continue; + } + return PH_E_INVALID_COMMANDLINE_ARG; } @@ -996,6 +1008,16 @@ mainCRTStartup( } } + if (Options.IocpBufferGuardPagesPresent) { + Result = Server->Vtbl->SetIocpBufferGuardPages( + Server, + Options.IocpBufferGuardPages + ); + if (FAILED(Result)) { + goto Error; + } + } + GlobalServer = Server; SetConsoleCtrlHandler(PerfectHashServerConsoleCtrlHandler, TRUE); diff --git a/tests/PerfectHashIocpBufferPoolTests.cpp b/tests/PerfectHashIocpBufferPoolTests.cpp index 1d63e594..d17fc072 100644 --- a/tests/PerfectHashIocpBufferPoolTests.cpp +++ b/tests/PerfectHashIocpBufferPoolTests.cpp @@ -137,6 +137,34 @@ TEST_F(PerfectHashIocpBufferPoolTests, InitializeAcquireRelease) { PerfectHashIocpBufferPoolRundown(&rtl_, nullptr, &pool); } +TEST_F(PerfectHashIocpBufferPoolTests, GuardPagesFlagPropagates) { + PERFECT_HASH_IOCP_BUFFER_POOL pool; + std::memset(&pool, 0, sizeof(pool)); + + HRESULT result = PerfectHashIocpBufferPoolInitialize( + &rtl_, + &pool, + 4096, + 0, + PERFECT_HASH_IOCP_BUFFER_POOL_FLAG_GUARD_PAGES, + nullptr, + nullptr); + + ASSERT_GE(result, 0); + + PPERFECT_HASH_IOCP_BUFFER buffer = nullptr; + result = PerfectHashIocpBufferPoolAcquire(&rtl_, + nullptr, + &pool, + &buffer); + ASSERT_GE(result, 0); + ASSERT_NE(buffer, nullptr); + EXPECT_NE(buffer->Flags & PERFECT_HASH_IOCP_BUFFER_FLAG_GUARD_PAGES, 0u); + + PerfectHashIocpBufferPoolRelease(&pool, buffer); + PerfectHashIocpBufferPoolRundown(&rtl_, nullptr, &pool); +} + TEST_F(PerfectHashIocpBufferPoolTests, SizeClassHelpers) { EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 0), 0); EXPECT_EQ(PerfectHashIocpBufferGetClassIndex(&rtl_, 1), 0); From 7a5d2e977c06e3950eb680b0b1fdae638b0a7a53 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Tue, 24 Feb 2026 12:10:52 -0800 Subject: [PATCH 9/9] Update .md ledgers. --- IOCP-LOGS.md | 17 ++++ IOCP-PROMPT.md | 238 +++++++++++++++++++++---------------------------- IOCP-README.md | 72 +++++++++++++++ IOCP-TOOD.md | 149 +++++++++++++++++++++---------- 4 files changed, 293 insertions(+), 183 deletions(-) create mode 100644 IOCP-README.md diff --git a/IOCP-LOGS.md b/IOCP-LOGS.md index aaec4171..d1b4d792 100644 --- a/IOCP-LOGS.md +++ b/IOCP-LOGS.md @@ -1,5 +1,22 @@ # IOCP Logs +## 2026-02-24 +- Consolidated IOCP project ledgers for fresh-session handoff: `IOCP-LOGS.md`, `IOCP-TOOD.md`, `IOCP-PROMPT.md`, and new `IOCP-README.md`. +- Captured current implementation status: + - IOCP server/client architecture is additive (new executables, OG paths retained). + - Per-NUMA-node IOCP runtime exists (one port per node, manual worker threads, affinity wiring). + - Named-pipe protocol supports ping/shutdown/table-create/bulk-directory requests. + - Bulk-directory request model provides single-token completion for client wait semantics. + - CHM01 async path exists and is IOCP-driven, with per-file concurrency ramp controls. + - IOCP file I/O path includes overlapped writes and overlapped keys loads (OG mmap path preserved). + - IOCP buffer pooling moved toward lookaside-style size classes with guard-page toggle support. +- Captured key stability/perf findings to carry forward: + - Fixed multiple hangs/crashes caused by async accounting leaks, request lifetime bugs, and stale file-write byte tracking. + - Fixed `TpIsTimerSet` crash by guarding legacy threadpool callbacks when IOCP mode bypasses threadpool init. + - Full sys32 file-I/O IOCP runs complete; runtime is better than OG in some runs, but tuning is workload-sensitive. + - High-concurrency configurations can over-allocate memory; pool sizing policy requires further hardening. +- No source-code behavior changes were introduced in this pass beyond documentation updates. + ## 2026-01-19 - Added IOCP context/server/client interfaces and COM scaffolding. - Added NUMA-aware IOCP runtime skeleton (per-node IOCPs, worker threads with affinity). diff --git a/IOCP-PROMPT.md b/IOCP-PROMPT.md index 414c451b..b80f523a 100644 --- a/IOCP-PROMPT.md +++ b/IOCP-PROMPT.md @@ -1,135 +1,105 @@ -# IOCP Backend Status (PerfectHash) - -This prompt summarizes the current IOCP backend work for the perfecthash server/client, including what is implemented, recent fixes, and next steps. - -## Current State - -The IOCP backend exists as a parallel, additive path (does not replace existing executables). It provides: -- A NUMA-aware IOCP runtime with one completion port per node and manually created worker threads pinned to node affinity. -- A named-pipe server with an IOCP state machine for request/response. -- A client that submits requests and optionally waits on bulk completion tokens. -- A bulk-create directory request that fans out per-file work items to IOCP nodes. - -The core IOCP pipeline runs on Windows with IOCP worker threads (no Windows threadpools for IOCP workers). Per-file perfect hash work still uses legacy `PERFECT_HASH_CONTEXT` / table-create logic, including its internal threadpool usage. - -## Key Components / Files - -IOCP runtime + NUMA: -- `src/PerfectHash/PerfectHashContextIocp.c` -- `src/PerfectHash/PerfectHashContextIocp.h` - -Server / pipe IOCP: -- `src/PerfectHash/PerfectHashServer.c` -- `src/PerfectHash/PerfectHashServer.h` - -Client: -- `src/PerfectHash/PerfectHashClient.c` -- `src/PerfectHash/PerfectHashClient.h` - -Server exe: -- `src/PerfectHashServerExe/PerfectHashServerExe.c` - -Client exe: -- `src/PerfectHashClientExe/PerfectHashClientExe.c` - -Protocol constants and bulk result struct: -- `include/PerfectHash.h` - -Scripts: -- `scripts/iocp-smoke.ps1` (table create over IOCP) -- `scripts/stress-sys32-iocp.ps1` (bulk-create directory request) -- `scripts/stress-sys32.ps1` (baseline bulk create) - -Logs / TODOs: -- `IOCP-LOGS.md` -- `IOCP-TOOD.md` - -## Implemented Features (IOCP Backend) - -- NUMA-aware IOCP runtime: - - Enumerates NUMA nodes. - - Creates one IOCP per node. - - Spawns worker threads per node with group affinity. - - Configurable max concurrency and NUMA node mask. - - Implemented in `src/PerfectHash/PerfectHashContextIocp.c`. - -- Server named-pipe IOCP transport: - - Per-node pipe instances tied to the node IOCP. - - IOCP state machine for connect, read header/payload, write response. - - Local-only vs allow-remote configuration. - - Implemented in `src/PerfectHash/PerfectHashServer.c`. - -- Client request handling: - - Supports `--TableCreate`, `--BulkCreate`, `--BulkCreateDirectory`, `--Shutdown`. - - Waits on bulk-create completion tokens via event + mapping. - - Implemented in `src/PerfectHashClientExe/PerfectHashClientExe.c` and `src/PerfectHash/PerfectHashClient.c`. - -- Bulk-create directory request: - - Server walks a directory and queues per-file IOCP work items. - - Uses round-robin dispatch across NUMA nodes. - - Completion logic tracks per-node counts and outstanding totals to signal a single completion token. - - Implemented in `src/PerfectHash/PerfectHashServer.c`. - -## Recent Fixes / Diagnostics - -- Fixed a crash in IOCP bulk-create caused by `Context->CommandLineW` being NULL when `PrepareCHeaderFileChm01` emits the command line. - - Bulk request now keeps a copy of the command line and assigns it to each per-file context. - - Files: `src/PerfectHash/PerfectHashServer.c` - -- Fixed use-after-free in bulk work callback (request freed before work item cleanup). - - Files: `src/PerfectHash/PerfectHashServer.c` - -- Fixed `CommandLineToArgvW` lifetime for bulk directory requests: - - `ArgvW` is now retained for the request lifetime, and freed on completion. - - Files: `src/PerfectHash/PerfectHashServer.c` - -- Added richer crash logging for server process: - - Records exception address, module base, offset, and thread ID before minidump attempt. - - Files: `src/PerfectHashServerExe/PerfectHashServerExe.c` - - Use env var `PH_LOG_SERVER_CRASH=1` to enable. - - `PH_SERVER_MINIDUMP_FORCE_FALLBACK=1` forces fallback minidump behavior. - -- Adjusted IOCP stress script to accept success exit code: - - `PH_S_SERVER_BULK_CREATE_ALL_SUCCEEDED` (0x2004000F). - - Files: `scripts/stress-sys32-iocp.ps1` - -## What Works Now (Observed) - -- `scripts/iocp-smoke.ps1` succeeds (table create via IOCP server/client). -- `scripts/stress-sys32-iocp.ps1` succeeds on a small directory (single keys file) after crash fixes. - -## Known Gaps / Risks - -- IOCP bulk-create directory with full `..\perfecthash-keys\sys32` has not been rerun post-fix; expected to be next. -- Per-file work still uses legacy table-create which relies on its internal threadpool and console hooks; IOCP orchestration does not yet replace that internal compute pipeline. -- Backpressure/queueing policies are still “post everything immediately” (no high-water mark). -- Linux/`io_uring` path is not implemented; current IOCP design is Windows-only. - -## Next Steps (Plan) - -1. Run full IOCP stress: - - `scripts/stress-sys32-iocp.ps1` against `..\perfecthash-keys\sys32` with Release and desired concurrency. -2. Performance comparison: - - Compare baseline `scripts/stress-sys32.ps1` vs IOCP run (same hash/mask/concurrency). -3. Native IOCP pipeline: - - Replace legacy context delegation with explicit IOCP-driven phases (key load, graph solve, file work). - - Add proper queue/backpressure per request and per node. -4. Multi-node tracking polish: - - Expand per-node completion tracking if needed for more granular results. -5. Docs: - - Draft `IOCP-README.md` once behavior and protocol stabilize. - -## Suggested Commands - -Build: -- `cmake --build build-win --config Release --target PerfectHashServerExe PerfectHashClientExe` - -Smoke: -- `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\iocp-smoke.ps1 -BuildDir build-win -Config Release` - -IOCP stress (sys32): -- `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32-iocp.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` - -Baseline: -- `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` +# IOCP Backend Handoff Prompt (Start Here) + +Continue IOCP server/client development for PerfectHash with the current state below. + +## Goal + +Make bulk perfect-hash creation saturate CPU effectively by decoupling work from workers with a NUMA-aware IOCP pipeline, while keeping OG executables/paths intact. + +## What Is Implemented + +- Additive IOCP architecture (new server/client executables; OG untouched). +- Per-NUMA node IOCP runtime: + - one completion port per node + - manually created worker threads + - node/group affinity wiring. +- Named-pipe request/response state machine with: + - ping/pong readiness + - shutdown + - table-create and bulk-directory requests. +- Bulk directory request flow: + - server walks `.keys` files + - dispatches per-file work across configured nodes + - returns a single completion token (event/mapping) to client. +- CHM01 async state-machine path present with per-file ramp controls: + - `--InitialPerFileConcurrency` + - `--MaxPerFileConcurrency` + - `--IncreaseConcurrencyAfterMilliseconds`. +- IOCP file-work path has overlapped save I/O and keys-load overlapped reads. +- IOCP buffer pool infrastructure exists and is being reworked toward NUMA lookaside-like size classes with optional guard pages. + +## Important Fixes Already Landed + +- Lifetime fixes: + - retained command-line/argv buffers for bulk request lifetime + - fixed use-after-free in bulk callback path. +- Async accounting fixes: + - requeue failure now decrements outstanding + - graph-submit failure rollback fixed (`ActiveGraphs`, loop counters, cleanup). +- Crash fixes: + - guarded legacy threadpool callback usage (`TpIsTimerSet` crash source) + - fixed stale `NumberOfBytesWritten`/sizing bugs that produced oversized files + - fixed IOCP file sizing index bug (`FileId` vs `FileWorkId`). +- Added crash diagnostics/minidump improvements and server/client wait-for-ready behavior. + +## Observed Performance Snapshot (Recent) + +- On some file-I/O workloads, IOCP outperformed OG. +- On some no-file-I/O workloads, OG remained faster. +- High concurrency (`IocpConcurrency=64`) exposed memory pressure/over-allocation; pool policy still needs tuning. + +## Current Risks / Gaps + +- Buffer pool policy can overprovision at high concurrency. +- Need stricter invariants around async completion counters to prevent latent hangs. +- Large-payload file writes need explicit chunk/flush strategy validation. +- Need broader repeatable E2E coverage (test1/hard/sys32 subsets/full). + +## Next Execution Order + +1. Lock correctness first: + - counter invariants + - completion-once guarantees + - failure-path decrements. +2. Finish buffer-pool redesign: + - per-NUMA size-class global pools + - oversize reuse pools + - safe guarded-list rundown. +3. Harden overlapped file-I/O: + - large write chunking + - strict bounds/fail-fast behavior + - OG path unchanged. +4. Re-run standardized perf matrix and tune defaults. + +## Key Files + +- IOCP runtime: + - `src/PerfectHash/PerfectHashContextIocp.c` + - `src/PerfectHash/PerfectHashContextIocp.h` +- Server/client core: + - `src/PerfectHash/PerfectHashServer.c` + - `src/PerfectHash/PerfectHashClient.c` +- Async engine: + - `src/PerfectHash/PerfectHashAsync.c` + - `src/PerfectHash/Chm01Async.c` +- IOCP file work/buffer pool: + - `src/PerfectHash/Chm01FileWork.c` + - `src/PerfectHash/Chm01FileWorkIocp.c` + - `src/PerfectHash/PerfectHashIocpBufferPool.c` + - `src/PerfectHash/PerfectHashIocpBufferPool.h` +- Ledgers: + - `IOCP-LOGS.md` + - `IOCP-TOOD.md` + - `IOCP-README.md` + +## Quick Commands + +- Build: + - `cmake --build build-win --config Release --target PerfectHashServerExe PerfectHashClientExe PerfectHashBulkCreateExe` +- IOCP smoke: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\iocp-smoke.ps1 -BuildDir build-win -Config Release` +- IOCP stress: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32-iocp.ps1 -BuildDir build-win -Config Release -IocpConcurrency 32 -MaxThreads 64` +- OG stress: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` diff --git a/IOCP-README.md b/IOCP-README.md new file mode 100644 index 00000000..688ba7a4 --- /dev/null +++ b/IOCP-README.md @@ -0,0 +1,72 @@ +# IOCP PerfectHash Backend + +This document is the current high-level reference for the additive IOCP implementation. + +## Scope + +- New server/client path for table and bulk create requests. +- Keep OG `CreateExe` / `BulkCreateExe` behavior intact. +- Use manual Windows threads + `GetQueuedCompletionStatus()` loops. +- NUMA-aware design: one IOCP per node, affinity-aware workers. + +## Design Summary + +1. Client sends request over named pipe. +2. Server accepts request via IOCP pipe state machine. +3. For bulk-directory requests: + - enumerate `.keys` files + - dispatch per-file work items (round-robin across selected NUMA nodes) + - track per-request counters + - signal one completion token when all file work is complete. +4. Client waits on token and receives a final bulk status code. + +## Concurrency Controls + +- `--IocpConcurrency`: + - completion-port concurrency level (also aliasable from older `--MaxConcurrency` usage in some paths/scripts). +- `--MaxThreads`: + - max worker threads to create (default: `IocpConcurrency * 2`). +- Per-file async ramp: + - `--InitialPerFileConcurrency` + - `--MaxPerFileConcurrency` + - `--IncreaseConcurrencyAfterMilliseconds`. + +## File I/O Mode + +- IOCP path: + - overlapped writes for generated files + - overlapped keys reads + - pooled intermediate buffers. +- OG path: + - existing memory-mapped behavior preserved. + +## Buffering + +- Direction: NUMA-aware lookaside-style pooling. +- Current state: + - size-class pools (power-of-two) are in place and evolving + - oversize buffer handling exists and needs further hardening/tuning + - optional guard-page behavior is supported for safer debug/development runs. + +## Operational Flags + +- `--WaitForServer` and `--ConnectTimeout=` on client for robust startup coordination. +- `--Verbose` on server gates console output (silent by default). +- `--NoFileIo` supported for rapid compute-only stress loops. + +## Current Status + +- Core IOCP pipeline is functional. +- Small/medium dataset runs are stable. +- Full sys32 runs have succeeded in both `NoFileIo` and file-I/O modes under tested settings. +- Remaining work is mostly: + - correctness hardening for edge/failure paths + - buffer pool policy/memory scaling + - deeper performance tuning and regression automation. + +## See Also + +- Execution log: `IOCP-LOGS.md` +- Active backlog: `IOCP-TOOD.md` +- Session handoff prompt: `IOCP-PROMPT.md` + diff --git a/IOCP-TOOD.md b/IOCP-TOOD.md index 2cb70fa4..72405c14 100644 --- a/IOCP-TOOD.md +++ b/IOCP-TOOD.md @@ -1,51 +1,102 @@ # IOCP TODO -- Overlapped I/O plan (IOCP only; OG remains memory-mapped): - - Done: added `UseOverlappedIo` flag to `PERFECT_HASH_CONTEXT` (IOCP contexts set it). - - Done: added `PerfectHashFileCreateFlags.SkipMapping` so output files can be created with `FILE_FLAG_OVERLAPPED` and no mapping. -- Done: rebuild IOCP file-work buffers as NUMA size-class pools (lookaside-style). - - Pools keyed by payload size class (power-of-two); 4KB–16MB classes. - - Dynamic oversize pools keyed by next power-of-two above max size class. - - Each pool has an `SLIST_HEADER` free list and a `GUARDED_LIST` of all buffers for rundown. - - New buffer header: `SLIST_ENTRY` + `LIST_ENTRY` + `OwnerPool` + `PayloadSize` + - `AllocationSize` + `PayloadOffset` + `BytesWritten` + flags. - - Base address for file work writes points *after* the header; sizing still uses EOF multipliers + trap-on-overrun. - - Allocate on demand; buffers pushed to pool free list and guarded list for teardown. - - One-off allocations removed; buffers reused via size-class or oversize pools. - - Done: add a guard-page allocation flag to toggle `RtlCreateBuffer()` vs raw allocation. - - Output file pipeline (IOCP): - - Pop a buffer for each file work save stage. - - Set `File->BaseAddress` to buffer base (after header) to reuse existing `OUTPUT_*` macros. - - Track `File->NumberOfBytesWritten` from `Output` - `BaseAddress`. - - Done: issue `WriteFile` with IOCP work item and handle completion in IOCP callback (no `GetOverlappedResult` waits). - - Done: `UnmapFileChm01()` no-ops in overlapped mode; `CloseFileChm01()` uses `File->NumberOfBytesWritten` (no mapping). - - Input file pipeline (IOCP): - - Done: replace memory-mapped keys file reads with overlapped `ReadFile` into a keys buffer (reuse size-class pool). - - Done: parse keys from buffer; keep mapped path for OG. -- Add lightweight stats (debug-only): pool exhaustion count, max buffers in use, per-bucket hits; optional ETW later. -- Add unit tests for IOCP file-write completion handling (buffer return, event signal, error path). -- Audit async accounting for leaks: - - Ensure `PerfectHashAsyncIocpCompletionCallback` decrements outstanding if requeue fails. - - Add cleanup in `Chm01AsyncDispatchGraphWork` when `PerfectHashAsyncSubmit` fails (decrement `ActiveGraphs`/`RemainingSolverLoops`, free `GraphWork`). - - Add logging/asserts for `Job->ActiveGraphs`, `Context->RemainingSolverLoops`, and `Job->Async.Outstanding` to catch leaks. - -- Get a sys32 `--NoFileIo` run to complete (Release, max concurrency); if it hangs, capture bulk-count logs and identify the stuck state. -- Diagnose 1000-file sys32 subset hang with `--IocpConcurrency=32 --MaxThreads=64` (Release, `--NoFileIo`); identify which work items are stuck and why. -- Re-run sys32 with file I/O to `G:\\Scratch` once `--NoFileIo` completes; record timing and correctness. -- Decide how to guard against stale PCH/obj after `PERFECT_HASH_CONTEXT` layout changes (force clean or touch header). -- Track down `FlushConsoleInputBuffer` failures and fully disable console work for IOCP contexts if still triggered. -- Validate access-denied fallback for per-file context threadpool minimum failures at higher concurrency. -- Recheck named-pipe endpoint handling if `PerfectHashServer-StressSys32` continues to fail. -- Decide whether BulkCreateDirectory should accept a single-directory short form or keep output dir required. -- Exercise IOCP file work dispatch path (bulk create) to confirm outstanding event signaling and non-threadpool file work callbacks. -- Identify root cause of remaining IOCP bulk-create failures (`E_UNEXPECTED` on `shell32-13803.keys` / `CoreUIComponents-7995.keys`) if they recur. -- Investigate unexpected ~1500 threadpool worker threads (`ZwWaitForWorkViaWorkerFactory`) observed during IOCP server runs; identify source (RPC/TP APIs) and whether it contributes to hangs. -- Re-run a larger file-I/O set to confirm the `PerfectHashFileExtend` skip-mapping fix and one-off buffer fallback eliminate `SetEndOfFile` 1224 and disk-full pre-size failures; decide whether context-file skip is still required. -- Decide whether to keep the one-off buffer fallback or implement a pool resize/rehash strategy for oversized file-work payloads. -- Decide whether `PerfectHashClientExe` should wait on bulk-create tokens for `BulkCreate=` requests now that the server routes them through the bulk-directory path. -- Choose a node-selection policy for single table-create server requests (round-robin vs node 0 vs affinity). -- Add IOCP work item queueing/state to drive per-request pipelines. -- Integrate per-request concurrency caps and queueing policies. -- Validate per-file concurrency ramp behavior (`--InitialPerFileConcurrency`, `--MaxPerFileConcurrency`, `--IncreaseConcurrencyAfterMilliseconds`) on `hard` and sys32 subsets; confirm no early failure. -- Capture and diagnose `PerfectHashServer.exe` `0xC0000005` crash on `hard` (Release, `--NoFileIo`, concurrency 4); set `PH_SERVER_CRASH_DIR` for minidumps if needed. -- Draft `IOCP-README.md` once protocol and dispatch behavior settle. +This is the prioritized, session-ready backlog for the IOCP server/client work. + +## P0: Correctness + Completion Guarantees + +- [ ] Re-verify bulk completion accounting under stress (single token must always signal exactly once): + - `OutstandingWorkItems` + - `PendingCompletions` + - per-node decrement paths + - final request completion transition +- [ ] Add/assert invariants in debug builds for async counters: + - `Job->ActiveGraphs` + - `Context->RemainingSolverLoops` + - `Job->Async.Outstanding` +- [ ] Audit all failure/requeue paths again to ensure every submitted work item has exactly one completion/decrement. +- [ ] Add targeted tests for: + - `PerfectHashAsyncRequeueWork()` failure path + - graph submit failure rollback + - bulk finalization when one node completes last. +- [ ] Confirm server/client connection lifecycle robustness: + - `--WaitForServer` + - `--ConnectTimeout=` + - ping/pong readiness before bulk submission. + +## P0: IOCP-Only Runtime Hygiene + +- [ ] Ensure no legacy threadpool work submission is used by IOCP backend paths (except unavoidable OS-internal TP activity). +- [ ] Keep server silent by default; output only when `--Verbose` is specified. +- [ ] Re-verify `--NoFileIo` on server for fast stress loops and CI-style regression runs. + +## P1: Buffer Pool Redesign Completion (Lookaside/NUMA Style) + +- [ ] Finalize transition to per-NUMA global size-class pools (4KB..16MB default classes). +- [ ] Ensure oversize buffers are pooled/reusable (not leaked one-offs), with guarded-list ownership for teardown. +- [ ] Validate size-class mapping and payload offsets: + - header precedes payload + - `File->BaseAddress` points to payload only + - overrun should fail fast (`PH_RAISE`). +- [ ] Add pool diagnostics (debug-only): + - allocation count + - in-use count + - exhaustion count + - per-class hit/miss. +- [ ] Harden teardown/rundown: + - safe list flush + - safe free after server stop + - no outstanding buffer references. + +## P1: File I/O Pipeline Hardening (Overlapped) + +- [ ] Audit all CHM01 save callbacks for large-write safety with bounded, chunked writes. +- [ ] Add explicit handling for payloads that exceed current buffer capacity: + - predictable flush-and-continue logic + - no undefined pointer arithmetic on overflow. +- [ ] Keep OG memory-mapped path untouched and validated. +- [ ] Re-check ReFS/ADS-related edge cases (table-size stream behavior) and preserve `ADS.md` notes. + +## P1: Tests (Unit + E2E) + +- [ ] Expand unit tests around IOCP buffer pool: + - guard-page mode on/off + - oversize class reuse + - multi-thread pop/push behavior. +- [ ] Add IOCP file-write completion unit tests: + - success path (event signal + buffer return) + - error path (propagated HRESULT + release semantics). +- [ ] Add repeatable E2E matrix scripts: + - `test1`, `hard`, `sys32-200`, `sys32-1000` + - `NoFileIo` and file-I/O modes + - standard ramp presets. + +## P2: Performance Characterization + Tuning + +- [ ] Re-run and capture normalized OG vs IOCP timings (same algo/flags/output drive): + - `Mulshrolate4RX` + - concurrency sets: `4`, `32`, `64`. +- [ ] Sweep per-file ramp knobs: + - `--InitialPerFileConcurrency` + - `--MaxPerFileConcurrency` + - `--IncreaseConcurrencyAfterMilliseconds` +- [ ] Profile system-vs-user overhead and memory footprint at high concurrency. +- [ ] Tune default knobs for balanced throughput and memory use. + +## P2: API/CLI Polish + +- [ ] Keep/confirm naming split: + - `--IocpConcurrency` + - `--MaxThreads` (default `IocpConcurrency * 2`) + - `--MaxPerFileConcurrency` (must be `<= IocpConcurrency`). +- [ ] Validate NUMA targeting options and document examples (all nodes, single node, mask). +- [ ] Decide whether `BulkCreate=` request form should also block on token like `BulkCreateDirectory`. + +## Regression Commands (Baseline) + +- IOCP smoke: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\iocp-smoke.ps1 -BuildDir build-win -Config Release` +- IOCP stress (NoFileIo): + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32-iocp.ps1 -BuildDir build-win -Config Release -IocpConcurrency 32 -MaxThreads 64 -NoFileIo` +- OG stress: + - `powershell -NoProfile -ExecutionPolicy Bypass -File scripts\stress-sys32.ps1 -BuildDir build-win -Config Release -MaximumConcurrency 32` +