diff --git a/cmake/libmultiprocess.cmake b/cmake/libmultiprocess.cmake index 027109a90511..1ba656972366 100644 --- a/cmake/libmultiprocess.cmake +++ b/cmake/libmultiprocess.cmake @@ -29,10 +29,4 @@ function(add_libmultiprocess subdir) # Add tests to "all" target so ctest can run them set_target_properties(mptest PROPERTIES EXCLUDE_FROM_ALL OFF) endif() - # Exclude examples from compilation database, because the examples are not - # built by default, and they contain generated c++ code. Without this - # exclusion, tools like clang-tidy and IWYU that make use of compilation - # database would complain that the generated c++ source files do not exist. An - # alternate fix could build "mpexamples" by default like "mptests" above. - set_target_properties(mpcalculator mpprinter mpexample PROPERTIES EXPORT_COMPILE_COMMANDS OFF) endfunction() diff --git a/doc/developer-notes.md b/doc/developer-notes.md index d838ddff4b04..a05152396835 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1136,9 +1136,15 @@ To update the subtree: ```sh git fetch libmultiprocess -git subtree pull --prefix=src/ipc/libmultiprocess libmultiprocess master --squash +git subtree pull --prefix=src/ipc/libmultiprocess libmultiprocess lib --squash ``` +The `lib` branch is published by libmultiprocess upstream and contains +only the files needed to do a full build (regular and via `depends`). It +is produced from `master` with `git subtree split --prefix=lib`; CI +scripts, documentation, and other upstream-only files are intentionally +excluded from the subtree. + The ultimate upstream of the few externally managed subtrees are: - src/leveldb diff --git a/src/ipc/libmultiprocess/.clang-tidy b/src/ipc/libmultiprocess/.clang-tidy deleted file mode 100644 index 9a2afcc5248c..000000000000 --- a/src/ipc/libmultiprocess/.clang-tidy +++ /dev/null @@ -1,40 +0,0 @@ -Checks: ' --*, -bugprone-argument-comment, -bugprone-move-forwarding-reference, -bugprone-string-constructor, -bugprone-use-after-move, -bugprone-lambda-function-name, -bugprone-unhandled-self-assignment, -misc-unused-using-decls, -misc-no-recursion, -modernize-deprecated-headers, -modernize-use-default-member-init, -modernize-use-emplace, -modernize-use-equals-default, -modernize-use-noexcept, -modernize-use-nullptr, -modernize-use-starts-ends-with, -performance-*, --performance-avoid-endl, --performance-enum-size, --performance-inefficient-string-concatenation, --performance-no-int-to-ptr, --performance-noexcept-move-constructor, --performance-unnecessary-value-param, -readability-const-return-type, -readability-redundant-declaration, -readability-redundant-string-init, -clang-analyzer-core.*, --clang-analyzer-core.UndefinedBinaryOperatorResult, -clang-analyzer-optin.core.*, -' -HeaderFilterRegex: '.' -WarningsAsErrors: '*' -CheckOptions: - - key: modernize-deprecated-headers.CheckHeaderFile - value: false - - key: performance-move-const-arg.CheckTriviallyCopyableMove - value: false - - key: bugprone-unhandled-self-assignment.WarnOnlyIfThisHasSuspiciousField - value: false diff --git a/src/ipc/libmultiprocess/.github/workflows/bitcoin-core-ci.yml b/src/ipc/libmultiprocess/.github/workflows/bitcoin-core-ci.yml deleted file mode 100644 index e6ac83f0bd45..000000000000 --- a/src/ipc/libmultiprocess/.github/workflows/bitcoin-core-ci.yml +++ /dev/null @@ -1,300 +0,0 @@ -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/license/mit. - -# Test libmultiprocess inside Bitcoin Core by replacing the subtree copy -# with the version from this PR, then building and running IPC-related -# unit & functional tests. - -name: Bitcoin Core CI - -on: - push: - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - BITCOIN_REPO: bitcoin/bitcoin - LLVM_VERSION: 22 - LIBCXX_DIR: /tmp/libcxx-build/ - -jobs: - bitcoin-core: - name: ${{ matrix.name }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - - strategy: - fail-fast: false - matrix: - include: - - name: 'ASan + UBSan' - unit_test_runs: 15 - functional_test_runs: 20 - nproc_multiplier: 2 - functional_timeout_factor: 40 - runner: ubuntu-24.04 - apt-llvm: true - packages: >- - ccache - clang-22 - llvm-22 - libclang-rt-22-dev - libevent-dev - libboost-dev - libsqlite3-dev - libcapnp-dev - capnproto - ninja-build - pkgconf - python3-pip - pip-packages: --break-system-packages pycapnp - cmake-args: |- - -DSANITIZERS=address,float-divide-by-zero,integer,undefined - -DCMAKE_C_COMPILER=clang - -DCMAKE_CXX_COMPILER=clang++ - -DCMAKE_C_FLAGS=-ftrivial-auto-var-init=pattern - -DCMAKE_CXX_FLAGS=-ftrivial-auto-var-init=pattern - - - name: 'macOS' - unit_test_runs: 50 - functional_test_runs: 20 - nproc_multiplier: 2 - functional_timeout_factor: 40 - runner: macos-15 - brew-packages: ccache capnp boost libevent sqlite pkgconf ninja - pip-packages: --break-system-packages pycapnp - cmake-args: |- - -DREDUCE_EXPORTS=ON - - env: - CCACHE_MAXSIZE: 400M - CCACHE_DIR: ${{ github.workspace }}/.ccache - - steps: - - name: Checkout Bitcoin Core - uses: actions/checkout@v4 - with: - repository: ${{ env.BITCOIN_REPO }} - fetch-depth: 1 - - - name: Checkout libmultiprocess - uses: actions/checkout@v4 - with: - path: _libmultiprocess - - - name: Replace libmultiprocess subtree - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh replace_subtree - - - name: Add LLVM apt repository - if: matrix.apt-llvm - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh add_llvm_apt_repository - - - name: Install APT packages - if: matrix.packages - run: _libmultiprocess/ci/scripts/ci_helpers.sh install_apt_packages ${{ matrix.packages }} - - - name: Configure LLVM alternatives - if: matrix.packages - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh install_llvm_alternatives - - - name: Install Homebrew packages - if: matrix.brew-packages - run: _libmultiprocess/ci/scripts/ci_helpers.sh install_homebrew_packages ${{ matrix.brew-packages }} - - - name: Install pip packages - if: matrix.pip-packages - run: _libmultiprocess/ci/scripts/ci_helpers.sh install_pip_packages ${{ matrix.pip-packages }} - - - name: Determine parallelism - run: _libmultiprocess/ci/scripts/ci_helpers.sh determine_parallelism "${{ matrix.nproc_multiplier }}" - - - name: Restore ccache - id: ccache-restore - uses: actions/cache/restore@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} - restore-keys: | - ccache-${{ matrix.name }}-${{ github.ref }}- - ccache-${{ matrix.name }}- - - - name: Reset ccache stats - if: matrix.packages || matrix.brew-packages - run: _libmultiprocess/ci/scripts/ci_helpers.sh reset_ccache_stats - - - name: CMake configure - env: - BITCOIN_CORE_CMAKE_ARGS: ${{ matrix.cmake-args }} - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh configure_bitcoin_core - - - name: Build - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_bitcoin_core - - - name: Show ccache stats - if: matrix.packages || matrix.brew-packages - run: _libmultiprocess/ci/scripts/ci_helpers.sh show_ccache_stats - - - name: Run IPC unit tests - env: - ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 - LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan - UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1 - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_unit_tests "${{ matrix.unit_test_runs }}" - - - name: Run IPC functional tests - env: - ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 - LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan - UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1 - CI_FAILFAST_TEST_LEAVE_DANGLING: 1 - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_functional_tests "${{ matrix.functional_test_runs }}" "${{ matrix.functional_timeout_factor }}" - - - name: Save ccache - uses: actions/cache/save@v4 - if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true' - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} - - bitcoin-core-tsan: - name: ${{ matrix.name }} - runs-on: ubuntu-24.04 - timeout-minutes: 180 - - strategy: - matrix: - include: - - name: TSan - unit_test_runs: 8 - functional_test_runs: 25 - nproc_multiplier: 2 - functional_timeout_factor: 40 - - env: - CCACHE_MAXSIZE: 400M - CCACHE_DIR: ${{ github.workspace }}/.ccache - LIBCXX_FLAGS: >- - -fsanitize=thread - -nostdinc++ - -nostdlib++ - -isystem /tmp/libcxx-build/include/c++/v1 - -L/tmp/libcxx-build/lib - -Wl,-rpath,/tmp/libcxx-build/lib - -lc++ - -lc++abi - -lpthread - -Wno-unused-command-line-argument - TSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1 - - steps: - - name: Checkout Bitcoin Core - uses: actions/checkout@v4 - with: - repository: ${{ env.BITCOIN_REPO }} - fetch-depth: 1 - - - name: Checkout libmultiprocess - uses: actions/checkout@v4 - with: - path: _libmultiprocess - - - name: Add LLVM apt repository - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh add_llvm_apt_repository - - - name: Install packages - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh install_tsan_packages - - - name: Determine parallelism - run: _libmultiprocess/ci/scripts/ci_helpers.sh determine_parallelism "${{ matrix.nproc_multiplier }}" - - - name: Restore instrumented libc++ cache - id: libcxx-cache - uses: actions/cache@v4 - with: - path: ${{ env.LIBCXX_DIR }} - key: libcxx-Thread-llvmorg-${{ env.LLVM_VERSION }}.1.0 - - - name: Build instrumented libc++ - if: steps.libcxx-cache.outputs.cache-hit != 'true' - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_instrumented_libcxx - - - name: Determine host - id: host - run: _libmultiprocess/ci/scripts/ci_helpers.sh determine_host - - - name: Restore depends cache - id: depends-cache - uses: actions/cache/restore@v4 - with: - path: | - depends/built - depends/${{ steps.host.outputs.host }} - key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }} - - - name: Build depends (stage 1, without IPC) - if: steps.depends-cache.outputs.cache-hit != 'true' - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_depends_without_ipc - - - name: Save depends cache - uses: actions/cache/save@v4 - if: steps.depends-cache.outputs.cache-hit != 'true' - with: - path: | - depends/built - depends/${{ steps.host.outputs.host }} - key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }} - - - name: Replace libmultiprocess subtree - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh replace_subtree - - - name: Build depends (stage 2, IPC packages including libmultiprocess) - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_depends_with_ipc - - - name: Restore ccache - id: ccache-restore - uses: actions/cache/restore@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-TSan-${{ github.ref }}-${{ github.sha }} - restore-keys: | - ccache-TSan-${{ github.ref }}- - ccache-TSan- - - - name: Reset ccache stats - run: _libmultiprocess/ci/scripts/ci_helpers.sh reset_ccache_stats - - - name: CMake configure - env: - BITCOIN_CORE_CMAKE_ARGS: |- - -DSANITIZERS=thread - -DAPPEND_CPPFLAGS=-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES - -DCMAKE_TOOLCHAIN_FILE=depends/${{ steps.host.outputs.host }}/toolchain.cmake - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh configure_bitcoin_core - - - name: Build - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh build_bitcoin_core - - - name: Show ccache stats - run: _libmultiprocess/ci/scripts/ci_helpers.sh show_ccache_stats - - - name: Run IPC unit tests - env: - LD_LIBRARY_PATH: depends/${{ steps.host.outputs.host }}/lib - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_unit_tests "${{ matrix.unit_test_runs }}" - - - name: Run IPC functional tests - env: - LD_LIBRARY_PATH: depends/${{ steps.host.outputs.host }}/lib - CI_FAILFAST_TEST_LEAVE_DANGLING: 1 - run: _libmultiprocess/ci/scripts/bitcoin_core_ci.sh run_ipc_functional_tests "${{ matrix.functional_test_runs }}" "${{ matrix.functional_timeout_factor }}" - - - name: Save ccache - uses: actions/cache/save@v4 - if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true' - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-TSan-${{ github.ref }}-${{ github.sha }} diff --git a/src/ipc/libmultiprocess/.github/workflows/ci.yml b/src/ipc/libmultiprocess/.github/workflows/ci.yml deleted file mode 100644 index ffaca1cac93c..000000000000 --- a/src/ipc/libmultiprocess/.github/workflows/ci.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: CI - -on: - push: - pull_request: - -jobs: - build-netbsd: - runs-on: ubuntu-latest - name: build • netbsd ${{ matrix.release }} - defaults: - run: - shell: netbsd {0} - strategy: - fail-fast: false - matrix: - # Test all supported releases. - # See https://www.netbsd.org/releases/. - include: - - release: 9.4 - capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0 -DKJ_USE_KQUEUE=0"' - - release: 10.1 - capnproto-cppflags: 'CPPFLAGS="-DKJ_NO_EXCEPTIONS=0"' - steps: - - uses: actions/checkout@v6 - - - name: Start NetBSD VM - uses: vmactions/netbsd-vm@v1 - with: - release: ${{ matrix.release }} - # The installed compiler version must match the CXX variable - # defined in `ci/configs/netbsd.bash`. - prepare: | - pkg_add cmake ninja-build gcc14 - # capnproto prerequisites. - # See the following "Install capnproto" step. - run: | - set -e - pkg_add digest libtool-base mktools pkgconf cwrappers - pkg_admin -K /usr/pkg/pkgdb fetch-pkg-vulnerabilities - cd /usr - cvs -danoncvs@anoncvs.NetBSD.org:/cvsroot checkout -P \ - pkgsrc/devel/capnproto \ - pkgsrc/devel/libtool-base \ - pkgsrc/devel/pkgconf \ - pkgsrc/devel/zlib \ - `# gcc15 is referenced here because the pkgsrc framework requires lang/gcc15/version.mk to exist` \ - `# during the "make install" step below, even though we compile our project with gcc14.` \ - pkgsrc/lang/gcc15 \ - pkgsrc/mk \ - pkgsrc/pkgtools \ - pkgsrc/security/openssl \ - pkgsrc/sysutils/install-sh/files - sync: 'rsync' - copyback: false - - - name: Install capnproto - run: | - cd /usr/pkgsrc/devel/capnproto/ - unset PKG_PATH - make ${{ matrix.capnproto-cppflags }} install - - - name: Run CI script - run: | - cd ${{ github.workspace }} - CI_CONFIG="ci/configs/netbsd.bash" bash ci/scripts/ci.sh - - build-openbsd: - runs-on: ubuntu-latest - name: build • openbsd - defaults: - run: - shell: openbsd {0} - steps: - - uses: actions/checkout@v5 - - - name: Start OpenBSD VM - uses: vmactions/openbsd-vm@v1 - with: - prepare: | - pkg_add -v cmake ninja bash capnproto - sync: 'rsync' - copyback: false - - - name: Run CI script - run: | - cd ${{ github.workspace }} - CI_CONFIG="ci/configs/openbsd.bash" bash ci/scripts/ci.sh - - build-freebsd: - runs-on: ubuntu-latest - name: build • freebsd - defaults: - run: - shell: freebsd {0} - steps: - - uses: actions/checkout@v5 - - - name: Start FreeBSD VM - uses: vmactions/freebsd-vm@v1 - with: - prepare: | - pkg install -y cmake ninja bash capnproto - sync: 'rsync' - copyback: false - - - name: Run CI script - run: | - cd ${{ github.workspace }} - CI_CONFIG="ci/configs/freebsd.bash" bash ci/scripts/ci.sh - - build-macos: - runs-on: macos-latest - name: build • macos - - steps: - - uses: actions/checkout@v5 - - - name: Install dependencies - env: - HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 - run: | - brew install --quiet ninja capnp llvm - - - name: Run CI script - run: | - export PATH="$(brew --prefix llvm)/bin:$PATH" - CI_CONFIG="ci/configs/macos.bash" bash ci/scripts/ci.sh - - build: - runs-on: ubuntu-latest - - env: - NIXPKGS_CHANNEL: nixos-25.05 - NIX_EXTRA_CONFIG: | - keep-env-derivations = true - keep-outputs = true - NIX_EXTRA_CONFIG_ACT: | - sandbox = false - filter-syscalls = false - - strategy: - fail-fast: false - matrix: - config: [default, llvm, gnu32, sanitize, olddeps] - - name: build • ${{ matrix.config }} - - steps: - - uses: actions/checkout@v5 - - - name: Determine CI configuration - id: config - env: - CI_CONFIG: ci/configs/${{ matrix.config }}.bash - run: ci/scripts/config.sh - - - name: Install Nix - uses: cachix/install-nix-action@v31 # 2025-05-27, from https://github.com/cachix/install-nix-action/tags - with: - nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/${{ steps.config.outputs.nixpkgs_rev }}.tar.gz - # Act executes inside an unprivileged container (Docker or Podman), - # so KVM support isn't available. - enable_kvm: "${{ github.actor != 'nektos/act' }}" - extra_nix_config: | - ${{ env.NIX_EXTRA_CONFIG }} - ${{ github.actor == 'nektos/act' && env.NIX_EXTRA_CONFIG_ACT || '' }} - - - name: Cache Nix store - if: steps.config.outputs.cache_nix_store == 'true' - uses: nix-community/cache-nix-action@v7 - with: - primary-key: nix-${{ runner.os }}-${{ matrix.config }}-${{ steps.config.outputs.nixpkgs_rev }}-${{ hashFiles('shell.nix', 'ci/patches/*.patch', format('ci/configs/{0}.bash', matrix.config)) }} - restore-prefixes-first-match: | - nix-${{ runner.os }}-${{ matrix.config }}-${{ steps.config.outputs.nixpkgs_rev }}- - nix-${{ runner.os }}-${{ matrix.config }}- - nix-${{ runner.os }}- - gc-max-store-size-linux: 10G - - - name: Run CI script - env: - CI_CONFIG: ci/configs/${{ matrix.config }}.bash - run: ci/scripts/run.sh diff --git a/src/ipc/libmultiprocess/.gitignore b/src/ipc/libmultiprocess/.gitignore deleted file mode 100644 index 30b4b7864881..000000000000 --- a/src/ipc/libmultiprocess/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# CMake artifacts -/*build* - -# Git artifacts -*.patch diff --git a/src/ipc/libmultiprocess/CMakeLists.txt b/src/ipc/libmultiprocess/CMakeLists.txt index a36023b18101..99b992e7cc57 100644 --- a/src/ipc/libmultiprocess/CMakeLists.txt +++ b/src/ipc/libmultiprocess/CMakeLists.txt @@ -13,7 +13,14 @@ endif() include("cmake/compat_find.cmake") find_package(Threads REQUIRED) -find_package(CapnProto 0.7 QUIET NO_MODULE) +# When this file is processed via add_subdirectory() from the standalone +# wrapper, find_package(CapnProto) has already been called there to make the +# IMPORTED targets visible to the example/ sibling directory. Re-running it +# here would fail on platforms whose CapnProto cmake config registers global +# targets (e.g. NetBSD pkgsrc), so skip it when CapnProto is already found. +if(NOT CapnProto_FOUND) + find_package(CapnProto 0.7 QUIET NO_MODULE) +endif() if(NOT CapnProto_FOUND) message(FATAL_ERROR "Cap'n Proto is required but was not found.\n" @@ -113,6 +120,7 @@ endif() include("cmake/compat_config.cmake") include("cmake/pthread_checks.cmake") +include("cmake/TargetCapnpSources.cmake") include(GNUInstallDirs) # Set MP_INCLUDE_DIR as a global property so target_capnp_sources function can @@ -120,12 +128,17 @@ include(GNUInstallDirs) # to avoid "error: Import failed: /mp/proxy.capnp" failures from capnproto. set_property(GLOBAL PROPERTY MP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") -# Set a convenience variable for subdirectories. -if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - set(MP_STANDALONE TRUE) - include(CTest) -else() - set(MP_STANDALONE FALSE) +# Set a convenience variable for subdirectories. May be pre-set by a parent +# CMakeLists.txt (e.g. the top-level wrapper used for `cmake -S .` standalone +# builds) so that test/ and other subdirectories know they should expose +# convenience targets even though lib/ is not the top-level source. +if(NOT DEFINED MP_STANDALONE) + if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(MP_STANDALONE TRUE) + include(CTest) + else() + set(MP_STANDALONE FALSE) + endif() endif() # Prevent include directories from parent project from leaking into this one. @@ -259,6 +272,5 @@ add_custom_target(install-lib VERBATIM) add_dependencies(install-lib multiprocess) -# Example and test subdirectories -add_subdirectory(example EXCLUDE_FROM_ALL) +# Test subdirectory add_subdirectory(test EXCLUDE_FROM_ALL) diff --git a/src/ipc/libmultiprocess/COPYING b/src/ipc/libmultiprocess/COPYING deleted file mode 100644 index 9d54ecbde122..000000000000 --- a/src/ipc/libmultiprocess/COPYING +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2009-2019 The Bitcoin Core developers -Copyright (c) 2009-2019 Bitcoin Developers - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/src/ipc/libmultiprocess/README.md b/src/ipc/libmultiprocess/README.md deleted file mode 100644 index 9a6e65795b3f..000000000000 --- a/src/ipc/libmultiprocess/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# libmultiprocess - -`libmultiprocess` is a C++ library and code generator making it easy to call functions and reference objects in different processes. - -For more information see the [usage instructions](doc/usage.md), [installation instructions](doc/install.md), or [design documentation](doc/design.md). - -If you have any questions, comments, or feedback, please submit an [issue](https://github.com/bitcoin-core/libmultiprocess/issues/new). -Duplicate issues are perfectly fine and all discussion about the project is welcome, since there isn't another discussion forum currently. diff --git a/src/ipc/libmultiprocess/ci/README.md b/src/ipc/libmultiprocess/ci/README.md deleted file mode 100644 index fef1c022b5c8..000000000000 --- a/src/ipc/libmultiprocess/ci/README.md +++ /dev/null @@ -1,52 +0,0 @@ -### CI quick-reference - -All CI is just bash and nix. - -* **Workflow**: - - `.github/workflows/ci.yml` – lists the jobs (`default`, `llvm`, …). -* **Scripts**: - - `ci/scripts/run.sh` – spins up the Nix shell then calls… - - `ci/scripts/ci.sh` – …to configure, build, and test. -* **Configuration**: - - `ci/configs/*.sh` – defines flags for each job. - - `shell.nix` – defines build environment (compilers, tools, libraries). -* **Build directories**: - - `build-*/` – separate build directories (like `build-default`, `build-llvm`) will be created for each job. - -To run jobs locally: - -```bash -CI_CONFIG=ci/configs/default.bash ci/scripts/run.sh -CI_CONFIG=ci/configs/llvm.bash ci/scripts/run.sh -CI_CONFIG=ci/configs/gnu32.bash ci/scripts/run.sh -CI_CONFIG=ci/configs/sanitize.bash ci/scripts/run.sh -CI_CONFIG=ci/configs/olddeps.bash ci/scripts/run.sh -``` - -By default CI jobs will reuse their build directories. `CI_CLEAN=1` can be specified to delete them before running instead. - -### Running workflows with `act` - -You can run either the entire workflow or a single matrix entry locally. On -macOS or Linux: - -1. Install [`act`](https://github.com/nektos/act) and either Docker or - Podman. -2. Inside the Podman VM, create a named volume for the Nix store (ext4, - case-sensitive) so builds persist across runs. Recreate it any time you want - a clean cache: - ```bash - podman volume create libmultiprocess-nix - ``` -3. From the repo root, launch the workflow. The example below targets the - sanitize matrix entry; drop the `--matrix` flag to run every configuration. - ```bash - act \ - --reuse \ - -P ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-24.04 \ - --container-options "-v libmultiprocess-nix:/nix" \ - -j build \ - --matrix config:sanitize - ``` - - diff --git a/src/ipc/libmultiprocess/ci/configs/default.bash b/src/ipc/libmultiprocess/ci/configs/default.bash deleted file mode 100644 index 56231228d4f3..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/default.bash +++ /dev/null @@ -1,5 +0,0 @@ -CI_DESC="CI job using default libraries and tools, and running IWYU" -CI_DIR=build-default -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -CMAKE_ARGS=(-DMP_ENABLE_IWYU=ON) -BUILD_ARGS=(-k) diff --git a/src/ipc/libmultiprocess/ci/configs/freebsd.bash b/src/ipc/libmultiprocess/ci/configs/freebsd.bash deleted file mode 100644 index 017f2186653e..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/freebsd.bash +++ /dev/null @@ -1,5 +0,0 @@ -CI_DESC="CI config for FreeBSD" -CI_DIR=build-freebsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -CMAKE_ARGS=(-G Ninja) -BUILD_ARGS=(-k 0) diff --git a/src/ipc/libmultiprocess/ci/configs/gnu32.bash b/src/ipc/libmultiprocess/ci/configs/gnu32.bash deleted file mode 100644 index 04d78a24fc5c..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/gnu32.bash +++ /dev/null @@ -1,11 +0,0 @@ -CI_DESC="CI job cross-compiling to 32-bit" -CI_DIR=build-gnu32 -# Cache the heaviest Nix job to stay within GitHub's cache budget. -CI_CACHE_NIX_STORE=true -NIX_ARGS=( - --arg minimal true - --arg crossPkgs 'import { crossSystem = { config = "i686-unknown-linux-gnu"; }; }' -) -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -CMAKE_ARGS=(-G Ninja) -BUILD_ARGS=(-k 0) diff --git a/src/ipc/libmultiprocess/ci/configs/llvm.bash b/src/ipc/libmultiprocess/ci/configs/llvm.bash deleted file mode 100644 index ba34cbf467bb..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/llvm.bash +++ /dev/null @@ -1,11 +0,0 @@ -CI_DESC="CI job using LLVM-based libraries and tools (clang, libc++, clang-tidy, iwyu) and testing Ninja" -CI_DIR=build-llvm -NIX_ARGS=(--arg enableLibcxx true) -export CXX=clang++ -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter" -CMAKE_ARGS=( - -G Ninja - -DMP_ENABLE_CLANG_TIDY=ON - -DMP_ENABLE_IWYU=ON -) -BUILD_ARGS=(-k 0) diff --git a/src/ipc/libmultiprocess/ci/configs/macos.bash b/src/ipc/libmultiprocess/ci/configs/macos.bash deleted file mode 100644 index 895d795f8912..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/macos.bash +++ /dev/null @@ -1,5 +0,0 @@ -CI_DESC="CI config for macOS" -CI_DIR=build-macos -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -CMAKE_ARGS=(-G Ninja -DMP_ENABLE_CLANG_TIDY=ON) -BUILD_ARGS=(-k 0) diff --git a/src/ipc/libmultiprocess/ci/configs/netbsd.bash b/src/ipc/libmultiprocess/ci/configs/netbsd.bash deleted file mode 100644 index c3e7d258e685..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/netbsd.bash +++ /dev/null @@ -1,11 +0,0 @@ -CI_DESC="CI config for NetBSD" -CI_DIR=build-netbsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -# Hardcode GCC 14, since default GCC versions installed by NetBSD are older -# and may not be compatible with libmultiprocess. GCC 14 was chosen because -# it's the latest compiler available on all versions of NetBSD that we test. -# Note that the GCC version specified here must match the version specified -# in pkg_add in ci.yml. -export CXX="/usr/pkg/gcc14/bin/g++" -CMAKE_ARGS=(-G Ninja) -BUILD_ARGS=(-k 0) diff --git a/src/ipc/libmultiprocess/ci/configs/olddeps.bash b/src/ipc/libmultiprocess/ci/configs/olddeps.bash deleted file mode 100644 index 95f44128d90c..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/olddeps.bash +++ /dev/null @@ -1,5 +0,0 @@ -CI_DESC="CI job using old Cap'n Proto and cmake versions" -CI_DIR=build-olddeps -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds" -NIX_ARGS=(--argstr capnprotoVersion "0.7.1" --argstr cmakeVersion "3.12.4") -BUILD_ARGS=(-k) diff --git a/src/ipc/libmultiprocess/ci/configs/openbsd.bash b/src/ipc/libmultiprocess/ci/configs/openbsd.bash deleted file mode 100644 index a404e2bc630a..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/openbsd.bash +++ /dev/null @@ -1,5 +0,0 @@ -CI_DESC="CI config for OpenBSD" -CI_DIR=build-openbsd -export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter" -CMAKE_ARGS=(-G Ninja) -BUILD_ARGS=(-k 0) diff --git a/src/ipc/libmultiprocess/ci/configs/sanitize.bash b/src/ipc/libmultiprocess/ci/configs/sanitize.bash deleted file mode 100644 index 0375fd3ab2d2..000000000000 --- a/src/ipc/libmultiprocess/ci/configs/sanitize.bash +++ /dev/null @@ -1,8 +0,0 @@ -CI_DESC="CI job running ThreadSanitizer" -CI_DIR=build-sanitize -NIX_ARGS=(--arg enableLibcxx true --argstr libcxxSanitizers "Thread" --argstr capnprotoSanitizers "thread") -export CXX=clang++ -export CXXFLAGS="-ggdb -Werror -Wall -Wextra -Wpedantic -Wthread-safety -Wno-unused-parameter -fsanitize=thread" -CMAKE_ARGS=() -BUILD_ARGS=(-k -j4) -BUILD_TARGETS=(mptest) diff --git a/src/ipc/libmultiprocess/ci/patches/spaceship.patch b/src/ipc/libmultiprocess/ci/patches/spaceship.patch deleted file mode 100644 index 08a885ddf0ab..000000000000 --- a/src/ipc/libmultiprocess/ci/patches/spaceship.patch +++ /dev/null @@ -1,29 +0,0 @@ -commit e3da7da967b94f373c29a198ce45f30fb9f0e517 -Author: Ed Catmur -Date: Tue Jan 31 16:27:04 2023 +0000 - - Remove operator!= synthesized by spaceship - - An operator!= suppresses the reversed equality comparison candidate generation. - - This is visible in clang 16 (rc0 just released) and in gcc trunk (so probably gcc 13). - -diff --git a/c++/src/kj/string.h b/c++/src/kj/string.h -index 193442aa..17835892 100644 ---- a/c++/src/kj/string.h -+++ b/c++/src/kj/string.h -@@ -122,10 +122,14 @@ public: - inline constexpr const char* end() const { return content.end() - 1; } - - inline constexpr bool operator==(decltype(nullptr)) const { return content.size() <= 1; } -+#if !__cpp_impl_three_way_comparison - inline constexpr bool operator!=(decltype(nullptr)) const { return content.size() > 1; } -+#endif - - inline bool operator==(const StringPtr& other) const; -+#if !__cpp_impl_three_way_comparison - inline bool operator!=(const StringPtr& other) const { return !(*this == other); } -+#endif - inline bool operator< (const StringPtr& other) const; - inline bool operator> (const StringPtr& other) const { return other < *this; } - inline bool operator<=(const StringPtr& other) const { return !(other < *this); } diff --git a/src/ipc/libmultiprocess/ci/scripts/bitcoin_core_ci.sh b/src/ipc/libmultiprocess/ci/scripts/bitcoin_core_ci.sh deleted file mode 100755 index 7f3c5cbd4b97..000000000000 --- a/src/ipc/libmultiprocess/ci/scripts/bitcoin_core_ci.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C - -set -o errexit -o nounset -o pipefail -o xtrace - -readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - -source "${SCRIPT_DIR}/ci_helpers.sh" - -replace_subtree() { - rm -rf src/ipc/libmultiprocess - cp -a _libmultiprocess src/ipc/libmultiprocess - rm -rf src/ipc/libmultiprocess/.git -} - -add_llvm_apt_repository() { - curl -s "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo tee "/etc/apt/trusted.gpg.d/apt.llvm.org.asc" > /dev/null - source /etc/os-release - echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${LLVM_VERSION} main" | sudo tee "/etc/apt/sources.list.d/llvm.list" - sudo apt-get update -} - -install_llvm_alternatives() { - sudo update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-${LLVM_VERSION}" 100 - sudo update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-${LLVM_VERSION}" 100 - sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" 100 -} - -set_llvm_alternatives() { - sudo update-alternatives --set clang "/usr/bin/clang-${LLVM_VERSION}" - sudo update-alternatives --set clang++ "/usr/bin/clang++-${LLVM_VERSION}" - sudo update-alternatives --set llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" -} - -install_tsan_packages() { - install_apt_packages \ - ccache \ - "clang-${LLVM_VERSION}" \ - "llvm-${LLVM_VERSION}" \ - "llvm-${LLVM_VERSION}-dev" \ - "libclang-${LLVM_VERSION}-dev" \ - "libclang-rt-${LLVM_VERSION}-dev" \ - ninja-build \ - pkgconf \ - python3-pip \ - bison - install_llvm_alternatives - set_llvm_alternatives - install_pip_packages --break-system-packages pycapnp -} - -build_instrumented_libcxx() { - export PATH="/usr/lib/llvm-${LLVM_VERSION}/bin:${PATH}" - - ls -l /usr/bin/clang /usr/bin/clang++ /usr/bin/llvm-symbolizer - which clang clang++ llvm-symbolizer - clang --version - clang++ --version - "/usr/bin/clang-${LLVM_VERSION}" --version - "/usr/bin/clang++-${LLVM_VERSION}" --version - git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-${LLVM_VERSION}.1.0" /tmp/llvm-project - cmake -G Ninja -B "${LIBCXX_DIR}" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_USE_SANITIZER=Thread \ - -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" \ - -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ - -DLIBCXX_INCLUDE_TESTS=OFF \ - -DLIBCXXABI_INCLUDE_TESTS=OFF \ - -DLIBUNWIND_INCLUDE_TESTS=OFF \ - -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ - -S /tmp/llvm-project/runtimes - grep -E 'CMAKE_(C|CXX)_COMPILER' "${LIBCXX_DIR}/CMakeCache.txt" - ninja -C "${LIBCXX_DIR}" -j "${BUILD_PARALLEL}" -v - rm -rf /tmp/llvm-project -} - -configure_bitcoin_core() { - local cmake_arg - local cmake_args=() - - if [[ -n "${BITCOIN_CORE_CMAKE_ARGS:-}" ]]; then - while IFS= read -r cmake_arg; do - [[ -n "${cmake_arg}" ]] || continue - cmake_args+=("${cmake_arg}") - done <<< "${BITCOIN_CORE_CMAKE_ARGS}" - fi - - cmake -S . -B build \ - --preset=dev-mode \ - -DCMAKE_BUILD_TYPE=Debug \ - -DBUILD_GUI=OFF \ - -DBUILD_GUI_TESTS=OFF \ - -DWITH_ZMQ=OFF \ - -DWITH_USDT=OFF \ - -DBUILD_BENCH=OFF \ - -DBUILD_FUZZ_BINARY=OFF \ - -DWITH_QRENCODE=OFF \ - -G Ninja \ - "${cmake_args[@]}" -} - -build_bitcoin_core() { - cmake --build build --parallel "${BUILD_PARALLEL}" -} - -build_depends_without_ipc() { - make -C depends -j "${BUILD_PARALLEL}" \ - CC=clang \ - CXX=clang++ \ - CXXFLAGS="${LIBCXX_FLAGS}" \ - NO_QT=1 \ - NO_ZMQ=1 \ - NO_USDT=1 \ - NO_QR=1 \ - NO_IPC=1 -} - -build_depends_with_ipc() { - make -C depends -j "${BUILD_PARALLEL}" \ - CC=clang \ - CXX=clang++ \ - CXXFLAGS="${LIBCXX_FLAGS}" \ - NO_QT=1 \ - NO_ZMQ=1 \ - NO_USDT=1 \ - NO_QR=1 -} - -run_ipc_unit_tests() { - local runs="$1" - - for _ in $(seq 1 "${runs}"); do - build/bin/test_bitcoin --run_test=ipc_tests,miner_tests --catch_system_error=no --log_level=nothing --report_level=no - done -} - -run_ipc_functional_tests() { - local runs="$1" - local timeout_factor="$2" - local test_scripts - local test_args=() - - test_scripts=$(python3 -c "import sys; import os; sys.path.append(os.path.abspath('build/test/functional')); from test_runner import ALL_SCRIPTS; print(' '.join(s for s in ALL_SCRIPTS if s.startswith('interface_ipc')))") - for _ in $(seq 1 "${runs}"); do - for script in $test_scripts; do - test_args+=("$script") - done - done - build/test/functional/test_runner.py "${test_args[@]}" --jobs "${PARALLEL}" --timeout-factor="${timeout_factor}" --failfast --combinedlogslen=99999999 -} - -main() { - local command="${1:?missing command}" - shift - - [[ "${command}" =~ ^[a-z_][a-z0-9_]*$ ]] || { - echo "Invalid command: ${command}" >&2 - exit 1 - } - - if declare -F "${command}" >/dev/null; then - "${command}" "$@" - else - echo "Unknown command: ${command}" >&2 - exit 1 - fi -} - -main "$@" diff --git a/src/ipc/libmultiprocess/ci/scripts/ci.sh b/src/ipc/libmultiprocess/ci/scripts/ci.sh deleted file mode 100755 index d989e9f45969..000000000000 --- a/src/ipc/libmultiprocess/ci/scripts/ci.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -set -o errexit -o nounset -o pipefail -o xtrace - -[ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" - -: "${CI_DIR:=build}" -if ! [ -v BUILD_TARGETS ]; then - BUILD_TARGETS=(all tests mpexamples) -fi - -[ -n "${CI_CLEAN-}" ] && rm -rf "${CI_DIR}" - -cmake --version -cmake_ver=$(cmake --version | awk '/version/{print $3; exit}') -ver_ge() { [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ]; } - -src_dir=$PWD -mkdir -p "$CI_DIR" -cd "$CI_DIR" -cmake "$src_dir" "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" -if ver_ge "$cmake_ver" "3.15"; then - cmake --build . -t "${BUILD_TARGETS[@]}" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" -else - # Older versions of cmake can only build one target at a time with --target, - # and do not support -t shortcut - for t in "${BUILD_TARGETS[@]}"; do - cmake --build . --target "$t" -- "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" - done -fi -ctest --output-on-failure diff --git a/src/ipc/libmultiprocess/ci/scripts/ci_helpers.sh b/src/ipc/libmultiprocess/ci/scripts/ci_helpers.sh deleted file mode 100755 index 1b6d0121969e..000000000000 --- a/src/ipc/libmultiprocess/ci/scripts/ci_helpers.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -export LC_ALL=C - -set -o errexit -o nounset -o pipefail -o xtrace - -write_env_var() { - local key="$1" - local value="$2" - - if [[ -n "${GITHUB_ENV:-}" ]]; then - echo "${key}=${value}" >> "${GITHUB_ENV}" - else - export "${key}=${value}" - fi -} - -write_output_var() { - local key="$1" - local value="$2" - - if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - echo "${key}=${value}" >> "${GITHUB_OUTPUT}" - else - echo "${key}=${value}" - fi -} - -available_nproc() { - if command -v nproc >/dev/null 2>&1; then - nproc - else - sysctl -n hw.logicalcpu - fi -} - -determine_parallelism() { - local multiplier="$1" - local available - - available="$(available_nproc)" - write_env_var BUILD_PARALLEL "${available}" - write_env_var PARALLEL "$((available * multiplier))" -} - -reset_ccache_stats() { - which ccache - ccache --version - ccache --zero-stats -} - -show_ccache_stats() { - ccache --show-stats -} - -install_apt_packages() { - sudo apt-get install --no-install-recommends -y "$@" -} - -install_homebrew_packages() { - HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 brew install --quiet "$@" -} - -install_pip_packages() { - pip3 install "$@" -} - -determine_host() { - local config_guess="${1:-./depends/config.guess}" - local output_key="${2:-host}" - - write_output_var "${output_key}" "$("${config_guess}")" -} - -ci_helpers_main() { - local command="${1:?missing command}" - shift - - [[ "${command}" =~ ^[a-z_][a-z0-9_]*$ ]] || { - echo "Invalid command: ${command}" >&2 - exit 1 - } - - if declare -F "${command}" >/dev/null; then - "${command}" "$@" - else - echo "Unknown command: ${command}" >&2 - exit 1 - fi -} - -if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then - ci_helpers_main "$@" -fi diff --git a/src/ipc/libmultiprocess/ci/scripts/config.sh b/src/ipc/libmultiprocess/ci/scripts/config.sh deleted file mode 100755 index 7fe8118d7bee..000000000000 --- a/src/ipc/libmultiprocess/ci/scripts/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Source CI configuration and output variables needed by the workflow. - -export LC_ALL=C - -set -o errexit -o nounset -o pipefail -o xtrace - -readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - -source "${SCRIPT_DIR}/ci_helpers.sh" - -[ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" - -# Resolve the nixpkgs channel to a specific revision for use in cache keys. -if [[ -n "${NIXPKGS_CHANNEL:-}" ]]; then - rev="$(curl --fail --location --silent --show-error "https://channels.nixos.org/${NIXPKGS_CHANNEL}/git-revision")" - test -n "${rev}" - write_output_var nixpkgs_rev "${rev}" -fi - -write_output_var cache_nix_store "${CI_CACHE_NIX_STORE:-false}" diff --git a/src/ipc/libmultiprocess/ci/scripts/run.sh b/src/ipc/libmultiprocess/ci/scripts/run.sh deleted file mode 100755 index 04ac8596d490..000000000000 --- a/src/ipc/libmultiprocess/ci/scripts/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -set -o errexit -o nounset -o pipefail -o xtrace - -[ "${CI_CONFIG+x}" ] && source "$CI_CONFIG" - -nix develop --ignore-environment --keep CI_CONFIG --keep CI_CLEAN "${NIX_ARGS[@]+"${NIX_ARGS[@]}"}" -f shell.nix --command ci/scripts/ci.sh - -# Create a GC root for the shell closure so the cache-nix-action save step -# does not garbage-collect it. -if [ -n "${CI_CACHE_NIX_STORE-}" ]; then - nix-build shell.nix \ - -o "$CI_DIR/gcroot" \ - "${NIX_ARGS[@]+"${NIX_ARGS[@]}"}" - # Verify the closure is complete so the cache-nix-action save step has - # everything it needs. - nix-store --query --requisites "$CI_DIR/gcroot" >/dev/null -fi diff --git a/src/ipc/libmultiprocess/doc/design.md b/src/ipc/libmultiprocess/doc/design.md deleted file mode 100644 index 113cafc4af39..000000000000 --- a/src/ipc/libmultiprocess/doc/design.md +++ /dev/null @@ -1,227 +0,0 @@ -# libmultiprocess Design - -Given an interface description of an object with one or more methods, libmultiprocess generates: - -* A C++ `ProxyClient` class template specialization with an implementation of each interface method that sends a request over a socket, waits for a response, and returns the result. -* A C++ `ProxyServer` class template specialization that listens for requests over a socket and calls a wrapped C++ object implementing the same interface to actually execute the requests. - -The function call ⇆ request translation supports input and output arguments, standard types like `unique_ptr`, `vector`, `map`, and `optional`, and bidirectional calls between processes through interface pointer and `std::function` arguments. - -If the wrapped C++ object inherits from an abstract base class declaring virtual methods, the generated `ProxyClient` objects can inherit from the same class, allowing interprocess calls to replace local calls without changes to existing code. - -There is also optional support for thread mapping, so each thread making interprocess calls can have a dedicated thread processing requests from it, and callbacks from processing threads are executed on corresponding request threads (so recursive mutexes and thread names function as expected in callbacks). - -Libmultiprocess acts as a pure wrapper or layer over the underlying protocol. Clients and servers written in other languages, but using a shared capnproto schema can communicate with interprocess counterparties using libmultiprocess without having to use libmultiprocess themselves or having to know about the implementation details of libmultiprocess. - -## Core Architecture - -The `ProxyClient` and `ProxyServer` generated classes are not directly exposed to the user, as described in [usage.md](usage.md). Instead, they wrap C++ interfaces and appear to the user as pointers to an interface. They are first instantiated when calling `ConnectStream` and `ServeStream` respectively for creating the `InitInterface`. These methods establish connections through sockets, internally creating `Connection` objects wrapping a `capnp::RpcSystem` configured for client and server mode respectively. - -The `InitInterface` interface will typically have methods which return other interfaces, giving the connecting process the ability to call other functions in the serving process. Interfaces can also have methods accepting other interfaces as parameters, giving serving processes the ability to call back and invoke functions in connecting processes. Creating new interfaces does not create new connections, and typically many interface objects will share the same connection. - -Both `ConnectStream` and `ServeStream` also require an instantiation of the `EventLoop`. The `EventLoop` owns pending requests, notifies on request dispatch, allows clients from multiple threads to make synchronous calls, and handles some cleanup routines on exit. It must be run in a separate thread so it is always active and can process incoming requests from local clients and remote connections. - -When a generated method on the `ProxyClient` is called, it calls `clientInvoke` with the capnp-translated types. `clientInvoke` creates a self-executing promise (`kj::TaskSet`) that drives the execution of the request and gives ownership of it to the `EventLoop`. `clientInvoke` blocks until a response is received, or until there is a call from the server that needs to run on the same client thread, using a `Waiter` object. - -On the server side, the `capnp::RpcSystem` receives the capnp request and invokes the corresponding C++ method through the corresponding `ProxyServer` and the heavily templated `serverInvoke` triggering a `ServerCall`. The return values from the actual C++ methods are copied into capnp responses by `ServerRet` and exceptions are caught and copied by `ServerExcept`. The two are connected through `ServerField`. The main method driving execution of a request is `PassField`, which is invoked through `ServerField`. Instantiated interfaces, or capabilities in capnp speak, are tracked and owned by the server's `capnp::RpcSystem`. - -## Request and Response Flow - -Method parameters and return values are serialized using Cap'n Proto's Builder objects (for sending) and Reader objects (for receiving). Input parameters flow from the client to the server, while output parameters (return values) flow back from the server to the client. - -```mermaid -sequenceDiagram - participant clientInvoke - participant BuildField as BuildField
(Client) - participant ReadField_C as ReadField
(Client) - participant Request as Request
message - participant serverInvoke - participant ReadField as ReadField
(Server) - participant BuildField_S as BuildField
(Server) - participant Response as Response
message - - Note over clientInvoke,ReadField: Input Parameter Flow - clientInvoke->>BuildField: BuildField(input_arg) - BuildField->>Request: Serialize input - Request->>serverInvoke: Cap'n Proto message - serverInvoke->>ReadField: Deserialize input - - Note over clientInvoke,Response: Output Parameter Flow - serverInvoke-->>BuildField_S: BuildField(output) - BuildField_S-->Response: Serialize output - Response-->>ReadField_C: Cap'n Proto message - ReadField_C-->>clientInvoke: Deserialize output -``` - -### Detailed Serialization Mechanism - -Parameters are represented as Fields that must be set on Cap'n Proto Builder objects (for sending) and read from Reader objects (for receiving). - -#### Building Fields - -`BuildField` uses a generated parameter `Accessor` to set the appropriate field in the Cap'n Proto Builder object. - -```mermaid -sequenceDiagram - participant clientInvoke as clientInvoke or
serverInvoke - participant BuildField - participant Accessor - participant Builder as Params::Builder - - Note over clientInvoke,Builder: Serializing Parameters - clientInvoke->>BuildField: BuildField(param1) - BuildField->>Accessor: Use generated field accessor - Accessor->>Builder: builder.setField1(param1) - - clientInvoke->>BuildField: BuildField(param2) - BuildField->>Accessor: Use generated field Accessor - Accessor->>Builder: builder.setField2(param2) -``` - -#### Reading Fields - -`ReadField` uses a generated parameter `Accessor` to read the appropriate field from the Cap'n Proto Reader object and reconstruct C++ parameters. - -```mermaid -sequenceDiagram - participant serverInvoke as clientInvoke or
serverInvoke - participant ReadField - participant Accessor - participant Reader as Params::Reader - participant ServerCall - - Note over serverInvoke,ServerCall: Deserializing Parameters - serverInvoke->>ReadField: Read param1 - ReadField->>Accessor: Use generated field accessor - Accessor->>Reader: reader.getField1() - Reader-->>ServerCall: call function with param1 -``` - -## Server-Side Request Processing - -The generated server code uses a Russian nesting doll structure to process method fields. Each `ServerField` wraps another `ServerField` (for the next parameter), or wraps `ServerRet` (for the return value), which finally wraps `ServerCall` (which invokes the actual C++ method). - -Each `ServerField` invokes `PassField`, which: -1. Calls `ReadField` to deserialize the parameter from the `Params::Reader` -2. Calls the next nested layer's `invoke()` with the accumulated parameters -3. Calls `BuildField` to serialize the parameter back if it's an output parameter - -`ServerRet` invokes the next layer (typically `ServerCall`), stores the result, and calls `BuildField` to serialize it into the `Results::Builder`. - -`ServerCall` uses the generated `ProxyMethod::impl` pointer-to-member to invoke the actual C++ method on the wrapped implementation object. - -```mermaid -sequenceDiagram - participant serverInvoke - participant SF1 as ServerField
(param 1) - participant SF2 as ServerField
(param 2) - participant SR as ServerRet
(return value) - participant SC as ServerCall - participant PMT as ProxyMethodTraits - participant Impl as Actual C++ Method - - serverInvoke->>SF1: SF1::invoke - SF1->>SF2: SF2::invoke - SF2->>SR: SR::invoke - SR->>SC: SC::invoke - SC->>PMT: PMT::invoke - PMT->>Impl: Call impl method - Impl->>PMT: return - PMT->>SC: return - SC->>SR: return - SR->>SF2: return - SF2->>SF1: return - SF1->>serverInvoke: return -``` - -## Advanced Features - -### Callbacks - -Callbacks (passed as `std::function` arguments) are intercepted by `CustomBuildField` and converted into Cap'n Proto capabilities that can be invoked across process boundaries. On the receiving end, `CustomReadField` intercepts the capability and constructs a `ProxyCallFn` object with an `operator()` that sends function calls back over the socket to invoke the original callback. - -```mermaid -sequenceDiagram - participant CT as Client Thread - participant C as clientInvoke - participant CBF1 as CustomBuildField (Client) - participant S as Socket - participant CRF1 as CustomReadField (Server) - participant Srv as Server Code - participant PCF as ProxyCallFn - - C->>CBF1: send function parameter - CBF1->>S: creates a Server for the function and sends a capability - S->>CRF1: receives a capability and creates ProxyCallFn - CRF1->>Srv: - Srv->>PCF: call the callback - PCF-->>CT: sends request to Client -``` - -### Thread Mapping - -Thread mapping enables each client thread to have a dedicated server thread processing its requests, preserving thread-local state and allowing recursive mutex usage across process boundaries. - -Thread mapping is initialized by defining an interface method with a `ThreadMap` parameter and/or response. The example below adds `ThreadMap` to the `construct` method because libmultiprocess calls the `construct` method automatically. - -```capnp -interface InitInterface $Proxy.wrap("Init") { - construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); -} -``` - -- **ThreadMap in parameter**: The client's `CustomBuildField` creates a `ThreadMap::Server` capability and sends it to the server, where `CustomReadField` stores the `ThreadMap::Client` in `connection.m_thread_map` -- **ThreadMap in response**: The server's `CustomBuildField` creates a `ThreadMap::Server` capability and sends it to the client, where `CustomReadField` stores the `ThreadMap::Client` in `connection.m_thread_map` - -You can specify ThreadMap in the parameter only, response only, or both: -- **Parameter only**: Server can create threads on the client -- **Response only**: Client can create threads on the server -- **Both (as shown)**: Bidirectional thread creation - -When both parameter and response include ThreadMap, both processes end up with `ThreadMap::Client` capabilities pointing to each other's `ThreadMap::Server`, allowing both sides to create threads on the other process. - -### Async Processing with Context - -By adding a `Context` parameter to a method in the capnp interface file, you enable async processing where the client tells the server to execute the request in a separate worker thread. For example: - -```capnp -processData @5 (context :Proxy.Context, data :Data) -> (result :Result); -``` - -If a method does not have a `Context` parameter, then libmultiprocess will execute IPC requests invoking that method on the I/O event loop thread. This is fine for fast and non-blocking methods, but should be avoided for any methods that are slow or blocking or make any IPC calls(including callbacks to the client), since as long as the method is executing, the Cap'n Proto event loop will not be able to perform any I/O. - -When a method has a `Context` parameter: - -**Client side** (`CustomBuildField`): -If this is the first asynchronous request made from the current client thread, `CustomBuildField` will: -1. Call `connection.m_thread_map.makeThreadRequest()` to request a dedicated worker thread on the server (stored in `request_threads` map) -2. Set the remote thread capability in `Context.thread` -3. Create a local `Thread::Server` object for the current thread (stored in `callback_threads` map) -4. Set the local thread capability in `Context.callbackThread` - -Subsequent requests will reuse the existing thread capabilities held in `callback_threads` and `request_threads`. - -**Server side** (`PassField`): -1. Looks up the local `Thread::Server` object specified by `context.thread` -2. The worker thread: - - Stores `context.callbackThread` in its `request_threads` map (so callbacks go to the right client thread) - - Posts the work lambda to that thread's queue via `waiter->post(invoke)` - - Cleans up the `request_threads` entry - -## Interface Definitions - -As explained in the [usage](usage.md) document, interface descriptions need to be consumed both by the _libmultiprocess_ code generator, and by C++ code that calls and implements the interfaces. The C++ code only needs to know about C++ arguments and return types, while the code generator only needs to know about capnp arguments and return types, but both need to know class and method names, so the corresponding `.h` and `.capnp` source files contain some of the same information, and have to be kept in sync manually when methods or parameters change. Despite the redundancy, reconciling the interface definitions is designed to be _straightforward_ and _safe_. _Straightforward_ because there is no need to write manual serialization code or use awkward intermediate types like [`UniValue`](https://github.com/bitcoin/bitcoin/blob/master/src/univalue/include/univalue.h) instead of native types. _Safe_ because if there are any inconsistencies between API and data definitions (even minor ones like using a narrow int data type for a wider int API input), there are errors at build time instead of errors or bugs at runtime. - -In the future, it would be possible to combine API and data definitions together using [C++ attributes](https://en.cppreference.com/w/cpp/language/attributes). To do this we would add attributes to the API definition files, and then generate the data definitions from the API definitions and attributes. I didn't take this approach mostly because it would be extra work, but also because until C++ standardizes reflection, this would require either hooking into compiler APIs like https://github.com/RosettaCommons/binder, or parsing C++ code manually like http://www.swig.org/. - -## What is `kj`? - -KJ is a concurrency framework [bundled with -capnproto](https://capnproto.org/cxxrpc.html#kj-concurrency-framework); it is used as a -basis in this library to construct the event-loop necessary to service IPC requests. - -## Future directions - -_libmultiprocess_ uses the [Cap'n Proto](https://capnproto.org) interface description language and protocol, but it could be extended or changed to use a different IDL/protocol like [gRPC](https://grpc.io). The nice thing about _Cap'n Proto_ compared to _gRPC_ and most other lower level protocols is that it allows interface pointers (_Services_ in gRPC parlance) to be passed as method arguments and return values, so object references and bidirectional requests work out of the box. Supporting a lower-level protocol would require adding maps and tracking code to proxy objects. - -_libmultiprocess_ is currently compatible with sandboxing but could add platform-specific sandboxing support or integration with a sandboxing library like [SAPI](https://github.com/google/sandboxed-api). diff --git a/src/ipc/libmultiprocess/doc/install.md b/src/ipc/libmultiprocess/doc/install.md deleted file mode 100644 index ac2f7f6a777c..000000000000 --- a/src/ipc/libmultiprocess/doc/install.md +++ /dev/null @@ -1,52 +0,0 @@ -# libmultiprocess Installation - -Installation currently requires Cap'n Proto 0.7 or higher: - -```sh -apt install libcapnp-dev capnproto -brew install capnp cmake -dnf install capnproto -``` - -Installation steps are: - -```sh -mkdir build -cd build -cmake .. -make -make check # Optionally build and run tests -make install -``` - -To build with libmultiprocess in a CMake project can specify: - -```cmake -find_package(Libmultiprocess) -target_capnp_sources(mytarget ${CMAKE_CURRENT_SOURCE_DIR} myschema.capnp) -``` - -Which will locate the libmultiprocess cmake package, and call the -`target_capnp_sources` function to generate C++ files and link them into a -library or executable target. See `example/CMakeLists.txt` for a complete -example. - -To build with libmultiprocess in a non-CMake project can use installed -`/include/mpgen.mk` Makefile rule to generate C++ files, and -`/lib/pkgconfig/libmultiprocess.pc` pkg-config definition to link -against the runtime library. - -For cross-compilation, it may be useful to build the runtime library and code -generation binaries separately, which can be done with: - -```sh -make install-bin # install bin/mpgen and related files -make install-lib # install lib/libmultiprocess.a and related files -``` - -It is also possible to import CMake targets separately with: - -```cmake -find_package(Libmultiprocess COMPONENTS Bin) -find_package(Libmultiprocess COMPONENTS Lib) -``` diff --git a/src/ipc/libmultiprocess/doc/usage.md b/src/ipc/libmultiprocess/doc/usage.md deleted file mode 100644 index f387db4d8d8a..000000000000 --- a/src/ipc/libmultiprocess/doc/usage.md +++ /dev/null @@ -1,27 +0,0 @@ -# libmultiprocess Usage - -## Overview - -_libmultiprocess_ is a library and code generator that allows calling C++ class interfaces across different processes. For an interface to be available from other processes, it needs two definitions: - -- An **API definition** declaring how the interface is called. Included examples: [calculator.h](../example/calculator.h), [printer.h](../example/printer.h), [init.h](../example/init.h). Bitcoin examples: [node.h](https://github.com/bitcoin/bitcoin/blob/master/src/interfaces/node.h), [wallet.h](https://github.com/bitcoin/bitcoin/blob/master/src/interfaces/wallet.h), [echo.h](https://github.com/bitcoin/bitcoin/blob/master/src/interfaces/echo.h), [init.h](https://github.com/bitcoin/bitcoin/blob/master/src/interfaces/init.h). - -- A **data definition** declaring how interface calls get sent across the wire. Included examples: [calculator.capnp](../example/calculator.capnp), [printer.capnp](../example/printer.capnp), [init.capnp](../example/init.capnp). Bitcoin examples: [node.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/node.capnp), [wallet.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/wallet.capnp), [echo.capnp](https://github.com/bitcoin/bitcoin/blob/master/src/ipc/capnp/echo.capnp), [init.capnp](https://github.com/bitcoin/bitcoin/blob/master/src/ipc/capnp/init.capnp). - -The `*.capnp` data definition files are consumed by the _libmultiprocess_ code generator and each `X.capnp` file generates `X.capnp.c++`, `X.capnp.h`, `X.capnp.proxy-client.c++`, `X.capnp.proxy-server.c++`, `X.capnp.proxy-types.c++`, `X.capnp.proxy-types.h`, and `X.capnp.proxy.h` output files. The generated files include `mp::ProxyClient` and `mp::ProxyServer` class specializations for all the interfaces in the `.capnp` files. These allow methods on C++ objects in one process to be called from other processes over IPC sockets. - -The `ProxyServer` objects help translate IPC requests from a socket to method calls on a local object. The `ProxyServer` objects are just used internally by the `mp::ServeStream(loop, socket, wrapped_object)` and `mp::ListenConnections(loop, socket, wrapped_object)` functions, and aren't exposed externally. The `ProxyClient` classes are exposed, and returned from the `mp::ConnectStream(loop, socket)` function and meant to be used directly. The classes implement methods described in `.capnp` definitions, and whenever any method is called, a request with the method arguments is sent over the associated IPC connection, and the corresponding `wrapped_object` method on the other end of the connection is called, with the `ProxyClient` method blocking until it returns and forwarding back any return value to the `ProxyClient` method caller. - -## Example - -A simple interface description can be found at [test/mp/test/foo.capnp](../test/mp/test/foo.capnp), implementation in [test/mp/test/foo.h](../test/mp/test/foo.h), and usage in [test/mp/test/test.cpp](../test/mp/test/test.cpp). - -A more complete example can be found in [example](../example/) and run with: - -```sh -mkdir build -cd build -cmake .. -make mpexamples -example/mpexample -``` diff --git a/src/ipc/libmultiprocess/doc/versions.md b/src/ipc/libmultiprocess/doc/versions.md deleted file mode 100644 index 2c2ec50e7c6d..000000000000 --- a/src/ipc/libmultiprocess/doc/versions.md +++ /dev/null @@ -1,78 +0,0 @@ -# libmultiprocess Versions - -Library versions are tracked with simple -[tags](https://github.com/bitcoin-core/libmultiprocess/tags) and -[branches](https://github.com/bitcoin-core/libmultiprocess/branches). - -Versioning policy is described in the [version.h](../include/mp/version.h) -include. - -## v10 -- Current unstable version. - -## [v9.0](https://github.com/bitcoin-core/libmultiprocess/commits/v9.0) -- Fixes race conditions where worker thread could be used after destruction, where getParams() could be called after request cancel, and where m_on_cancel could be called after request finishes. -- Adds `CustomHasField` hook to map Cap'n Proto null values to C++ null values. -- Improves `CustomBuildField` for `std::optional` to use move semantics. -- Adds LLVM 22 compatibility fix in type-map. -- Used in Bitcoin Core master branch, pulled in by [#34804](https://github.com/bitcoin/bitcoin/pull/34804). Also pulled into Bitcoin Core 31.x stable branch by [#34952](https://github.com/bitcoin/bitcoin/pull/34952). - -## [v8.0](https://github.com/bitcoin-core/libmultiprocess/commits/v8.0) -- Better support for non-libmultiprocess IPC clients: avoiding errors on unclean disconnects, and allowing simultaneous requests to worker threads which previously triggered "thread busy" errors. -- Intermediate version used in Bitcoin Core master branch between 30.x and 31.x branches, pulled in by [#34422](https://github.com/bitcoin/bitcoin/pull/34422). - -## [v7.0](https://github.com/bitcoin-core/libmultiprocess/commits/v7.0) -- Adds SpawnProcess race fix, cmake `target_capnp_sources` option, ci and documentation improvements. Adds `version.h` header declaring major and minor version numbers. -- Intermediate version used in Bitcoin Core master branch between 30.x and 31.x branches, pulled in by [#34363](https://github.com/bitcoin/bitcoin/pull/34363). - -## [v7.0-pre2](https://github.com/bitcoin-core/libmultiprocess/commits/v7.0-pre2) -- Fixes intermittent mptest hang and makes other minor improvements. -- Used in Bitcoin Core 30.1 and 30.2 releases and 30.x branch, pulled in by [#33518](https://github.com/bitcoin/bitcoin/pull/33518) and [#33519](https://github.com/bitcoin/bitcoin/pull/33519). - -## [v7.0-pre1](https://github.com/bitcoin-core/libmultiprocess/commits/v7.0-pre1) -- Adds support for log levels to reduce logging and "thread busy" error to avoid a crash on misuse. -- Minimum required version since Bitcoin Core 30.1, pulled in by [#33412](https://github.com/bitcoin/bitcoin/pull/33412), [#33518](https://github.com/bitcoin/bitcoin/pull/33518), and [#33519](https://github.com/bitcoin/bitcoin/pull/33519). - -## [v6.0](https://github.com/bitcoin-core/libmultiprocess/commits/v6.0) -- Adds CI scripts and build and test fixes. -- Used in Bitcoin Core 30.0 release, pulled in by [#32345](https://github.com/bitcoin/bitcoin/pull/32345), [#33241](https://github.com/bitcoin/bitcoin/pull/33241), and [#33322](https://github.com/bitcoin/bitcoin/pull/33322). - -## [v6.0-pre1](https://github.com/bitcoin-core/libmultiprocess/commits/v6.0-pre1) -- Adds fixes for unclean shutdowns and thread sanitizer issues. -- Drops `EventLoop::addClient` and `EventLoop::removeClient` methods, - requiring clients to use new `EventLoopRef` class instead. -- Minimum required version for Bitcoin Core 30.0 release, pulled in by [#31741](https://github.com/bitcoin/bitcoin/pull/31741), [#32641](https://github.com/bitcoin/bitcoin/pull/32641), and [#32345](https://github.com/bitcoin/bitcoin/pull/32345). - -## [v5.0](https://github.com/bitcoin-core/libmultiprocess/commits/v5.0) -- Adds build improvements and tidy/warning fixes. -- Used in Bitcoin Core 29 releases, pulled in by [#31945](https://github.com/bitcoin/bitcoin/pull/31945). - -## [v5.0-pre1](https://github.com/bitcoin-core/libmultiprocess/commits/v5.0-pre1) -- Adds many improvements to Bitcoin Core mining interface: splitting up type headers, fixing shutdown bugs, adding subtree build support. -- Broke up `proxy-types.h` into `type-*.h` files requiring clients to explicitly - include overloads needed for C++ ↔️ Cap'n Proto type conversions. -- Now requires C++ 20 support. -- Minimum required version for Bitcoin Core 29 releases, pulled in by [#30509](https://github.com/bitcoin/bitcoin/pull/30509), [#30510](https://github.com/bitcoin/bitcoin/pull/30510), [#31105](https://github.com/bitcoin/bitcoin/pull/31105), [#31740](https://github.com/bitcoin/bitcoin/pull/31740). - -## [v4.0](https://github.com/bitcoin-core/libmultiprocess/commits/v4.0) -- Added better cmake support, installing cmake package files so clients do not - need to use pkgconfig. -- Used in Bitcoin Core 28 releases, pulled in by [#30490](https://github.com/bitcoin/bitcoin/pull/30490) and [#30513](https://github.com/bitcoin/bitcoin/pull/30513). - -## [v3.0](https://github.com/bitcoin-core/libmultiprocess/commits/v3.0) -- Dropped compatibility with Cap'n Proto versions before 0.7. -- Used in Bitcoin Core 27 releases, pulled in by [#28735](https://github.com/bitcoin/bitcoin/pull/28735), [#28907](https://github.com/bitcoin/bitcoin/pull/28907), and [#29276](https://github.com/bitcoin/bitcoin/pull/29276). - -## [v2.0](https://github.com/bitcoin-core/libmultiprocess/commits/v2.0) -- Changed `PassField` function signature. -- Now requires C++17 support. -- Used in Bitcoin Core 25 and 26 releases, pulled in by [#26672](https://github.com/bitcoin/bitcoin/pull/26672). - -## [v1.0](https://github.com/bitcoin-core/libmultiprocess/commits/v1.0) -- Dropped hardcoded includes in generated files, now requiring `include` and - `includeTypes` annotations. -- Used in Bitcoin Core 22, 23, and 24 releases, pulled in by [#19160](https://github.com/bitcoin/bitcoin/pull/19160). - -## [v0.0](https://github.com/bitcoin-core/libmultiprocess/commits/v0.0) -- Initial version used in a downstream release. -- Used in Bitcoin Core 21 releases, pulled in by [#16367](https://github.com/bitcoin/bitcoin/pull/16367), [#18588](https://github.com/bitcoin/bitcoin/pull/18588), and [#18677](https://github.com/bitcoin/bitcoin/pull/18677). diff --git a/src/ipc/libmultiprocess/example/CMakeLists.txt b/src/ipc/libmultiprocess/example/CMakeLists.txt deleted file mode 100644 index 7da049c3a3f7..000000000000 --- a/src/ipc/libmultiprocess/example/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -include(${PROJECT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake) - -add_executable(mpcalculator - calculator.cpp -) -target_capnp_sources(mpcalculator ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) -target_include_directories(mpcalculator PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(mpcalculator PRIVATE Threads::Threads) - -add_executable(mpprinter - printer.cpp -) -target_capnp_sources(mpprinter ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) -target_include_directories(mpprinter PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(mpprinter PRIVATE Threads::Threads) - -add_executable(mpexample - example.cpp -) -target_capnp_sources(mpexample ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) -target_include_directories(mpexample PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(mpexample PRIVATE Threads::Threads) - -add_custom_target(mpexamples DEPENDS mpexample mpcalculator mpprinter) diff --git a/src/ipc/libmultiprocess/example/calculator.capnp b/src/ipc/libmultiprocess/example/calculator.capnp deleted file mode 100644 index 945518836016..000000000000 --- a/src/ipc/libmultiprocess/example/calculator.capnp +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -@0xb67dbf34061180a9; - -using Cxx = import "/capnp/c++.capnp"; -using Proxy = import "/mp/proxy.capnp"; - -$Proxy.include("calculator.h"); -$Proxy.includeTypes("types.h"); - -interface CalculatorInterface $Proxy.wrap("Calculator") { - destroy @0 (context :Proxy.Context) -> (); - solveEquation @1 (context :Proxy.Context, eqn: Text) -> (); -} diff --git a/src/ipc/libmultiprocess/example/calculator.cpp b/src/ipc/libmultiprocess/example/calculator.cpp deleted file mode 100644 index 86ce388b0cf9..000000000000 --- a/src/ipc/libmultiprocess/example/calculator.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include -#include -#include // NOLINT(misc-include-cleaner) // IWYU pragma: keep - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -class CalculatorImpl : public Calculator -{ -public: - CalculatorImpl(std::unique_ptr printer) : m_printer(std::move(printer)) {} - void solveEquation(const std::string& eqn) override { m_printer->print("Wow " + eqn + ", that's a tough one.\n"); } - std::unique_ptr m_printer; -}; - -class InitImpl : public Init -{ -public: - std::unique_ptr makeCalculator(std::unique_ptr printer) override - { - return std::make_unique(std::move(printer)); - } -}; - -// Exercises deprecated log callback signature -static void LogPrint(bool raise, const std::string& message) -{ - if (raise) throw std::runtime_error(message); - std::ofstream("debug.log", std::ios_base::app) << message << std::endl; -} - -int main(int argc, char** argv) -{ - if (argc != 2) { - std::cout << "Usage: mpcalculator \n"; - return 1; - } - int fd; - if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) { - std::cerr << argv[1] << " is not a number or is larger than an int\n"; - return 1; - } - mp::EventLoop loop("mpcalculator", LogPrint); - std::unique_ptr init = std::make_unique(); - mp::ServeStream(loop, fd, *init); - loop.loop(); - return 0; -} diff --git a/src/ipc/libmultiprocess/example/calculator.h b/src/ipc/libmultiprocess/example/calculator.h deleted file mode 100644 index 342a3c1d154a..000000000000 --- a/src/ipc/libmultiprocess/example/calculator.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef EXAMPLE_CALCULATOR_H -#define EXAMPLE_CALCULATOR_H - -#include - -class Calculator -{ -public: - virtual ~Calculator() = default; - virtual void solveEquation(const std::string& eqn) = 0; -}; - -#endif // EXAMPLE_CALCULATOR_H diff --git a/src/ipc/libmultiprocess/example/example.cpp b/src/ipc/libmultiprocess/example/example.cpp deleted file mode 100644 index 3831397781ce..000000000000 --- a/src/ipc/libmultiprocess/example/example.cpp +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include -#include - -#include // IWYU pragma: keep -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -static auto Spawn(mp::EventLoop& loop, const std::string& process_argv0, const std::string& new_exe_name) -{ - int pid; - const int fd = mp::SpawnProcess(pid, [&](int fd) -> std::vector { - fs::path path = process_argv0; - path.remove_filename(); - path.append(new_exe_name); - return {path.string(), std::to_string(fd)}; - }); - return std::make_tuple(mp::ConnectStream(loop, fd), pid); -} - -static void LogPrint(mp::LogMessage log_data) -{ - if (log_data.level == mp::Log::Raise) throw std::runtime_error(log_data.message); - std::ofstream("debug.log", std::ios_base::app) << log_data.message << std::endl; -} - -int main(int argc, char** argv) -{ - if (argc != 1) { - std::cout << "Usage: mpexample\n"; - return 1; - } - - std::promise promise; - std::thread loop_thread([&] { - mp::EventLoop loop("mpexample", LogPrint); - promise.set_value(&loop); - loop.loop(); - }); - mp::EventLoop* loop = promise.get_future().get(); - - auto [printer_init, printer_pid] = Spawn(*loop, argv[0], "mpprinter"); - auto [calc_init, calc_pid] = Spawn(*loop, argv[0], "mpcalculator"); - auto calc = calc_init->makeCalculator(printer_init->makePrinter()); - while (true) { - std::string eqn; - std::cout << "Enter the equation, or \"exit\" to quit: "; - std::getline(std::cin, eqn); - if (eqn == "exit") break; - calc->solveEquation(eqn); - } - calc.reset(); - calc_init.reset(); - mp::WaitProcess(calc_pid); - printer_init.reset(); - mp::WaitProcess(printer_pid); - loop_thread.join(); - std::cout << "Bye!" << std::endl; - return 0; -} diff --git a/src/ipc/libmultiprocess/example/init.capnp b/src/ipc/libmultiprocess/example/init.capnp deleted file mode 100644 index 01897f13dca4..000000000000 --- a/src/ipc/libmultiprocess/example/init.capnp +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -@0xba5a7448664901b1; - -using Cxx = import "/capnp/c++.capnp"; -using Proxy = import "/mp/proxy.capnp"; -using Calculator = import "calculator.capnp"; -using Printer = import "printer.capnp"; - -$Proxy.include("calculator.h"); -$Proxy.include("init.h"); -$Proxy.include("printer.h"); -$Proxy.includeTypes("types.h"); - -interface InitInterface $Proxy.wrap("Init") { - construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); - makeCalculator @1 (context :Proxy.Context, print :Printer.PrinterInterface) -> (result :Calculator.CalculatorInterface); - makePrinter @2 (context :Proxy.Context) -> (result :Printer.PrinterInterface); -} diff --git a/src/ipc/libmultiprocess/example/init.h b/src/ipc/libmultiprocess/example/init.h deleted file mode 100644 index 54e36da8db1d..000000000000 --- a/src/ipc/libmultiprocess/example/init.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef EXAMPLE_INIT_H -#define EXAMPLE_INIT_H - -#include -#include -#include - -class Init -{ -public: - virtual ~Init() = default; - virtual std::unique_ptr makePrinter() { return nullptr; } - virtual std::unique_ptr makeCalculator(std::unique_ptr printer) { return nullptr; } -}; - -#endif // EXAMPLE_INIT_H diff --git a/src/ipc/libmultiprocess/example/printer.capnp b/src/ipc/libmultiprocess/example/printer.capnp deleted file mode 100644 index 0f407b771411..000000000000 --- a/src/ipc/libmultiprocess/example/printer.capnp +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -@0x893db95f456ed0e3; - -using Cxx = import "/capnp/c++.capnp"; -using Proxy = import "/mp/proxy.capnp"; - -$Proxy.include("printer.h"); -$Proxy.includeTypes("types.h"); - -interface PrinterInterface $Proxy.wrap("Printer") { - destroy @0 (context :Proxy.Context) -> (); - print @1 (context :Proxy.Context, text: Text) -> (); -} diff --git a/src/ipc/libmultiprocess/example/printer.cpp b/src/ipc/libmultiprocess/example/printer.cpp deleted file mode 100644 index 9150d59b5d30..000000000000 --- a/src/ipc/libmultiprocess/example/printer.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include - -#include -#include // NOLINT(misc-include-cleaner) // IWYU pragma: keep - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -class PrinterImpl : public Printer -{ -public: - void print(const std::string& message) override { std::cout << "mpprinter: " << message << std::endl; } -}; - -class InitImpl : public Init -{ -public: - std::unique_ptr makePrinter() override { return std::make_unique(); } -}; - -static void LogPrint(mp::LogMessage log_data) -{ - if (log_data.level == mp::Log::Raise) throw std::runtime_error(log_data.message); - std::ofstream("debug.log", std::ios_base::app) << log_data.message << std::endl; -} - -int main(int argc, char** argv) -{ - if (argc != 2) { - std::cout << "Usage: mpprinter \n"; - return 1; - } - int fd; - if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) { - std::cerr << argv[1] << " is not a number or is larger than an int\n"; - return 1; - } - mp::EventLoop loop("mpprinter", LogPrint); - std::unique_ptr init = std::make_unique(); - mp::ServeStream(loop, fd, *init); - loop.loop(); - return 0; -} diff --git a/src/ipc/libmultiprocess/example/printer.h b/src/ipc/libmultiprocess/example/printer.h deleted file mode 100644 index fbdb35c057ea..000000000000 --- a/src/ipc/libmultiprocess/example/printer.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef EXAMPLE_PRINTER_H -#define EXAMPLE_PRINTER_H - -#include - -class Printer -{ -public: - virtual ~Printer() = default; - virtual void print(const std::string& message) = 0; -}; - -#endif // EXAMPLE_PRINTER_H diff --git a/src/ipc/libmultiprocess/example/types.h b/src/ipc/libmultiprocess/example/types.h deleted file mode 100644 index c926a00b40df..000000000000 --- a/src/ipc/libmultiprocess/example/types.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef EXAMPLE_TYPES_H -#define EXAMPLE_TYPES_H - -#include -#include - -// IWYU pragma: begin_exports -#include -#include -#include -#include -#include -// IWYU pragma: end_exports - -struct InitInterface; // IWYU pragma: export -struct CalculatorInterface; // IWYU pragma: export -struct PrinterInterface; // IWYU pragma: export - -#endif // EXAMPLE_TYPES_H diff --git a/src/ipc/libmultiprocess/include/mp/version.h b/src/ipc/libmultiprocess/include/mp/version.h index 964667a98fb6..423ed460070c 100644 --- a/src/ipc/libmultiprocess/include/mp/version.h +++ b/src/ipc/libmultiprocess/include/mp/version.h @@ -24,7 +24,7 @@ //! pointing at the prior merge commit. The /doc/versions.md file should also be //! updated, noting any significant or incompatible changes made since the //! previous version. -#define MP_MAJOR_VERSION 10 +#define MP_MAJOR_VERSION 11 //! Minor version number. Should be incremented in stable branches after //! backporting changes. The /doc/versions.md file should also be updated to diff --git a/src/ipc/libmultiprocess/shell.nix b/src/ipc/libmultiprocess/shell.nix deleted file mode 100644 index 9ebbc0a034e9..000000000000 --- a/src/ipc/libmultiprocess/shell.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ pkgs ? import {} -, crossPkgs ? import {} -, enableLibcxx ? false # Whether to use libc++ toolchain and libraries instead of libstdc++ -, minimal ? false # Whether to create minimal shell without extra tools (faster when cross compiling) -, capnprotoVersion ? null -, capnprotoSanitizers ? null # Optional sanitizers to build cap'n proto with -, cmakeVersion ? null -, libcxxSanitizers ? null # Optional LLVM_USE_SANITIZER value to use for libc++, see https://llvm.org/docs/CMake.html -}: - -let - lib = pkgs.lib; - llvmBase = crossPkgs.llvmPackages_21; - llvm = llvmBase // lib.optionalAttrs (libcxxSanitizers != null) { - libcxx = llvmBase.libcxx.override { - devExtraCmakeFlags = [ "-DLLVM_USE_SANITIZER=${libcxxSanitizers}" ]; - }; - }; - capnprotoHashes = { - "0.7.0" = "sha256-Y/7dUOQPDHjniuKNRw3j8dG1NI9f/aRWpf8V0WzV9k8="; - "0.7.1" = "sha256-3cBpVmpvCXyqPUXDp12vCFCk32ZXWpkdOliNH37UwWE="; - "0.8.0" = "sha256-rfiqN83begjJ9eYjtr21/tk1GJBjmeVfa3C3dZBJ93w="; - "0.8.1" = "sha256-OZqNVYdyszro5rIe+w6YN00g6y8U/1b8dKYc214q/2o="; - "0.9.0" = "sha256-yhbDcWUe6jp5PbIXzn5EoKabXiWN8lnS08hyfxUgEQ0="; - "0.9.2" = "sha256-BspWOPZcP5nCTvmsDE62Zutox+aY5pw42d6hpH3v4cM="; - "0.10.0" = "sha256-++F4l54OMTDnJ+FO3kV/Y/VLobKVRk461dopanuU3IQ="; - "0.10.4" = "sha256-45sxnVyyYIw9i3sbFZ1naBMoUzkpP21WarzR5crg4X8="; - "1.0.0" = "sha256-NLTFJdeOzqhk4ATvkc17Sh6g/junzqYBBEoXYGH/czo="; - "1.0.2" = "sha256-LVdkqVBTeh8JZ1McdVNtRcnFVwEJRNjt0JV2l7RkuO8="; - "1.1.0" = "sha256-gxkko7LFyJNlxpTS+CWOd/p9x/778/kNIXfpDGiKM2A="; - "1.2.0" = "sha256-aDcn4bLZGq8915/NPPQsN5Jv8FRWd8cAspkG3078psc="; - }; - capnprotoBase = if capnprotoVersion == null then crossPkgs.capnproto else crossPkgs.capnproto.overrideAttrs (old: { - version = capnprotoVersion; - src = crossPkgs.fetchFromGitHub { - owner = "capnproto"; - repo = "capnproto"; - rev = "v${capnprotoVersion}"; - hash = lib.attrByPath [capnprotoVersion] "" capnprotoHashes; - }; - patches = lib.optionals (lib.versionAtLeast capnprotoVersion "0.9.0" && lib.versionOlder capnprotoVersion "0.10.4") [ ./ci/patches/spaceship.patch ]; - } // (lib.optionalAttrs (lib.versionOlder capnprotoVersion "0.10") { - env = { }; # Drop -std=c++20 flag forced by nixpkgs - })); - capnproto = (capnprotoBase.overrideAttrs (old: lib.optionalAttrs (capnprotoSanitizers != null) { - env = (old.env or { }) // { - CXXFLAGS = - lib.concatStringsSep " " [ - (old.env.CXXFLAGS or "") - "-fsanitize=${capnprotoSanitizers}" - "-fno-omit-frame-pointer" - "-g" - ]; - }; - })).override (lib.optionalAttrs enableLibcxx { clangStdenv = llvm.libcxxStdenv; }); - clang = if enableLibcxx then llvm.libcxxClang else llvm.clang; - clang-tools = llvm.clang-tools.override { inherit enableLibcxx; }; - cmakeHashes = { - "3.12.4" = "sha256-UlVYS/0EPrcXViz/iULUcvHA5GecSUHYS6raqbKOMZQ="; - }; - cmakeBuild = if cmakeVersion == null then pkgs.cmake else (pkgs.cmake.overrideAttrs (old: { - version = cmakeVersion; - src = pkgs.fetchurl { - url = "https://cmake.org/files/v${lib.versions.majorMinor cmakeVersion}/cmake-${cmakeVersion}.tar.gz"; - hash = lib.attrByPath [cmakeVersion] "" cmakeHashes; - }; - patches = []; - })).override { isMinimalBuild = true; }; -in crossPkgs.mkShell { - buildInputs = [ - capnproto - ]; - nativeBuildInputs = with pkgs; [ - cmakeBuild - include-what-you-use - ninja - ] ++ lib.optionals (!minimal) [ - clang - clang-tools - ]; - - # Tell IWYU where its libc++ mapping lives - IWYU_MAPPING_FILE = if enableLibcxx then "${llvm.libcxx.dev}/include/c++/v1/libcxx.imp" else null; -} diff --git a/src/ipc/libmultiprocess/src/mp/proxy.cpp b/src/ipc/libmultiprocess/src/mp/proxy.cpp index d24208dbfd89..963050c3bcfc 100644 --- a/src/ipc/libmultiprocess/src/mp/proxy.cpp +++ b/src/ipc/libmultiprocess/src/mp/proxy.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -245,7 +246,12 @@ void EventLoop::loop() if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly"); Lock lock(m_mutex); if (m_post_fn) { - Unlock(lock, *m_post_fn); + // m_post_fn throwing is never expected. If it does happen, the caller + // of EventLoop::post() will return without any indication of failure, + // which will likely cause other bugs. Log the error and continue. + KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() MP_REQUIRES(m_mutex) { Unlock(lock, *m_post_fn); })) { + MP_LOG(*this, Log::Error) << "EventLoop: m_post_fn threw: " << kj::str(*exception).cStr(); + } m_post_fn = nullptr; m_cv.notify_all(); } else if (done()) { diff --git a/src/ipc/libmultiprocess/test/CMakeLists.txt b/src/ipc/libmultiprocess/test/CMakeLists.txt index 1f21ba44ca04..097bc8c5b58d 100644 --- a/src/ipc/libmultiprocess/test/CMakeLists.txt +++ b/src/ipc/libmultiprocess/test/CMakeLists.txt @@ -29,7 +29,7 @@ if(BUILD_TESTING AND TARGET CapnProto::kj-test) mp/test/spawn_tests.cpp mp/test/test.cpp ) - include(${PROJECT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake) + target_capnp_sources(mptest ${CMAKE_CURRENT_SOURCE_DIR} mp/test/foo.capnp) target_include_directories(mptest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(mptest PRIVATE CapnProto::kj-test) diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 534fadbad710..f704d6e01a12 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -2,14 +2,31 @@ # Copyright (c) 2015-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# The default check (no flags) verifies that the subtree directory has not +# been touched outside of a subtree merge. With -r, it additionally verifies +# that the recorded subtree commit exists in the local repository and that +# its tree matches what is checked in (assumes the upstream remote has been +# fetched). +# +# With -s, for subtrees imported from a 'git subtree split' branch (rather +# than upstream master directly), it re-runs the deterministic split locally +# on FETCH_HEAD and confirms the recorded subtree commit is reachable from +# the result. This catches the case where a malicious upstream publishes a +# split branch with contents that don't match what 'git subtree split' would +# produce from master. 'git subtree split' is deterministic and incremental, +# so re-runs are cheap (cached under refs/subtree-cache/) and any honest +# split SHA from past upstream history will still be reachable from a fresh +# split of current master. export LC_ALL=C check_remote=0 -while getopts "?hr" opt; do +split_prefix= +while getopts "?hrs:" opt; do case $opt in '?' | h) - echo "Usage: $0 [-r] DIR [COMMIT]" + echo "Usage: $0 [-r] [-s SPLIT_PREFIX] DIR [COMMIT]" echo " $0 -?" echo "" echo "Checks that a certain prefix is pure subtree, and optionally whether the" @@ -23,11 +40,26 @@ while getopts "?hr" opt; do echo "" echo " git fetch https://github.com/bitcoin-core/secp256k1.git" echo " test/lint/git-subtree-check.sh -r src/secp256k1" + echo "" + echo "-s SPLIT_PREFIX" + echo " For subtrees imported from a 'git subtree split' branch (rather" + echo " than from upstream master directly), verify that the recorded" + echo " subtree commit could have been produced by" + echo " 'git subtree split --prefix=SPLIT_PREFIX' from upstream master." + echo " Implies -r. Requires upstream master to have been fetched into" + echo " FETCH_HEAD immediately before running this script. Example:" + echo "" + echo " git fetch https://github.com/example/foo.git master" + echo " test/lint/git-subtree-check.sh -s lib src/foo" exit 1 ;; r) check_remote=1 ;; + s) + split_prefix="${OPTARG%/}" + check_remote=1 + ;; esac done shift $((OPTIND-1)) @@ -130,4 +162,39 @@ if [ "$check_remote" != "0" ]; then fi fi +if [ -n "$split_prefix" ]; then + # See header comment for rationale. $rev is the subtree commit recorded + # in the latest squash; check that it is reachable from a fresh local + # split of upstream FETCH_HEAD at $split_prefix. + if [ "d$(git cat-file -t FETCH_HEAD 2>/dev/null)" != dcommit ]; then + echo "FETCH_HEAD unavailable: cannot verify split. Did you fetch upstream master?" >&2 + exit 1 + fi + upstream_commit=$(git rev-parse --verify "FETCH_HEAD^{commit}") + echo "verifying $rev was produced by 'git subtree split --prefix=$split_prefix' from FETCH_HEAD ($upstream_commit)" + + if ! split_head=$( + repo_root=$(pwd) + split_worktree=$(mktemp -d "${TMPDIR:-/tmp}/git-subtree-check.XXXXXX") || exit 1 + trap 'git -c safe.directory="$repo_root" worktree remove --force "$split_worktree" >/dev/null 2>&1 || rm -rf "$split_worktree"' EXIT HUP INT TERM + + # git-subtree before 2.54 (commit a606fcdceb) required --prefix to + # exist in the current worktree, even when splitting a different + # revision. Run the split inside a detached worktree at FETCH_HEAD so + # the prefix is always present regardless of the local git version. + git -c safe.directory="$repo_root" worktree add --quiet --detach "$split_worktree" "$upstream_commit" >/dev/null 2>&1 || exit 1 + git -c safe.directory="$repo_root" -c safe.directory="$split_worktree" -C "$split_worktree" subtree split --prefix="$split_prefix" HEAD 2>/dev/null + ); then + echo "FAIL: 'git subtree split --prefix=$split_prefix FETCH_HEAD' failed (does the prefix exist in FETCH_HEAD?)" >&2 + exit 1 + fi + + if ! git merge-base --is-ancestor "$rev" "$split_head"; then + echo "FAIL: subtree commit $rev is not reachable from 'git subtree split --prefix=$split_prefix FETCH_HEAD' ($split_head)" >&2 + echo " The split branch was not honestly produced from upstream master." >&2 + exit 1 + fi + echo "$rev is contained in 'git subtree split --prefix=$split_prefix FETCH_HEAD' ($split_head)" +fi + echo "GOOD" diff --git a/test/lint/test_runner/src/lint_repo_hygiene.rs b/test/lint/test_runner/src/lint_repo_hygiene.rs index d2fc07918daf..ae9b31ace7fd 100644 --- a/test/lint/test_runner/src/lint_repo_hygiene.rs +++ b/test/lint/test_runner/src/lint_repo_hygiene.rs @@ -4,7 +4,7 @@ use std::process::Command; -use crate::util::{commit_range, get_subtrees, LintResult}; +use crate::util::{commit_range, get_split_subtrees, get_subtrees, git, LintResult}; pub fn lint_subtree() -> LintResult { // This only checks that the trees are pure subtrees, it is not doing a full @@ -17,6 +17,31 @@ pub fn lint_subtree() -> LintResult { .expect("command_error") .success(); } + // For subtrees imported from a 'git subtree split' branch, additionally + // verify that the recorded subtree commit could have been produced by + // splitting upstream master at the documented prefix. This requires a + // network fetch into FETCH_HEAD, which git-subtree-check.sh -s reads. + for sub in get_split_subtrees() { + // Fetch the published split branch first so the recorded subtree + // commit exists locally for the implicit '-r' check. Then fetch the + // source branch so FETCH_HEAD points to what should be split. + for fetch_ref in sub + .upstream_split_ref + .into_iter() + .chain(std::iter::once(sub.upstream_ref)) + { + good &= git() + .args(["fetch", "--quiet", sub.upstream_url, fetch_ref]) + .status() + .expect("command_error") + .success(); + } + good &= Command::new("test/lint/git-subtree-check.sh") + .args(["-s", sub.split_prefix, sub.path]) + .status() + .expect("command_error") + .success(); + } if good { Ok(()) } else { diff --git a/test/lint/test_runner/src/util.rs b/test/lint/test_runner/src/util.rs index 11e332e4c5e0..af1183ba6c30 100644 --- a/test/lint/test_runner/src/util.rs +++ b/test/lint/test_runner/src/util.rs @@ -72,6 +72,40 @@ pub fn get_subtrees() -> Vec<&'static str> { ] } +/// Description of a subtree imported via 'git subtree split'. +/// +/// Such subtrees are not pulled directly from upstream master; instead, upstream +/// publishes a 'split' branch produced by 'git subtree split --prefix=' +/// of master, and only that subdirectory is consumed here. To verify that the +/// split branch was honestly produced from master (and not tampered with), the +/// lint runner re-runs the deterministic split locally and confirms the +/// recorded subtree commit is reachable from the result. +pub struct SplitSubtree { + /// Path of the subtree within this repository. + pub path: &'static str, + /// Upstream repository URL to fetch from. + pub upstream_url: &'static str, + /// Branch on the upstream repository to fetch and split. + pub upstream_ref: &'static str, + /// Branch containing the published split history, if different. + pub upstream_split_ref: Option<&'static str>, + /// Prefix passed to 'git subtree split --prefix=' on the upstream side. + pub split_prefix: &'static str, +} + +/// Return all subtrees imported from a 'git subtree split' branch. +pub fn get_split_subtrees() -> Vec { + vec![SplitSubtree { + path: "src/ipc/libmultiprocess", + // Temporary [do not merge] override for PR branch validation until the + // split `lib` branch is pushed to bitcoin-core/libmultiprocess. + upstream_url: "https://github.com/Sjors/libmultiprocess.git", + upstream_ref: "2026/05/subtree-split", + upstream_split_ref: Some("lib"), + split_prefix: "lib", + }] +} + /// Return the pathspecs to exclude by default pub fn get_pathspecs_default_excludes() -> Vec { get_subtrees()