diff --git a/.github/workflows/auto-pr-review.yml b/.github/workflows/auto-pr-review.yml new file mode 100644 index 0000000000..6a585355f7 --- /dev/null +++ b/.github/workflows/auto-pr-review.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Auto PR Commenter" + +on: + pull_request_target: + types: [opened] + +jobs: + add-review-comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Add review comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + @codecov-ai-reviewer review diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index fc6d969e37..fa804e260c 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -1,3 +1,4 @@ + name: "3rd-party" on: @@ -32,7 +33,7 @@ jobs: - name: mvn install run: | - mvn install -Dmaven.test.skip=true -ntp + mvn install -Dmaven.test.skip=true -ntp --fail-at-end - name: generate current dependencies run: | bash $SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh current-dependencies.txt diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 538eb7f98c..d66dc8cee9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -51,10 +51,10 @@ jobs: # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 6915e22eb2..d4f2ea382f 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -10,7 +10,50 @@ on: # TODO: consider merge to one ci.yml file jobs: + struct: + runs-on: ubuntu-latest + env: + USE_STAGE: 'false' + steps: + - name: Install JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'zulu' + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 5 + + - name: Use staged maven repo settings + run: | + cp $HOME/.m2/settings.xml /tmp/settings.xml || true + mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + + - name: Resolve project revision + run: echo "REVISION=$(mvn -q -DforceStdout help:evaluate -Dexpression=revision -f pom.xml)" >> $GITHUB_ENV + + - name: Build or fetch hugegraph-struct + run: | + if [ -f hugegraph-struct/pom.xml ]; then + echo "[INFO] Found hugegraph-struct source, building from source" + mvn -U -ntp -DskipTests -pl hugegraph-struct -am install + else + echo "[INFO] hugegraph-struct source not found, fetching artifact $REVISION" + if [ -z "$REVISION" ]; then echo "[ERROR] revision not resolved"; exit 1; fi + mvn -U -ntp dependency:get -Dartifact=org.apache.hugegraph:hugegraph-struct:$REVISION + fi + pd: + needs: struct runs-on: ubuntu-latest env: # TODO: avoid duplicated env setup in pd & store @@ -29,7 +72,7 @@ jobs: - name: Cache Maven packages uses: actions/cache@v3 with: - path: ~/.m2 + path: ~/.m2/repository key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 @@ -55,8 +98,9 @@ jobs: # The above tests do not require starting a PD instance. - name: Package + # todo remove --fail-at-end after test run: | - mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp + mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - name: Prepare env and service run: | @@ -76,7 +120,7 @@ jobs: file: ${{ env.REPORT_DIR }}/*.xml store: - # TODO: avoid duplicated env setup + needs: struct runs-on: ubuntu-latest env: USE_STAGE: 'false' # Whether to include the stage repository. @@ -94,7 +138,7 @@ jobs: - name: Cache Maven packages uses: actions/cache@v3 with: - path: ~/.m2 + path: ~/.m2/repository key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 @@ -110,8 +154,9 @@ jobs: mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml - name: Package + # todo remove --fail-at-end after test run: | - mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp + mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - name: Prepare env and service run: | @@ -148,7 +193,7 @@ jobs: file: ${{ env.REPORT_DIR }}/*.xml hstore: - # TODO: avoid duplicated env setup + needs: struct runs-on: ubuntu-latest env: USE_STAGE: 'false' # Whether to include the stage repository. @@ -167,7 +212,7 @@ jobs: - name: Cache Maven packages uses: actions/cache@v3 with: - path: ~/.m2 + path: ~/.m2/repository key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 @@ -183,8 +228,9 @@ jobs: mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml - name: Package + # todo remove --fail-at-end after test run: | - mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp + mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - name: Prepare env and service run: | diff --git a/.gitignore b/.gitignore index c8c2e1eb55..f9670e332f 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ output/ tree.txt *.versionsBackup .flattened-pom.xml +**/dependency-reduced-pom.xml +install-dist/dist.sh # eclipse ignore @@ -87,3 +89,26 @@ hs_err_pid* # docker volumes ignore hugegraph-server/hugegraph-dist/docker/data/ + +# AI-IDE prompt files (We only keep AGENTS.md, other files could soft-linked it when needed) +# Claude Projects +CLAUDE.md +CLAUDE_*.md +# Gemini/Google +GEMINI.md +# GitHub Copilot / Microsoft +copilot-instructions.md +.copilot-instructions.md +# Cursor IDE +cursor-instructions.md +.cursor-instructions.md +cursor.md +# Windsurf/Codeium +windsurf.md +windsurf-instructions.md +codeium.md +codeium-instructions.md +# Other AI coding assistants +.ai-instructions.md +*.ai-prompt.md +WARP.md diff --git a/.licenserc.yaml b/.licenserc.yaml index 573ba55c43..3ebf89162d 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -100,6 +100,8 @@ header: # `header` section is configurations for source codes license header. - 'hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java' - 'hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherOpProcessor.java' - 'hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java' + - 'hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java' + - 'hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java' comment: on-failure # on what condition license-eye will comment on the pull request, `on-failure`, `always`, `never`. # license-location-threshold specifies the index threshold where the license header can be located, diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..a868739d84 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,261 @@ +# AGENTS.md + +This file provides guidance to an AI coding tool when working with code in this repository. + +## Project Overview + +Apache HugeGraph is a fast-speed and highly-scalable graph database that supports billions of vertices and edges. It is compliant with Apache TinkerPop 3 and supports both Gremlin and Cypher query languages. + +**Technology Stack**: +- Java 11+ (required) +- Apache Maven 3.5+ +- Apache TinkerPop 3.5.1 +- gRPC for distributed communication +- RocksDB as default storage backend + +## Architecture + +### Multi-Module Structure + +This is a Maven multi-module project with 7 main modules: + +1. **hugegraph-server**: Core graph engine, REST APIs, and backend implementations (13 submodules) +2. **hugegraph-pd**: Placement Driver (meta server) for distributed deployments (8 submodules) +3. **hugegraph-store**: Distributed storage backend with RocksDB and Raft (9 submodules) +4. **hugegraph-commons**: Shared utilities (locks, configs, RPC framework) +5. **hugegraph-struct**: Data structure definitions +6. **install-dist**: Distribution packaging +7. **hugegraph-cluster-test**: Cluster integration tests + +### Three-Tier Architecture + +```bash +Client Layer (Gremlin/Cypher queries, REST APIs) + ↓ +Server Layer (hugegraph-server) + ├─ REST API Layer (hugegraph-api): GraphAPI, SchemaAPI, GremlinAPI, CypherAPI, AuthAPI + ├─ Graph Engine Layer (hugegraph-core): Schema management, traversal optimization, task scheduling + └─ Backend Interface: Abstraction over storage backends + ↓ +Storage Layer (pluggable backends) + ├─ RocksDB (default, embedded) + ├─ HStore (distributed, production) + └─ Legacy: MySQL, PostgreSQL, Cassandra, ScyllaDB, HBase, Palo +``` + +### Distributed Components (Optional) + +For production distributed deployments: +- **hugegraph-pd**: Service discovery, partition management, metadata coordination +- **hugegraph-store**: Distributed storage with Raft consensus (typically 3+ nodes) +- **hugegraph-server**: Multiple server instances (typically 3+) + +All inter-service communication uses gRPC with Protocol Buffers. + +### Key Architectural Patterns + +1. **Pluggable Backend Architecture**: Storage backends implement the `BackendStore` interface, allowing new backends without modifying core code +2. **TinkerPop Compliance**: Full Apache TinkerPop 3 implementation with custom optimization strategies +3. **gRPC Communication**: All distributed components communicate via gRPC (proto definitions in `*/grpc/` directories) +4. **Multi-Language Queries**: Native Gremlin support + OpenCypher implementation in `hugegraph-api/opencypher` + +## Build & Development Commands + +### Prerequisites Check +```bash +# Verify Java version (11+ required) +java -version + +# Verify Maven version (3.5+ required) +mvn -version +``` + +### Full Build +```bash +# Clean build with all modules +mvn clean install -DskipTests + +# Build with tests +mvn clean install + +# Build specific module (e.g., server only) +mvn clean install -pl hugegraph-server -am -DskipTests +``` + +### Testing + +#### Server Module Tests +```bash +# Unit tests (memory backend) +mvn test -pl hugegraph-server/hugegraph-test -am -P unit-test + +# Core tests with specific backend +mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,memory +mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,rocksdb +mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,hbase + +# API tests with backend +mvn test -pl hugegraph-server/hugegraph-test -am -P api-test,rocksdb + +# TinkerPop compliance tests (for release branches) +mvn test -pl hugegraph-server/hugegraph-test -am -P tinkerpop-structure-test,memory +mvn test -pl hugegraph-server/hugegraph-test -am -P tinkerpop-process-test,memory +``` + +#### PD & Store Module Tests +```bash +# Build and test hugegraph-struct first (dependency) +mvn install -pl hugegraph-struct -am -DskipTests + +# Test PD module +mvn test -pl hugegraph-pd/hg-pd-test -am + +# Test Store module +mvn test -pl hugegraph-store/hg-store-test -am +``` + +### Code Quality & Validation + +```bash +# License header check (Apache RAT) +mvn apache-rat:check + +# Code style check (EditorConfig) +mvn editorconfig:check + +# Compile with warnings +mvn clean compile -Dmaven.javadoc.skip=true +``` + +### Running the Server + +Scripts are located in `hugegraph-server/hugegraph-dist/src/assembly/static/bin/`: + +```bash +# Initialize storage backend +bin/init-store.sh + +# Start server +bin/start-hugegraph.sh + +# Stop server +bin/stop-hugegraph.sh + +# Gremlin console +bin/gremlin-console.sh + +# Enable authentication +bin/enable-auth.sh +``` + +### Creating Distribution Package + +```bash +# Build distribution tarball (auto-enabled by default) +mvn clean package -DskipTests + +# Skip assembly creation (if needed) +mvn clean package -DskipTests -Dskip-assembly-hugegraph + +# Output: install-dist/target/hugegraph-.tar.gz +``` + +## Important File Locations + +### Configuration Files +- Server configs: `hugegraph-server/hugegraph-dist/src/assembly/static/conf/` + - `hugegraph.properties` - Main server configuration + - `rest-server.properties` - REST API settings + - `gremlin-server.yaml` - Gremlin server configuration +- PD configs: `hugegraph-pd/hg-pd-dist/src/assembly/static/conf/` +- Store configs: `hugegraph-store/hg-store-dist/src/assembly/static/conf/` + +### Proto Definitions (gRPC) +- PD protos: `hugegraph-pd/hg-pd-grpc/src/main/proto/` +- Store protos: `hugegraph-store/hg-store-grpc/src/main/proto/` + +### Core Implementation Paths +- Graph engine: `hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/` +- REST APIs: `hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/` +- Backend implementations: `hugegraph-server/hugegraph-{backend}/` (e.g., `hugegraph-rocksdb`) + +## Development Workflow + +### Code Style +Import the code style configuration from `hugegraph-style.xml` in your IDE (IntelliJ IDEA recommended). + +### Adding Dependencies + +When adding third-party dependencies: +1. Add license files to `install-dist/release-docs/licenses/` +2. Declare dependency in `install-dist/release-docs/LICENSE` +3. Append NOTICE info to `install-dist/release-docs/NOTICE` (if upstream has NOTICE) +4. Update `install-dist/scripts/dependency/known-dependencies.txt` (run `regenerate_known_dependencies.sh`) + +### Backend Development + +When working on storage backends: +- All backends extend `hugegraph-server/hugegraph-core` abstractions +- Implement the `BackendStore` interface +- Each backend is a separate Maven module in `hugegraph-server/` +- Backend selection is configured in `hugegraph.properties` via the `backend` property + +### gRPC Protocol Changes + +When modifying `.proto` files: +- Generated Java code goes to `*/grpc/` packages (excluded from Apache RAT checks) +- Run `mvn clean compile` to regenerate gRPC stubs +- Generated files are in `target/generated-sources/protobuf/` + +### Authentication System + +Authentication is optional and disabled by default: +- Enable via `bin/enable-auth.sh` or configuration +- Auth implementation: `hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/auth/` +- Multi-level: Users, Groups, Projects, Targets, Access control +- Required for production deployments + +## Common Workflows + +### Running a Single Test Class +```bash +# Use Maven's -Dtest parameter +mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,memory -Dtest=YourTestClass +``` + +### Working with Distributed Components + +The distributed architecture (PD + Store) is in BETA. For distributed development: +1. Build struct module first: `mvn install -pl hugegraph-struct -am -DskipTests` +2. Build PD: `mvn clean package -pl hugegraph-pd -am -DskipTests` +3. Build Store: `mvn clean package -pl hugegraph-store -am -DskipTests` +4. Build Server with HStore backend: `mvn clean package -pl hugegraph-server -am -DskipTests` + +See Docker Compose example: `hugegraph-server/hugegraph-dist/docker/example/` + +### Debugging Tips + +- Enable detailed logging in `hugegraph-server/hugegraph-dist/src/assembly/static/conf/log4j2.xml` +- Use `bin/dump-conf.sh` to view effective configuration +- Arthas diagnostics tool is included (version 3.7.1) +- Monitor with `bin/monitor-hugegraph.sh` + +## CI/CD Profiles + +The project uses multiple GitHub Actions workflows: +- `server-ci.yml`: Server module tests (memory, rocksdb, hbase backends) +- `pd-store-ci.yml`: PD and Store module tests +- `commons-ci.yml`: Commons module tests +- `cluster-test-ci.yml`: Distributed cluster integration tests +- `licence-checker.yml`: Apache RAT license validation + +## Special Notes + +### Cross-Module Dependencies +- `hugegraph-commons` is a shared dependency for all modules +- `hugegraph-struct` must be built before PD and Store +- Server backends depend on `hugegraph-core` + +### Version Management +- Version is managed via `${revision}` property (currently `1.7.0`) +- Flatten Maven plugin used for CI-friendly versioning diff --git a/LICENSE b/LICENSE index c8b9d6ed04..8445ec58dc 100644 --- a/LICENSE +++ b/LICENSE @@ -216,3 +216,5 @@ hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/C hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/StringEncoding.java from https://github.com/JanusGraph/janusgraph hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherOpProcessor.java from https://github.com/opencypher/cypher-for-gremlin hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java from https://github.com/opencypher/cypher-for-gremlin +hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java from https://github.com/JanusGraph/janusgraph +hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java from https://github.com/JanusGraph/janusgraph diff --git a/hugegraph-commons/AGENTS.md b/hugegraph-commons/AGENTS.md new file mode 100644 index 0000000000..c21ab4a5dc --- /dev/null +++ b/hugegraph-commons/AGENTS.md @@ -0,0 +1,285 @@ +# AGENTS.md + +This file provides guidance to an AI coding tool when working with code in this repository. + +## Project Overview + +hugegraph-commons is a shared utility module for Apache HugeGraph and its peripheral components. It provides core infrastructure components (locks, config, events, iterators, REST client, RPC framework) to simplify development across the HugeGraph ecosystem. + +**Technology Stack**: +- Java 8+ (compiler source/target: 1.8) +- Apache Maven 3.5+ +- Apache Commons Configuration2 for config management +- OkHttp 4.10.0 for REST client (hugegraph-common) +- Sofa-RPC 5.7.6 for RPC framework (hugegraph-rpc) +- JUnit 4.13.1 and Mockito 4.1.0 for testing + +## Architecture + +### Two-Module Structure + +This is a Maven multi-module project with 2 main modules: + +1. **hugegraph-common**: Core utilities library + - Lock implementations (atomic, key, row, lock groups) + - Configuration system with type-safe options + - Event hub for async notifications + - Iterator utilities (map, filter, flat-map, batch) + - RESTful client (OkHttp-based) + - Utilities (perf analysis, version checking, collections, logging) + - License management + +2. **hugegraph-rpc**: RPC communication framework + - Sofa-RPC based client/server implementation + - Consumer and provider configuration + - Service registration and discovery + - **Depends on hugegraph-common** + +### Key Design Patterns + +1. **Type-Safe Configuration System**: `HugeConfig` + `OptionSpace` pattern + - Config options defined as typed `ConfigOption` objects + - Supports both `.properties` and `.yaml` files + - Options organized in `OptionSpace` groups for validation + - Security checks on load + +2. **Lock Hierarchy**: Multiple lock implementations for different use cases + - `AtomicLock`: Basic atomic locking + - `KeyLock`: Lock by specific key + - `RowLock`: Row-level locking for table-like structures + - `LockGroup`: Manage multiple related locks + - `LockManager`: Central lock coordination + +3. **Event System**: Async event notification + - `EventHub`: Central event dispatcher + - `EventListener`: Typed event handlers + - Thread-safe event publishing + +4. **Iterator Composition**: Chainable iterator wrappers + - `MapperIterator`, `FilterIterator`, `LimitIterator` + - `FlatMapperIterator` for nested iteration + - `BatchMapperIterator` for batch processing + - All extend `ExtendableIterator` base + +5. **RPC Architecture**: Sofa-RPC abstraction layer + - `RpcServer`: Service provider side + - `RpcClientProvider`: Service consumer side + - `RpcProviderConfig`/`RpcConsumerConfig`: Configuration wrappers + - Supports multiple protocols (bolt, rest, grpc) + +## Build & Development Commands + +### Prerequisites +```bash +# Verify Java version (8+ required) +java -version + +# Verify Maven version (3.5+ required) +mvn -version +``` + +### Build Commands + +```bash +# Clean build without tests (fastest) +mvn clean install -DskipTests + +# Build with tests enabled +mvn clean install + +# Build specific module only +mvn clean install -pl hugegraph-common -DskipTests +mvn clean install -pl hugegraph-rpc -am -DskipTests # -am includes dependencies + +# Compile with warnings visible +mvn clean compile -Dmaven.javadoc.skip=true +``` + +**Note**: Tests are skipped by default via `true` in pom.xml. To run tests, override with `-DskipCommonsTests=false`. + +### Testing + +```bash +# Run all tests (override default skip) +mvn test -DskipCommonsTests=false + +# Run tests for specific module +mvn test -pl hugegraph-common -DskipCommonsTests=false +mvn test -pl hugegraph-rpc -am -DskipCommonsTests=false + +# Run single test class +mvn test -pl hugegraph-common -Dtest=HugeConfigTest -DskipCommonsTests=false + +# Run test suite (includes all unit tests) +mvn test -pl hugegraph-common -Dtest=UnitTestSuite -DskipCommonsTests=false +``` + +### Code Quality + +```bash +# License header check (Apache RAT) +mvn apache-rat:check + +# Checkstyle validation +mvn checkstyle:check + +# Both checks run automatically during validate phase +mvn validate +``` + +### Code Coverage + +```bash +# Generate JaCoCo coverage report +mvn clean test -DskipCommonsTests=false +# Report: target/jacoco/index.html +``` + +## Important File Locations + +### Source Code Structure +- hugegraph-common sources: `hugegraph-common/src/main/java/org/apache/hugegraph/` + - `concurrent/`: Lock implementations + - `config/`: Configuration system (HugeConfig, OptionSpace, ConfigOption) + - `event/`: Event hub and listeners + - `iterator/`: Iterator utilities + - `rest/`: REST client implementation + - `util/`: Various utilities (collections, logging, version, etc.) + - `perf/`: Performance measurement (PerfUtil, Stopwatch) + - `license/`: License management + +- hugegraph-rpc sources: `hugegraph-rpc/src/main/java/org/apache/hugegraph/` + - `rpc/`: RPC server and client implementations + - `config/`: RPC-specific config options + +### Test Structure +- Unit tests: `hugegraph-{module}/src/test/java/org/apache/hugegraph/unit/` +- Test suites: `UnitTestSuite.java` lists all test classes +- Test utilities: `hugegraph-common/src/main/java/org/apache/hugegraph/testutil/` + - `Whitebox`: Reflection utilities for testing private members + - `Assert`: Enhanced assertion utilities + +### Configuration Files +- Parent POM: `pom.xml` (defines all dependencies and versions) +- Module POMs: `hugegraph-{module}/pom.xml` +- Test resources: `hugegraph-rpc/src/test/resources/*.properties` +- Checkstyle config: `style/checkstyle.xml` (referenced in parent POM) + +### Version Management +- Version property: `${revision}` in pom.xml (currently 1.7.0) +- Version classes: + - `hugegraph-common/src/main/java/org/apache/hugegraph/version/CommonVersion.java` + - `hugegraph-rpc/src/main/java/org/apache/hugegraph/version/RpcVersion.java` +- **IMPORTANT**: When changing version in pom.xml, also update version in these Java files + +## Development Workflow + +### Module Dependencies + +**Dependency order**: +``` +hugegraph-common (no internal dependencies) + ↓ +hugegraph-rpc (depends on hugegraph-common) +``` + +When making changes: +- Changes to `hugegraph-common` require rebuilding `hugegraph-rpc` +- Always build common first: `mvn install -pl hugegraph-common -DskipTests` +- Then build rpc: `mvn install -pl hugegraph-rpc -am -DskipTests` + +### Working with Configuration System + +When adding new configuration options: +1. Define `ConfigOption` in appropriate config class (e.g., `RpcOptions.java`) +2. Register option in an `OptionSpace` for validation +3. Load via `HugeConfig.get(option)` or `config.get(option)` +4. Example: +```java +public static final ConfigOption RPC_SERVER_HOST = + new ConfigOption<>("rpc.server_host", "...", "127.0.0.1"); +``` + +### Working with RPC Framework + +RPC configuration pattern: +- Server side: Create `RpcServer` with `HugeConfig` +- Client side: Use `RpcClientProvider` for service proxies +- Config files: Properties format with `rpc.*` keys +- Protocol: Default is Sofa-Bolt (binary protocol) + +### Testing Patterns + +1. **Unit tests** extend `BaseUnitTest` (for common module) or use standard JUnit +2. **Test organization**: Tests mirror source package structure +3. **Naming**: `{ClassName}Test.java` for unit tests +4. **Mocking**: Use Mockito for external dependencies +5. **Reflection testing**: Use `Whitebox.setInternalState()` for private field access + +### Adding Dependencies + +When adding third-party dependencies: +1. Add to `dependencyManagement` section in parent pom if used by multiple modules +2. Declare version in `` section +3. Add license info to `hugegraph-dist/release-docs/licenses/` +4. Update `hugegraph-dist/release-docs/LICENSE` +5. Update `hugegraph-dist/release-docs/NOTICE` if upstream has NOTICE + +## Common Workflows + +### Running a Specific Test + +```bash +# Single test class +mvn test -pl hugegraph-common -Dtest=HugeConfigTest -DskipCommonsTests=false + +# Single test method +mvn test -pl hugegraph-common -Dtest=HugeConfigTest#testGetOption -DskipCommonsTests=false + +# Pattern matching +mvn test -pl hugegraph-common -Dtest=*ConfigTest -DskipCommonsTests=false +``` + +### Debugging Tips + +1. **Enable debug logging**: Modify `log4j2.xml` in test resources +2. **Maven debug output**: Add `-X` flag to any Maven command +3. **Skip checkstyle temporarily**: Add `-Dcheckstyle.skip=true` +4. **Force dependency updates**: `mvn clean install -U` + +### Working with Parent POM + +This module has a parent POM (`../pom.xml` - hugegraph main project). If working standalone: +- The `` property comes from parent (1.7.0) +- Flatten Maven plugin resolves `${revision}` to actual version +- `.flattened-pom.xml` is auto-generated (excluded from RAT checks) + +## Special Notes + +### Version Synchronization + +Three places to update when changing version: +1. `pom.xml`: `` property +2. `hugegraph-common/.../CommonVersion.java`: Update version constant +3. `hugegraph-rpc/.../RpcVersion.java`: Update version constant + +### REST Client Implementation + +The REST client in `hugegraph-common/rest/` uses OkHttp (not Jersey as older docs suggest): +- Switched from Jersey to OkHttp in recent versions +- Supports connection pooling, timeouts, interceptors +- See `AbstractRestClient.java` for base implementation + +### RPC Version Note + +hugegraph-rpc uses Sofa-RPC 5.7.6 which has known security issues. There's a TODO to upgrade to 5.12+: +- See comment in `hugegraph-rpc/pom.xml:65-66` +- This is a known technical debt item + +### Checkstyle Configuration + +Checkstyle runs during `validate` phase by default: +- Config: `style/checkstyle.xml` +- Failures block the build +- Skip with `-Dcheckstyle.skip=true` for quick iteration +- Always fix before committing diff --git a/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/rest/RestResult.java b/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/rest/RestResult.java index 0aa482b067..64ed10d1ce 100644 --- a/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/rest/RestResult.java +++ b/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/rest/RestResult.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.rest; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; @@ -25,6 +26,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import lombok.SneakyThrows; import okhttp3.Response; @@ -33,6 +35,12 @@ public class RestResult { private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // NOTE: jackson will synchronize DateFormat + MAPPER.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")); + } + private final int status; private final RestHeaders headers; private final String content; diff --git a/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/DateUtil.java b/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/DateUtil.java index 4e7ce13de0..39c44031f2 100644 --- a/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/DateUtil.java +++ b/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/DateUtil.java @@ -22,6 +22,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.hugegraph.date.SafeDateFormat; + import com.google.common.collect.ImmutableMap; public final class DateUtil { @@ -46,7 +47,7 @@ public static Date parse(String value) { } } throw new IllegalArgumentException(String.format( - "Expected date format is: %s, but got '%s'", VALID_DFS.values(), value)); + "Expected date format is: %s, but got '%s'", VALID_DFS.values(), value)); } public static Date parse(String value, String df) { diff --git a/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/JsonUtilCommon.java b/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/JsonUtilCommon.java index ad0acebeec..b7fdd75067 100644 --- a/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/JsonUtilCommon.java +++ b/hugegraph-commons/hugegraph-common/src/main/java/org/apache/hugegraph/util/JsonUtilCommon.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.util; import java.io.IOException; +import java.text.SimpleDateFormat; import org.apache.hugegraph.rest.SerializeException; @@ -25,6 +26,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; /** * Utility class for JSON operations. @@ -36,6 +38,12 @@ public final class JsonUtilCommon { */ private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + // NOTE: jackson will synchronize DateFormat + MAPPER.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")); + } + /** * Registers a module with the ObjectMapper. * diff --git a/hugegraph-commons/hugegraph-common/src/main/resources/version.properties b/hugegraph-commons/hugegraph-common/src/main/resources/version.properties index b413bf04e3..b6763a32a9 100644 --- a/hugegraph-commons/hugegraph-common/src/main/resources/version.properties +++ b/hugegraph-commons/hugegraph-common/src/main/resources/version.properties @@ -19,5 +19,5 @@ Version=${revision} ApiVersion=0.71 ApiCheckBeginVersion=1.0 -ApiCheckEndVersion=1.7 -VersionInBash=1.5.0 +ApiCheckEndVersion=2.0 +VersionInBash=1.7.0 diff --git a/hugegraph-commons/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-commons/hugegraph-dist/scripts/dependency/known-dependencies.txt index 9a421edcd4..5db5f373f8 100644 --- a/hugegraph-commons/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-commons/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -1,4 +1,7 @@ +animal-sniffer-annotations-1.18.jar annotations-13.0.jar +annotations-4.1.1.4.jar +bolt-1.6.2.jar checker-qual-3.5.0.jar commons-beanutils-1.9.4.jar commons-codec-1.13.jar @@ -10,14 +13,25 @@ commons-lang-2.6.jar commons-lang3-3.12.0.jar commons-logging-1.1.1.jar commons-text-1.9.jar +disruptor-3.3.7.jar error_prone_annotations-2.3.4.jar failureaccess-1.0.1.jar +grpc-api-1.28.1.jar +grpc-context-1.28.1.jar +grpc-core-1.28.1.jar +grpc-netty-shaded-1.28.0.jar +grpc-protobuf-1.28.0.jar +grpc-protobuf-lite-1.28.0.jar +grpc-stub-1.28.0.jar +gson-2.8.6.jar guava-30.0-jre.jar hamcrest-core-1.3.jar +hessian-3.3.7.jar j2objc-annotations-1.3.jar jackson-annotations-2.14.0-rc1.jar jackson-core-2.14.0-rc1.jar jackson-databind-2.14.0-rc1.jar +jackson-dataformat-yaml-2.9.3.jar jackson-jaxrs-base-2.14.0-rc1.jar jackson-jaxrs-json-provider-2.14.0-rc1.jar jackson-module-jaxb-annotations-2.14.0-rc1.jar @@ -39,7 +53,23 @@ log4j-api-2.18.0.jar log4j-core-2.18.0.jar log4j-slf4j-impl-2.18.0.jar logging-interceptor-4.10.0.jar -lombok-1.18.8.jar +lookout-api-1.4.1.jar +netty-all-4.1.42.Final.jar okhttp-4.10.0.jar okio-jvm-3.0.0.jar +opentracing-api-0.22.0.jar +opentracing-mock-0.22.0.jar +opentracing-noop-0.22.0.jar +opentracing-util-0.22.0.jar +perfmark-api-0.19.0.jar +proto-google-common-protos-1.17.0.jar +protobuf-java-3.11.0.jar slf4j-api-1.7.25.jar +snakeyaml-1.18.jar +sofa-common-tools-1.0.12.jar +sofa-rpc-all-5.7.6.jar +swagger-annotations-1.5.18.jar +swagger-core-1.5.18.jar +swagger-models-1.5.18.jar +tracer-core-3.0.8.jar +validation-api-1.1.0.Final.jar diff --git a/hugegraph-commons/pom.xml b/hugegraph-commons/pom.xml index 09cd71e5a7..59d12b99ad 100644 --- a/hugegraph-commons/pom.xml +++ b/hugegraph-commons/pom.xml @@ -90,7 +90,8 @@ - 1.5.0 + + 1.7.0 UTF-8 ${project.basedir}/.. 1.8 diff --git a/hugegraph-pd/AGENTS.md b/hugegraph-pd/AGENTS.md new file mode 100644 index 0000000000..e1d915491d --- /dev/null +++ b/hugegraph-pd/AGENTS.md @@ -0,0 +1,378 @@ +# AGENTS.md + +This file provides guidance to an AI coding tool when working with code in this repository. + +## Project Overview + +HugeGraph PD (Placement Driver) is a meta server for distributed HugeGraph deployments, responsible for: +- Service discovery and registration +- Partition information storage and management +- Store node monitoring and scheduling +- Metadata coordination using Raft consensus + +**Status**: BETA (since HugeGraph 1.5.0) + +**Technology Stack**: +- Java 11+ (required) +- Apache Maven 3.5+ +- gRPC + Protocol Buffers for RPC communication +- JRaft (Ant Design's Raft implementation) for consensus +- RocksDB for metadata persistence +- Spring Boot for REST APIs and dependency injection + +## Module Architecture + +HugeGraph PD is a Maven multi-module project with 8 modules: + +``` +hugegraph-pd/ +├── hg-pd-grpc # Protocol Buffers definitions and generated stubs +├── hg-pd-common # Shared utilities and common interfaces +├── hg-pd-core # Core PD logic (Raft, metadata, services) +├── hg-pd-service # gRPC service implementations and REST APIs +├── hg-pd-client # Java client library for PD +├── hg-pd-cli # Command-line interface tools +├── hg-pd-test # Integration and unit tests +└── hg-pd-dist # Distribution assembly (bin scripts, configs) +``` + +### Module Dependency Flow + +``` +hg-pd-grpc (protos) + ↓ +hg-pd-common (utilities) + ↓ +hg-pd-core (Raft + metadata stores) + ↓ +hg-pd-service (gRPC + REST endpoints) + ↓ +hg-pd-dist (assembly) + +hg-pd-client (depends on hg-pd-grpc, hg-pd-common) +hg-pd-cli (depends on hg-pd-client) +hg-pd-test (depends on hg-pd-core, hg-pd-service) +``` + +### Core Components (hg-pd-core) + +**Metadata Stores** (`meta/` package): +- `MetadataRocksDBStore`: RocksDB-backed persistence layer +- `PartitionMeta`: Partition assignment and shard group management +- `StoreInfoMeta`: Store node information and health status +- `TaskInfoMeta`: Distributed task coordination +- `IdMetaStore`, `ConfigMetaStore`, `DiscoveryMetaStore`: Domain-specific metadata + +**Services** (root package): +- `PartitionService`: Partition allocation, balancing, and splitting +- `StoreNodeService`: Store registration, heartbeat processing, status monitoring +- `StoreMonitorDataService`: Metrics collection and time-series data storage +- `TaskScheduleService`: Automated partition patrol and rebalancing +- `KvService`, `IdService`, `ConfigService`, `LogService`: Utility services + +**Raft Layer** (`raft/` package): +- `RaftEngine`: Raft group management and leadership +- `RaftStateMachine`: State machine applying metadata operations +- `RaftTaskHandler`: Async task execution via Raft proposals +- `KVOperation`, `KVStoreClosure`: Raft operation abstractions + +**Service Layer** (hg-pd-service): +- `ServiceGrpc`: Main gRPC service endpoint (partition, store, discovery RPCs) +- `PartitionAPI`: REST API for partition management +- `PDPulseService`: Heartbeat and monitoring +- `DiscoveryService`: Service discovery and registration + +### gRPC Protocol Definitions + +Located in `hg-pd-grpc/src/main/proto/`: +- `pdpb.proto`: Main PD service (GetMembers, RegisterStore, GetPartition) +- `metapb.proto`: Metadata objects (Partition, Shard, Store, Graph) +- `meta.proto`: Store and partition metadata +- `discovery.proto`: Service discovery protocol +- `kv.proto`: Key-value operations +- `pd_pulse.proto`: Heartbeat and monitoring +- `pd_watch.proto`: Watch notifications +- `metaTask.proto`: Task coordination + +**Important**: Generated Java code from `.proto` files is excluded from source control (`.gitignore`) and Apache RAT checks (Jacoco config). Regenerate after proto changes. + +## Build & Development Commands + +### Building PD Module + +```bash +# From hugegraph root directory, build PD and dependencies +mvn clean package -pl hugegraph-pd -am -DskipTests + +# From hugegraph-pd directory, build all modules +mvn clean install -DskipTests + +# Build with tests +mvn clean install + +# Build distribution package only +mvn clean package -pl hg-pd-dist -am -DskipTests +# Output: hg-pd-dist/target/apache-hugegraph-pd-incubating-.tar.gz +``` + +### Running Tests + +PD tests use Maven profiles defined in `pom.xml`: + +```bash +# All tests (default profiles active) +mvn test + +# Specific test profile +mvn test -P pd-core-test +mvn test -P pd-common-test +mvn test -P pd-client-test +mvn test -P pd-rest-test + +# Single test class (from hugegraph-pd directory) +mvn test -pl hg-pd-test -am -Dtest=StoreNodeServiceTest +mvn test -pl hg-pd-test -am -Dtest=PartitionServiceTest + +# From hugegraph root directory +mvn test -pl hugegraph-pd/hg-pd-test -am +``` + +Test files are located in `hg-pd-test/src/main/java/` (note: not `src/test/java`). + +### Regenerating gRPC Stubs + +```bash +# After modifying .proto files +mvn clean compile + +# Generated files location: +# target/generated-sources/protobuf/java/ +# target/generated-sources/protobuf/grpc-java/ +``` + +### Code Quality + +```bash +# License header check (Apache RAT) +mvn apache-rat:check + +# Clean build artifacts +mvn clean +# This also removes: *.tar, *.tar.gz, .flattened-pom.xml +``` + +## Running HugeGraph PD + +### Distribution Structure + +After building, extract the tarball: +``` +apache-hugegraph-pd-incubating-/ +├── bin/ +│ ├── start-hugegraph-pd.sh # Start PD server +│ ├── stop-hugegraph-pd.sh # Stop PD server +│ └── util.sh # Utility functions +├── conf/ +│ ├── application.yml # Main configuration +│ ├── application.yml.template # Configuration template +│ ├── log4j2.xml # Logging configuration +│ └── verify-license.json # License verification (optional) +├── lib/ # JAR dependencies +├── logs/ # Runtime logs +└── pd_data/ # RocksDB metadata storage (created at runtime) +``` + +### Starting PD + +```bash +cd apache-hugegraph-pd-incubating-/ +bin/start-hugegraph-pd.sh + +# With custom GC options +bin/start-hugegraph-pd.sh -g g1 + +# With custom JVM options +bin/start-hugegraph-pd.sh -j "-Xmx8g -Xms4g" + +# With OpenTelemetry enabled +bin/start-hugegraph-pd.sh -y true +``` + +Default ports: +- gRPC: 8686 (configure in `application.yml`: `grpc.port`) +- REST API: 8620 (configure in `application.yml`: `server.port`) +- Raft: 8610 (configure in `application.yml`: `raft.address`) + +JVM memory defaults (in `start-hugegraph-pd.sh`): +- Max heap: 32 GB +- Min heap: 512 MB + +### Stopping PD + +```bash +bin/stop-hugegraph-pd.sh +``` + +This sends SIGTERM to the PD process (tracked in `bin/pid`). + +## Key Configuration (application.yml) + +### Critical Settings for Distributed Deployment + +```yaml +grpc: + host: 127.0.0.1 # MUST change to actual IPv4 address in production + port: 8686 + +raft: + address: 127.0.0.1:8610 # This node's Raft address + peers-list: 127.0.0.1:8610 # Comma-separated list of all PD peers + # Example: 192.168.1.1:8610,192.168.1.2:8610,192.168.1.3:8610 + +pd: + data-path: ./pd_data # RocksDB metadata storage path + initial-store-count: 1 # Min stores required for cluster availability + initial-store-list: 127.0.0.1:8500 # Auto-activated store nodes (grpc_ip:grpc_port) + patrol-interval: 1800 # Partition rebalancing interval (seconds) + +partition: + default-shard-count: 1 # Replicas per partition (typically 3 in production) + store-max-shard-count: 12 # Max partitions per store + +store: + max-down-time: 172800 # Seconds before store is permanently offline (48h) + monitor_data_enabled: true # Enable metrics collection + monitor_data_interval: 1 minute # Metrics collection interval + monitor_data_retention: 1 day # Metrics retention period +``` + +### Common Configuration Errors + +1. **Raft peer discovery failure**: `raft.peers-list` must include all PD nodes' `raft.address` values +2. **Store connection issues**: `grpc.host` must be a reachable IP (not `127.0.0.1`) for distributed deployments +3. **Split-brain scenarios**: Always run 3 or 5 PD nodes in production for Raft quorum +4. **Partition imbalance**: Adjust `patrol-interval` for faster/slower rebalancing + +## Development Workflows + +### Adding a New gRPC Service + +1. Define `.proto` messages and service in `hg-pd-grpc/src/main/proto/` +2. Run `mvn compile` to generate Java stubs +3. Implement service in `hg-pd-service/src/main/java/.../service/` +4. Register service in gRPC server initialization (check existing `ServiceGrpc.java` pattern) +5. Add client methods in `hg-pd-client/` if needed + +### Adding a New Metadata Store + +1. Create meta class in `hg-pd-core/src/main/java/.../meta/` +2. Use `MetadataRocksDBStore` as the underlying persistence layer +3. Implement metadata operations as Raft proposals via `RaftTaskHandler` +4. Add corresponding service methods in `hg-pd-core/.../Service.java` +5. Expose via gRPC in `hg-pd-service/` + +### Modifying Partition Logic + +- Core partition logic: `hg-pd-core/.../PartitionService.java` (69KB file, 2000+ lines) +- Key methods: + - `splitPartition()`: Partition splitting logic + - `updatePartitionLeader()`: Leader election handling + - `balancePartitions()`: Auto-balancing algorithm + - `getPartitionByCode()`: Partition routing +- All partition changes must go through Raft consensus +- Test with `hg-pd-test/.../core/PartitionServiceTest.java` + +### Debugging Raft Issues + +- Enable Raft logging in `conf/log4j2.xml`: Set `com.alipay.sofa.jraft` to DEBUG +- Check Raft state: Leader election happens in `RaftEngine.java` +- Raft snapshots stored in `pd_data/raft/snapshot/` +- Raft logs stored in `pd_data/raft/log/` + +## Testing Strategy + +### Test Organization + +Tests are in `hg-pd-test/src/main/java/` (non-standard location): +- `BaseTest.java`: Base class with common setup +- `core/`: Core service tests (PartitionService, StoreNodeService, etc.) +- Suite tests: `PDCoreSuiteTest.java` runs all core tests + +### Running Integration Tests + +```bash +# From hugegraph root, run PD integration tests +mvn test -pl hugegraph-pd/hg-pd-test -am + +# These tests start embedded PD instances and verify: +# - Raft consensus and leader election +# - Partition allocation and balancing +# - Store heartbeat and monitoring +# - Metadata persistence and recovery +``` + +## Docker Deployment + +### Building Docker Image + +```bash +# From hugegraph root directory +docker build -f hugegraph-pd/Dockerfile -t hugegraph-pd:latest . +``` + +The Dockerfile uses multi-stage build: +1. Stage 1: Build with Maven +2. Stage 2: Runtime with OpenJDK 11 + +### Running in Docker + +```bash +# Single PD node (development) +docker run -d -p 8620:8620 -p 8686:8686 -p 8610:8610 \ + -v /path/to/pd_data:/hugegraph-pd/pd_data \ + hugegraph-pd:latest + +# For production clusters, use Docker Compose or Kubernetes +# See: hugegraph-server/hugegraph-dist/docker/example/ +``` + +Exposed ports: 8620 (REST), 8686 (gRPC), 8610 (Raft) + +## Cross-Module Dependencies + +When working on PD: +- **hugegraph-struct**: Protocol definitions shared with hugegraph-store + - Build struct first: `mvn install -pl hugegraph-struct -am -DskipTests` + - Required before building PD if struct changed +- **hugegraph-commons**: RPC framework, locks, and configuration utilities + - Changes to commons may affect PD's `hg-pd-common` module + +## CI/CD Integration + +PD tests run in `pd-store-ci.yml` GitHub Actions workflow: +- Triggered on pushes to PD module files +- Runs `mvn test -pl hugegraph-pd/hg-pd-test -am` +- JaCoCo coverage excludes generated gRPC code and config classes + +## Important Notes + +### Generated Code Exclusions +- `hg-pd-grpc/src/main/java/` is excluded from git (see `.gitignore`) +- Apache RAT skips `**/grpc/**.*` (see `pom.xml` Jacoco config) +- Always run `mvn clean compile` after pulling proto changes + +### Raft Consensus Requirements +- PD uses JRaft for distributed consensus +- All metadata writes are Raft proposals (see `KVOperation`, `KVStoreClosure`) +- Raft group requires 3 or 5 nodes for fault tolerance in production +- Single-node mode (peers-list with one address) is for development only + +### Store Interaction +- PD does not store graph data; it only stores metadata about store nodes and partitions +- Actual graph data resides in hugegraph-store nodes +- PD coordinates store nodes but doesn't handle data plane traffic + +### Version Compatibility +- PD version must match hugegraph-server and hugegraph-store versions +- Version managed via `${revision}` property (inherited from parent POM) +- Current version: 1.7.0 diff --git a/hugegraph-pd/README.md b/hugegraph-pd/README.md index 1aea07d7dd..3ff14b9e2a 100644 --- a/hugegraph-pd/README.md +++ b/hugegraph-pd/README.md @@ -1,8 +1,276 @@ -> Note: From revision 1.5.0, the code of HugeGraph-PD will be adapted to this location (WIP). +# HugeGraph PD -# HugeGraph PD (BETA) +[![License](https://img.shields.io/badge/license-Apache%202-0E78BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) +[![Version](https://img.shields.io/badge/version-1.7.0-blue)](https://github.com/apache/hugegraph) -HugeGraph PD is a meta server responsible for service discovery, partition information storage, and -node scheduling. +> **Note**: From revision 1.5.0, the HugeGraph-PD code has been adapted to this location. -> BTW, if you meet any problem when using HugeGraph PD, please feel free to contact us for help +## Overview + +HugeGraph PD (Placement Driver) is a meta server that provides cluster management and coordination services for HugeGraph distributed deployments. It serves as the central control plane responsible for: + +- **Service Discovery**: Automatic registration and discovery of Store and Server nodes +- **Partition Management**: Dynamic partition allocation, balancing, and rebalancing across Store nodes +- **Metadata Storage**: Centralized storage of cluster metadata, configuration, and state information +- **Node Scheduling**: Intelligent scheduling and load balancing of graph operations +- **Health Monitoring**: Continuous health checks and failure detection via heartbeat mechanism + +PD uses [Apache JRaft](https://github.com/sofastack/sofa-jraft) for Raft consensus and RocksDB for persistent metadata storage, ensuring high availability and consistency in distributed environments. + +## Architecture + +HugeGraph PD is a Maven multi-module project consisting of 8 modules: + +| Module | Description | +|--------|-------------| +| **hg-pd-grpc** | gRPC protocol definitions (`.proto` files) and generated Java stubs for inter-service communication | +| **hg-pd-common** | Shared utilities, constants, and helper classes used across PD modules | +| **hg-pd-core** | Core PD logic: Raft integration, metadata stores, partition allocation, store monitoring, task scheduling | +| **hg-pd-service** | gRPC service implementations and REST API (Spring Boot) for management and metrics | +| **hg-pd-client** | Java client library for applications to communicate with PD cluster | +| **hg-pd-cli** | Command-line utilities for PD administration and debugging | +| **hg-pd-test** | Unit and integration tests for all PD components | +| **hg-pd-dist** | Distribution assembly: packaging, configuration templates, startup scripts | + +For detailed architecture and design, see [Architecture Documentation](docs/architecture.md). + +## Quick Start + +### Prerequisites + +- **Java**: 11 or higher +- **Maven**: 3.5 or higher +- **Disk Space**: At least 1GB for PD data directory + +### Build + +From the project root (build PD and all dependencies): + +```bash +mvn clean package -pl hugegraph-pd -am -DskipTests +``` + +Or build from the `hugegraph-pd` directory: + +```bash +cd hugegraph-pd +mvn clean install -DskipTests +``` + +The assembled distribution will be available at: +``` +hugegraph-pd/hg-pd-dist/target/hugegraph-pd-.tar.gz +``` + +### Run + +Extract the distribution package and start PD: + +```bash +tar -xzf hugegraph-pd-.tar.gz +cd hugegraph-pd- + +# Start PD server +bin/start-hugegraph-pd.sh + +# Stop PD server +bin/stop-hugegraph-pd.sh +``` + +#### Startup Options + +```bash +bin/start-hugegraph-pd.sh [-g GC_TYPE] [-j "JVM_OPTIONS"] [-y ENABLE_OTEL] +``` + +- `-g`: GC type (`g1` or `ZGC`, default: `g1`) +- `-j`: Custom JVM options (e.g., `-j "-Xmx4g -Xms4g"`) +- `-y`: Enable OpenTelemetry tracing (`true` or `false`, default: `false`) + +### Configuration + +Key configuration file: `conf/application.yml` + +#### Core Settings + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `grpc.host` | `127.0.0.1` | gRPC server bind address (use actual IP for production) | +| `grpc.port` | `8686` | gRPC server port | +| `server.port` | `8620` | REST API port for management and metrics | +| `raft.address` | `127.0.0.1:8610` | Raft service address for this PD node | +| `raft.peers-list` | `127.0.0.1:8610` | Comma-separated list of all PD nodes in the Raft cluster | +| `pd.data-path` | `./pd_data` | Directory for storing PD metadata and Raft logs | + +#### Single-Node Example + +```yaml +grpc: + host: 127.0.0.1 + port: 8686 + +server: + port: 8620 + +raft: + address: 127.0.0.1:8610 + peers-list: 127.0.0.1:8610 + +pd: + data-path: ./pd_data +``` + +#### 3-Node Cluster Example + +For a production 3-node PD cluster, configure each node: + +**Node 1** (`192.168.1.10`): +```yaml +grpc: + host: 192.168.1.10 + port: 8686 +raft: + address: 192.168.1.10:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 +``` + +**Node 2** (`192.168.1.11`): +```yaml +grpc: + host: 192.168.1.11 + port: 8686 +raft: + address: 192.168.1.11:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 +``` + +**Node 3** (`192.168.1.12`): +```yaml +grpc: + host: 192.168.1.12 + port: 8686 +raft: + address: 192.168.1.12:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 +``` + +For detailed configuration options and production tuning, see [Configuration Guide](docs/configuration.md). + +### Verify Deployment + +Check if PD is running: + +```bash +# Check process +ps aux | grep hugegraph-pd + +# Test REST API +curl http://localhost:8620/actuator/health + +# Check logs +tail -f logs/hugegraph-pd.log +``` + +## gRPC API + +PD exposes several gRPC services for cluster management. Key services include: + +- **PD Service** (`PDGrpc`): Store registration, partition queries, leader election +- **KV Service** (`KvServiceGrpc`): Distributed key-value operations for metadata +- **Watch Service** (`HgPdWatchGrpc`): Watch for partition and store changes +- **Pulse Service** (`HgPdPulseGrpc`): Heartbeat and health monitoring + +Proto definitions are located in: +``` +hugegraph-pd/hg-pd-grpc/src/main/proto/ +``` + +For API reference and usage examples, see [API Reference](docs/api-reference.md). + +## Testing + +Run PD tests: + +```bash +# All PD tests +mvn test -pl hugegraph-pd/hg-pd-test -am + +# Specific test class +mvn test -pl hugegraph-pd/hg-pd-test -am -Dtest=YourTestClass +``` + +## Docker + +Build PD Docker image: + +```bash +# From project root +docker build -f hugegraph-pd/Dockerfile -t hugegraph-pd:latest . + +# Run container +docker run -d \ + -p 8620:8620 \ + -p 8686:8686 \ + -p 8610:8610 \ + -v /path/to/conf:/hugegraph-pd/conf \ + -v /path/to/data:/hugegraph-pd/pd_data \ + --name hugegraph-pd \ + hugegraph-pd:latest +``` + +For Docker Compose examples with HugeGraph Store and Server, see: +``` +hugegraph-server/hugegraph-dist/docker/example/ +``` + +## Documentation + +- [Architecture Documentation](docs/architecture.md) - System design, module details, and interaction flows +- [API Reference](docs/api-reference.md) - gRPC API definitions and usage examples +- [Configuration Guide](docs/configuration.md) - Configuration options and production tuning +- [Development Guide](docs/development.md) - Build, test, and contribution workflows + +## Production Deployment Notes + +### Cluster Size + +- **Minimum**: 3 nodes (Raft quorum requirement) +- **Recommended**: 3 or 5 nodes for production (odd numbers for Raft election) + +### High Availability + +- PD uses Raft consensus for leader election and data replication +- Cluster can tolerate up to `(N-1)/2` node failures (e.g., 1 failure in 3-node cluster) +- Leader handles all write operations; followers handle read operations + +### Network Requirements + +- Ensure low latency (<5ms) between PD nodes for Raft consensus +- Open required ports: `8620` (REST), `8686` (gRPC), `8610` (Raft) + +### Monitoring + +PD exposes metrics via REST API at: +- Health check: `http://:8620/actuator/health` +- Metrics: `http://:8620/actuator/metrics` + +## Community + +- **Website**: https://hugegraph.apache.org +- **Documentation**: https://hugegraph.apache.org/docs/ +- **GitHub**: https://github.com/apache/hugegraph +- **Mailing List**: dev@hugegraph.apache.org + +## Contributing + +Contributions are welcome! Please read our [Development Guide](docs/development.md) and follow the Apache HugeGraph contribution guidelines. + +## License + +HugeGraph PD is licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). + +--- + +**Status**: BETA (from v1.5.0+) + +For questions or issues, please contact the HugeGraph community via GitHub issues or mailing list. diff --git a/hugegraph-pd/docs/api-reference.md b/hugegraph-pd/docs/api-reference.md new file mode 100644 index 0000000000..aa8cce8473 --- /dev/null +++ b/hugegraph-pd/docs/api-reference.md @@ -0,0 +1,1006 @@ +# HugeGraph PD API Reference + +This document provides comprehensive API reference for HugeGraph PD, including gRPC services, Protocol Buffers definitions, and usage examples. + +## Table of Contents + +- [gRPC Services Overview](#grpc-services-overview) +- [Protocol Buffers Definitions](#protocol-buffers-definitions) +- [Core gRPC APIs](#core-grpc-apis) +- [Java Client Library](#java-client-library) +- [REST API](#rest-api) + +## gRPC Services Overview + +HugeGraph PD exposes multiple gRPC services for cluster management and coordination: + +| Service | Proto File | Description | +|---------|------------|-------------| +| **PDGrpc** | `pdpb.proto` | Main PD service: store registration, partition queries, member management | +| **KvServiceGrpc** | `kv.proto` | Distributed key-value operations for metadata storage | +| **HgPdPulseGrpc** | `pd_pulse.proto` | Heartbeat and health monitoring for Store nodes | +| **HgPdWatchGrpc** | `pd_watch.proto` | Watch for partition and store change notifications | +| **DiscoveryServiceGrpc** | `discovery.proto` | Service discovery and registration | + +**Proto Location**: `hugegraph-pd/hg-pd-grpc/src/main/proto/` + +**Generated Stubs**: `hugegraph-pd/hg-pd-grpc/src/main/java/org/apache/hugegraph/pd/grpc/` + +## Protocol Buffers Definitions + +### Proto Files Structure + +``` +hg-pd-grpc/src/main/proto/ +├── pdpb.proto # Main PD service RPCs +├── metapb.proto # Core metadata objects (Partition, Shard, Store) +├── meta.proto # Extended metadata definitions +├── pd_common.proto # Common types and enums +├── kv.proto # Key-value service +├── pd_pulse.proto # Heartbeat protocol +├── pd_watch.proto # Watch notification protocol +├── discovery.proto # Service discovery +└── metaTask.proto # Task coordination +``` + +### Key Message Types + +#### Partition + +Represents a data partition in the cluster. + +```protobuf +message Partition { + uint64 id = 1; + string graph_name = 2; + uint64 start_key = 3; + uint64 end_key = 4; + repeated Shard shards = 5; + PartitionState state = 6; + uint64 version = 7; +} + +enum PartitionState { + PState_None = 0; + PState_Normal = 1; + PState_Splitting = 2; + PState_Offline = 3; +} +``` + +#### Shard + +Represents a replica of a partition. + +```protobuf +message Shard { + uint64 store_id = 1; + ShardRole role = 2; +} + +enum ShardRole { + None = 0; + Leader = 1; + Follower = 2; + Learner = 3; +} +``` + +#### Store + +Represents a Store node in the cluster. + +```protobuf +message Store { + uint64 id = 1; + string address = 2; // gRPC address (host:port) + string raft_address = 3; // Raft address for data replication + StoreState state = 4; + map labels = 5; // Topology labels (rack, zone, region) + StoreStats stats = 6; + int64 last_heartbeat = 7; // Unix timestamp + uint64 version = 8; +} + +enum StoreState { + Unknown = 0; + Up = 1; // Store is online and healthy + Offline = 2; // Store is temporarily offline + Tombstone = 3; // Store is permanently removed + Exiting = 4; // Store is in the process of shutting down +} + +message StoreStats { + uint64 capacity = 1; // Total disk capacity (bytes) + uint64 available = 2; // Available disk space (bytes) + uint32 partition_count = 3; // Number of partitions on this store + uint32 leader_count = 4; // Number of partitions where this store is leader +} +``` + +#### Graph + +Represents a graph in the cluster. + +```protobuf +message Graph { + string graph_name = 1; + uint32 partition_count = 2; + GraphState state = 3; +} + +enum GraphState { + Graph_Normal = 0; + Graph_Deleting = 1; +} +``` + +## Core gRPC APIs + +### 1. PD Service (PDGrpc) + +Main service for cluster management. + +#### GetMembers + +Retrieve all PD members in the cluster. + +**Request**: +```protobuf +message GetMembersRequest {} +``` + +**Response**: +```protobuf +message GetMembersResponse { + ResponseHeader header = 1; + repeated Member members = 2; + Member leader = 3; +} + +message Member { + string cluster_id = 1; + string member_id = 2; + string grpc_url = 3; // gRPC endpoint + string raft_url = 4; // Raft endpoint + MemberState state = 5; +} +``` + +**Java Example**: +```java +import org.apache.hugegraph.pd.grpc.PDGrpc; +import org.apache.hugegraph.pd.grpc.Pdpb; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; + +// Create gRPC channel +ManagedChannel channel = ManagedChannelBuilder + .forAddress("localhost", 8686) + .usePlaintext() + .build(); + +// Create blocking stub +PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(channel); + +// Get PD members +Pdpb.GetMembersRequest request = Pdpb.GetMembersRequest.newBuilder().build(); +Pdpb.GetMembersResponse response = stub.getMembers(request); + +System.out.println("Leader: " + response.getLeader().getGrpcUrl()); +for (Pdpb.Member member : response.getMembersList()) { + System.out.println("Member: " + member.getGrpcUrl() + " State: " + member.getState()); +} + +// Clean up +channel.shutdown(); +``` + +#### RegisterStore + +Register a new Store node with PD. + +**Request**: +```protobuf +message RegisterStoreRequest { + RequestHeader header = 1; + Store store = 2; +} +``` + +**Response**: +```protobuf +message RegisterStoreResponse { + ResponseHeader header = 1; + uint64 store_id = 2; // Assigned store ID +} +``` + +**Java Example**: +```java +import org.apache.hugegraph.pd.grpc.Metapb; + +// Build store information +Metapb.Store store = Metapb.Store.newBuilder() + .setAddress("192.168.1.100:8500") + .setRaftAddress("192.168.1.100:8501") + .setState(Metapb.StoreState.Up) + .putLabels("zone", "zone-1") + .putLabels("rack", "rack-a") + .build(); + +// Register store +Pdpb.RegisterStoreRequest request = Pdpb.RegisterStoreRequest.newBuilder() + .setStore(store) + .build(); + +Pdpb.RegisterStoreResponse response = stub.registerStore(request); +long storeId = response.getStoreId(); +System.out.println("Registered store with ID: " + storeId); +``` + +#### GetStoreInfo + +Retrieve Store node information. + +**Request**: +```protobuf +message GetStoreInfoRequest { + RequestHeader header = 1; + uint64 store_id = 2; +} +``` + +**Response**: +```protobuf +message GetStoreInfoResponse { + ResponseHeader header = 1; + Store store = 2; +} +``` + +**Java Example**: +```java +Pdpb.GetStoreInfoRequest request = Pdpb.GetStoreInfoRequest.newBuilder() + .setStoreId(storeId) + .build(); + +Pdpb.GetStoreInfoResponse response = stub.getStoreInfo(request); +Metapb.Store store = response.getStore(); + +System.out.println("Store " + store.getId() + " at " + store.getAddress()); +System.out.println("State: " + store.getState()); +System.out.println("Partitions: " + store.getStats().getPartitionCount()); +System.out.println("Capacity: " + store.getStats().getCapacity() / (1024*1024*1024) + " GB"); +``` + +#### GetPartition + +Retrieve partition information by partition code. + +**Request**: +```protobuf +message GetPartitionRequest { + RequestHeader header = 1; + string graph_name = 2; + uint64 partition_key = 3; // Hash code of the data key +} +``` + +**Response**: +```protobuf +message GetPartitionResponse { + ResponseHeader header = 1; + Partition partition = 2; + Shard leader = 3; // Current leader shard +} +``` + +**Java Example**: +```java +String graphName = "social_network"; +long partitionKey = 12345L; // Hash of vertex/edge key + +Pdpb.GetPartitionRequest request = Pdpb.GetPartitionRequest.newBuilder() + .setGraphName(graphName) + .setPartitionKey(partitionKey) + .build(); + +Pdpb.GetPartitionResponse response = stub.getPartition(request); +Metapb.Partition partition = response.getPartition(); +Metapb.Shard leader = response.getLeader(); + +System.out.println("Partition " + partition.getId() + " range: [" + + partition.getStartKey() + ", " + partition.getEndKey() + ")"); +System.out.println("Leader store: " + leader.getStoreId()); +System.out.println("Replicas: " + partition.getShardsCount()); +``` + +#### GetPartitionByCode + +Retrieve partition by exact partition code (optimized for routing). + +**Request**: +```protobuf +message GetPartitionByCodeRequest { + RequestHeader header = 1; + string graph_name = 2; + uint64 partition_id = 3; +} +``` + +**Response**: +```protobuf +message GetPartitionByCodeResponse { + ResponseHeader header = 1; + Partition partition = 2; +} +``` + +**Java Example**: +```java +Pdpb.GetPartitionByCodeRequest request = Pdpb.GetPartitionByCodeRequest.newBuilder() + .setGraphName("social_network") + .setPartitionId(5) + .build(); + +Pdpb.GetPartitionByCodeResponse response = stub.getPartitionByCode(request); +Metapb.Partition partition = response.getPartition(); + +// Find leader shard +Metapb.Shard leader = partition.getShardsList().stream() + .filter(s -> s.getRole() == Metapb.ShardRole.Leader) + .findFirst() + .orElse(null); + +if (leader != null) { + System.out.println("Route query to store: " + leader.getStoreId()); +} +``` + +### 2. KV Service (KvServiceGrpc) + +Distributed key-value operations for metadata storage. + +#### Put + +Store a key-value pair. + +**Request**: +```protobuf +message PutRequest { + string key = 1; + bytes value = 2; + int64 ttl = 3; // Time-to-live in seconds (0 = no expiration) +} +``` + +**Response**: +```protobuf +message PutResponse { + ResponseHeader header = 1; +} +``` + +**Java Example**: +```java +import org.apache.hugegraph.pd.grpc.kv.KvServiceGrpc; +import org.apache.hugegraph.pd.grpc.kv.Kv; + +KvServiceGrpc.KvServiceBlockingStub kvStub = KvServiceGrpc.newBlockingStub(channel); + +// Store configuration +String key = "config/max_retry_count"; +String value = "5"; + +Kv.PutRequest request = Kv.PutRequest.newBuilder() + .setKey(key) + .setValue(com.google.protobuf.ByteString.copyFromUtf8(value)) + .setTtl(0) // No expiration + .build(); + +Kv.PutResponse response = kvStub.put(request); +System.out.println("Stored: " + key); +``` + +#### Get + +Retrieve a value by key. + +**Request**: +```protobuf +message GetRequest { + string key = 1; +} +``` + +**Response**: +```protobuf +message GetResponse { + ResponseHeader header = 1; + bytes value = 2; +} +``` + +**Java Example**: +```java +Kv.GetRequest request = Kv.GetRequest.newBuilder() + .setKey("config/max_retry_count") + .build(); + +Kv.GetResponse response = kvStub.get(request); +String value = response.getValue().toStringUtf8(); +System.out.println("Retrieved value: " + value); +``` + +#### Scan + +Range scan for keys matching a prefix. + +**Request**: +```protobuf +message ScanRequest { + string start_key = 1; + string end_key = 2; + int32 limit = 3; // Max number of results +} +``` + +**Response**: +```protobuf +message ScanResponse { + ResponseHeader header = 1; + repeated KvPair kvs = 2; +} + +message KvPair { + string key = 1; + bytes value = 2; +} +``` + +**Java Example**: +```java +// Scan all configuration keys +Kv.ScanRequest request = Kv.ScanRequest.newBuilder() + .setStartKey("config/") + .setEndKey("config/\uffff") // End of prefix range + .setLimit(100) + .build(); + +Kv.ScanResponse response = kvStub.scan(request); +for (Kv.KvPair kv : response.getKvsList()) { + System.out.println(kv.getKey() + " = " + kv.getValue().toStringUtf8()); +} +``` + +### 3. Pulse Service (HgPdPulseGrpc) + +Heartbeat and health monitoring for Store nodes. + +#### Pulse (Streaming) + +Bidirectional streaming for continuous heartbeat. + +**Request Stream**: +```protobuf +message PulseRequest { + PulseType pulse_type = 1; + oneof notice { + PulseCreatePartition create_partition = 2; + PulseTransferLeader transfer_leader = 3; + PulseMovePartition move_partition = 4; + PulseDeletePartition delete_partition = 5; + } +} + +enum PulseType { + PULSE_TYPE_UNKNOWN = 0; + PULSE_TYPE_STORE_HEARTBEAT = 1; + PULSE_TYPE_PARTITION_HEARTBEAT = 2; +} +``` + +**Response Stream**: +```protobuf +message PulseResponse { + PulseType pulse_type = 1; + oneof notice { + PulseCreatePartition create_partition = 2; + PulseTransferLeader transfer_leader = 3; + PulseMovePartition move_partition = 4; + PulseDeletePartition delete_partition = 5; + } +} +``` + +**Java Example**: +```java +import org.apache.hugegraph.pd.grpc.pulse.HgPdPulseGrpc; +import org.apache.hugegraph.pd.grpc.pulse.HgPdPulse; +import io.grpc.stub.StreamObserver; + +HgPdPulseGrpc.HgPdPulseStub asyncStub = HgPdPulseGrpc.newStub(channel); + +// Response handler +StreamObserver responseObserver = new StreamObserver<>() { + @Override + public void onNext(HgPdPulse.PulseResponse response) { + System.out.println("Received instruction: " + response.getPulseType()); + // Handle instructions from PD (partition transfer, split, etc.) + } + + @Override + public void onError(Throwable t) { + System.err.println("Pulse stream error: " + t.getMessage()); + } + + @Override + public void onCompleted() { + System.out.println("Pulse stream completed"); + } +}; + +// Create bidirectional stream +StreamObserver requestObserver = asyncStub.pulse(responseObserver); + +// Send periodic heartbeat +ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); +scheduler.scheduleAtFixedRate(() -> { + HgPdPulse.PulseRequest heartbeat = HgPdPulse.PulseRequest.newBuilder() + .setPulseType(HgPdPulse.PulseType.PULSE_TYPE_STORE_HEARTBEAT) + .build(); + + requestObserver.onNext(heartbeat); +}, 0, 10, TimeUnit.SECONDS); +``` + +### 4. Watch Service (HgPdWatchGrpc) + +Watch for partition and store change notifications. + +#### WatchPartition + +Watch for partition changes in a graph. + +**Request**: +```protobuf +message WatchPartitionRequest { + RequestHeader header = 1; + string graph_name = 2; + WatchType watch_type = 3; +} + +enum WatchType { + WATCH_TYPE_PARTITION_CHANGE = 0; + WATCH_TYPE_STORE_CHANGE = 1; +} +``` + +**Response Stream**: +```protobuf +message WatchPartitionResponse { + ResponseHeader header = 1; + WatchChangeType change_type = 2; + Partition partition = 3; +} + +enum WatchChangeType { + WATCH_CHANGE_TYPE_ADD = 0; + WATCH_CHANGE_TYPE_DEL = 1; + WATCH_CHANGE_TYPE_ALTER = 2; +} +``` + +**Java Example**: +```java +import org.apache.hugegraph.pd.grpc.watch.HgPdWatchGrpc; +import org.apache.hugegraph.pd.grpc.watch.HgPdWatch; + +HgPdWatchGrpc.HgPdWatchStub watchStub = HgPdWatchGrpc.newStub(channel); + +// Watch partition changes +HgPdWatch.WatchPartitionRequest request = HgPdWatch.WatchPartitionRequest.newBuilder() + .setGraphName("social_network") + .setWatchType(HgPdWatch.WatchType.WATCH_TYPE_PARTITION_CHANGE) + .build(); + +StreamObserver responseObserver = new StreamObserver<>() { + @Override + public void onNext(HgPdWatch.WatchPartitionResponse response) { + WatchChangeType changeType = response.getChangeType(); + Metapb.Partition partition = response.getPartition(); + + switch (changeType) { + case WATCH_CHANGE_TYPE_ADD: + System.out.println("Partition added: " + partition.getId()); + break; + case WATCH_CHANGE_TYPE_DEL: + System.out.println("Partition deleted: " + partition.getId()); + break; + case WATCH_CHANGE_TYPE_ALTER: + System.out.println("Partition changed: " + partition.getId()); + // Refresh local cache + break; + } + } + + @Override + public void onError(Throwable t) { + System.err.println("Watch error: " + t.getMessage()); + } + + @Override + public void onCompleted() { + System.out.println("Watch completed"); + } +}; + +watchStub.watchPartition(request, responseObserver); +``` + +## Java Client Library + +HugeGraph PD provides a high-level Java client library (`hg-pd-client`) that simplifies interaction with PD. + +### PDClient + +Main client class for PD operations. + +**Initialization**: +```java +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.client.PDConfig; + +// Configure PD client +PDConfig config = PDConfig.builder() + .pdServers("192.168.1.10:8686,192.168.1.11:8686,192.168.1.12:8686") + .connectTimeout(5000) // 5 seconds + .requestTimeout(10000) // 10 seconds + .enableCache(true) // Enable partition cache + .build(); + +// Create client +PDClient client = new PDClient(config); + +// Use client... + +// Clean up +client.close(); +``` + +### Partition Operations + +```java +import org.apache.hugegraph.pd.common.PartitionEngine; + +// Get partition by key +String graphName = "social_network"; +long vertexId = 12345L; +long partitionKey = PartitionEngine.calcHashcode(vertexId); + +Metapb.Partition partition = client.getPartitionByKey(graphName, partitionKey); +System.out.println("Partition ID: " + partition.getId()); + +// Get all partitions for a graph +List partitions = client.getPartitionsByGraphName(graphName); +System.out.println("Total partitions: " + partitions.size()); + +// Get partition leader +Metapb.Shard leader = client.getPartitionLeader(graphName, partition.getId()); +Metapb.Store leaderStore = client.getStore(leader.getStoreId()); +System.out.println("Leader at: " + leaderStore.getAddress()); +``` + +### Store Operations + +```java +// Get all stores +List stores = client.getStores(); +for (Metapb.Store store : stores) { + System.out.println("Store " + store.getId() + ": " + store.getAddress() + + " (" + store.getState() + ")"); +} + +// Get active stores +List activeStores = client.getActiveStores(); +System.out.println("Active stores: " + activeStores.size()); + +// Get stores by graph +List graphStores = client.getStoresByGraphName(graphName); +``` + +### Watch Operations + +```java +import org.apache.hugegraph.pd.client.PDWatch; + +// Create watch listener +PDWatch.Listener listener = new PDWatch.Listener<>() { + @Override + public void onNext(PDWatch.WatchEvent event) { + System.out.println("Partition " + event.getTarget().getId() + + " " + event.getType()); + } + + @Override + public void onError(Throwable error) { + System.err.println("Watch error: " + error.getMessage()); + } +}; + +// Watch partition changes +PDWatch watch = client.watchPartition(graphName, listener); + +// Stop watching +watch.close(); +``` + +### KV Operations + +```java +// Put key-value +client.put("config/setting1", "value1".getBytes()); + +// Get value +byte[] value = client.get("config/setting1"); +System.out.println("Value: " + new String(value)); + +// Delete key +client.delete("config/setting1"); + +// Scan with prefix +Map results = client.scan("config/", "config/\uffff", 100); +for (Map.Entry entry : results.entrySet()) { + System.out.println(entry.getKey() + " = " + new String(entry.getValue())); +} +``` + +## REST API + +PD exposes a REST API for management and monitoring (default port: 8620). + +### Health Check + +```bash +curl http://localhost:8620/actuator/health +``` + +**Response**: +```json +{ + "status": "UP", + "groups": ["liveness", "readiness"] +} +``` + +### Metrics + +```bash +curl http://localhost:8620/actuator/metrics +``` + +**Response** (Prometheus format): +``` +# HELP pd_raft_state Raft state (0=Follower, 1=Candidate, 2=Leader) +# TYPE pd_raft_state gauge +pd_raft_state 2.0 + +# HELP pd_store_count Number of stores +# TYPE pd_store_count gauge +pd_store_count{state="Up"} 3.0 +pd_store_count{state="Offline"} 0.0 + +# HELP pd_partition_count Number of partitions +# TYPE pd_partition_count gauge +pd_partition_count 36.0 +``` + +### Partition API + +#### List Partitions + +```bash +curl http://localhost:8620/v1/partitions?graph_name=social_network +``` + +**Response**: +```json +{ + "partitions": [ + { + "id": 1, + "graph_name": "social_network", + "start_key": 0, + "end_key": 1000, + "shards": [ + {"store_id": 1, "role": "Leader"}, + {"store_id": 2, "role": "Follower"}, + {"store_id": 3, "role": "Follower"} + ], + "state": "PState_Normal" + } + ] +} +``` + +### Store API + +#### List Stores + +```bash +curl http://localhost:8620/v1/stores +``` + +**Response**: +```json +{ + "stores": [ + { + "id": 1, + "address": "192.168.1.100:8500", + "raft_address": "192.168.1.100:8501", + "state": "Up", + "labels": { + "zone": "zone-1", + "rack": "rack-a" + }, + "stats": { + "capacity": 107374182400, + "available": 53687091200, + "partition_count": 12, + "leader_count": 8 + }, + "last_heartbeat": 1620000000 + } + ] +} +``` + +## Error Handling + +### gRPC Status Codes + +PD uses standard gRPC status codes: + +| Code | Name | Description | +|------|------|-------------| +| 0 | OK | Success | +| 1 | CANCELLED | Operation cancelled | +| 2 | UNKNOWN | Unknown error | +| 3 | INVALID_ARGUMENT | Invalid request parameters | +| 4 | DEADLINE_EXCEEDED | Timeout | +| 5 | NOT_FOUND | Resource not found (store, partition, etc.) | +| 6 | ALREADY_EXISTS | Resource already exists | +| 7 | PERMISSION_DENIED | Insufficient permissions | +| 8 | RESOURCE_EXHAUSTED | Quota exceeded | +| 14 | UNAVAILABLE | Service unavailable (not leader, Raft not ready) | + +### Response Header + +All responses include a `ResponseHeader` with error information: + +```protobuf +message ResponseHeader { + uint64 cluster_id = 1; + Error error = 2; +} + +message Error { + ErrorType type = 1; + string message = 2; +} + +enum ErrorType { + OK = 0; + NOT_LEADER = 1; // Current node is not Raft leader + STORE_NOT_FOUND = 2; + PARTITION_NOT_FOUND = 3; + STORE_TOMBSTONE = 4; // Store is permanently removed + RAFT_ERROR = 5; +} +``` + +**Error Handling Example**: +```java +Pdpb.GetStoreInfoResponse response = stub.getStoreInfo(request); + +if (response.getHeader().hasError()) { + Error error = response.getHeader().getError(); + + if (error.getType() == ErrorType.NOT_LEADER) { + // Retry with leader node + String leaderUrl = getLeaderFromMembers(); + // Reconnect and retry... + } else { + System.err.println("Error: " + error.getMessage()); + } +} else { + Metapb.Store store = response.getStore(); + // Process store... +} +``` + +## Best Practices + +### 1. Connection Management + +- **Reuse gRPC channels**: Creating channels is expensive +- **Connection pooling**: Use multiple channels for high throughput +- **Automatic reconnection**: Handle disconnections gracefully + +```java +// Good: Reuse channel +ManagedChannel channel = ManagedChannelBuilder + .forAddress("pd-host", 8686) + .usePlaintext() + .keepAliveTime(30, TimeUnit.SECONDS) + .idleTimeout(60, TimeUnit.SECONDS) + .build(); + +// Bad: Create new channel per request +// ManagedChannel channel = ... +// channel.shutdown() // Don't do this after every request +``` + +### 2. Leader Detection + +- Always check `ResponseHeader.error.type` for `NOT_LEADER` +- Use `GetMembers()` to find current leader +- Cache leader information but refresh on errors + +### 3. Partition Caching + +- Cache partition routing information locally +- Use `WatchPartition` to invalidate cache on changes +- Set reasonable cache TTL (e.g., 5 minutes) + +### 4. Retry Strategy + +- Implement exponential backoff for retries +- Retry on transient errors (UNAVAILABLE, DEADLINE_EXCEEDED) +- Don't retry on permanent errors (NOT_FOUND, INVALID_ARGUMENT) + +```java +int maxRetries = 3; +int retryDelay = 1000; // milliseconds + +for (int i = 0; i < maxRetries; i++) { + try { + response = stub.getPartition(request); + break; // Success + } catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.Code.UNAVAILABLE && i < maxRetries - 1) { + Thread.sleep(retryDelay * (1 << i)); // Exponential backoff + } else { + throw e; + } + } +} +``` + +### 5. Timeout Configuration + +- Set appropriate timeouts for all RPCs +- Use shorter timeouts for read operations +- Use longer timeouts for write operations + +```java +PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(channel) + .withDeadlineAfter(5, TimeUnit.SECONDS); +``` + +## Summary + +HugeGraph PD provides comprehensive gRPC APIs for: +- Cluster membership and leadership management +- Store registration and monitoring +- Partition routing and querying +- Distributed key-value operations +- Real-time change notifications + +Use the high-level `PDClient` library for simplified integration, or use raw gRPC stubs for fine-grained control. + +For architecture details, see [Architecture Documentation](architecture.md). + +For configuration, see [Configuration Guide](configuration.md). diff --git a/hugegraph-pd/docs/architecture.md b/hugegraph-pd/docs/architecture.md new file mode 100644 index 0000000000..080189be95 --- /dev/null +++ b/hugegraph-pd/docs/architecture.md @@ -0,0 +1,713 @@ +# HugeGraph PD Architecture + +This document provides a comprehensive overview of HugeGraph PD's architecture, design principles, and internal components. + +## Table of Contents + +- [System Overview](#system-overview) +- [Module Architecture](#module-architecture) +- [Core Components](#core-components) +- [Raft Consensus Layer](#raft-consensus-layer) +- [Data Flow](#data-flow) +- [Interaction with Store and Server](#interaction-with-store-and-server) + +## System Overview + +### What is HugeGraph PD? + +HugeGraph PD (Placement Driver) is the control plane for HugeGraph distributed deployments. It acts as a centralized coordinator that manages cluster topology, partition allocation, and node scheduling while maintaining strong consistency through Raft consensus. + +### Key Responsibilities + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ HugeGraph PD Cluster │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Service Discovery & Registration │ │ +│ │ - Store node registration and health monitoring │ │ +│ │ - Server node discovery and load balancing │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Partition Management │ │ +│ │ - Partition allocation across stores │ │ +│ │ - Dynamic rebalancing and splitting │ │ +│ │ - Leader election coordination │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Metadata Storage │ │ +│ │ - Cluster configuration and state │ │ +│ │ - Graph metadata and schemas │ │ +│ │ - Distributed KV operations │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Task Scheduling │ │ +│ │ - Partition patrol and health checks │ │ +│ │ - Automated rebalancing triggers │ │ +│ │ - Metrics collection coordination │ │ +│ └──────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Technology Stack + +- **Consensus**: Apache JRaft (Raft implementation from Ant Design) +- **Storage**: RocksDB for persistent metadata +- **Communication**: gRPC with Protocol Buffers +- **Framework**: Spring Boot for REST APIs and dependency injection +- **Language**: Java 11+ + +## Module Architecture + +HugeGraph PD consists of 8 Maven modules organized in a layered architecture: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Client Layer │ +├─────────────────────────────────────────────────────────────┤ +│ hg-pd-client │ Java client library for PD access │ +│ hg-pd-cli │ Command-line tools for administration │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Service Layer │ +├─────────────────────────────────────────────────────────────┤ +│ hg-pd-service │ gRPC service implementations │ +│ │ REST API endpoints (Spring Boot) │ +│ │ Service discovery and pulse monitoring │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Core Layer │ +├─────────────────────────────────────────────────────────────┤ +│ hg-pd-core │ Raft consensus integration (JRaft) │ +│ │ Metadata stores (RocksDB-backed) │ +│ │ Partition allocation and balancing │ +│ │ Store node monitoring and scheduling │ +│ │ Task coordination and execution │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Foundation Layer │ +├─────────────────────────────────────────────────────────────┤ +│ hg-pd-grpc │ Protocol Buffers definitions │ +│ │ Generated gRPC stubs │ +│ hg-pd-common │ Shared utilities and interfaces │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Distribution Layer │ +├─────────────────────────────────────────────────────────────┤ +│ hg-pd-dist │ Assembly configuration │ +│ │ Startup/shutdown scripts │ +│ │ Configuration templates │ +│ hg-pd-test │ Integration and unit tests │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Module Dependencies + +``` +hg-pd-grpc (proto definitions) + ↓ +hg-pd-common (utilities) + ↓ +hg-pd-core (business logic) + ↓ +hg-pd-service (API layer) + ↓ +hg-pd-dist (packaging) + +Branch: +hg-pd-client ← hg-pd-grpc + hg-pd-common +hg-pd-cli ← hg-pd-client +hg-pd-test ← hg-pd-core + hg-pd-service +``` + +### Module Details + +#### hg-pd-grpc + +Protocol Buffers definitions and generated gRPC code. + +**Key Proto Files**: +- `pdpb.proto`: Main PD service RPCs (GetMembers, RegisterStore, GetPartition) +- `metapb.proto`: Core metadata objects (Partition, Shard, Store, Graph) +- `discovery.proto`: Service discovery protocol +- `kv.proto`: Distributed key-value operations +- `pd_pulse.proto`: Heartbeat and monitoring protocol +- `pd_watch.proto`: Change notification watchers +- `metaTask.proto`: Distributed task coordination + +**Location**: `hg-pd-grpc/src/main/proto/` + +**Generated Code**: Excluded from source control; regenerated via `mvn compile` + +#### hg-pd-common + +Shared utilities and common interfaces used across modules. + +**Key Components**: +- Configuration POJOs +- Common exceptions and error codes +- Utility classes for validation and conversion + +#### hg-pd-core + +Core business logic and metadata management. This is the heart of PD. + +**Package Structure**: +``` +org.apache.hugegraph.pd/ +├── meta/ # Metadata stores (RocksDB-backed) +│ ├── MetadataRocksDBStore # Base persistence layer +│ ├── PartitionMeta # Partition and shard group management +│ ├── StoreInfoMeta # Store node information +│ ├── TaskInfoMeta # Distributed task coordination +│ ├── IdMetaStore # Auto-increment ID generation +│ ├── ConfigMetaStore # Configuration management +│ └── DiscoveryMetaStore # Service discovery metadata +├── raft/ # Raft integration layer +│ ├── RaftEngine # Raft group lifecycle management +│ ├── RaftStateMachine # State machine for metadata operations +│ ├── RaftTaskHandler # Async task execution via Raft +│ ├── KVOperation # Raft operation abstraction +│ └── KVStoreClosure # Raft callback handling +├── PartitionService # Partition allocation and balancing +├── StoreNodeService # Store registration and monitoring +├── StoreMonitorDataService # Metrics collection and time-series +├── TaskScheduleService # Automated partition patrol +├── KvService # Distributed KV operations +├── IdService # ID generation service +├── ConfigService # Configuration management +└── LogService # Operational logging +``` + +#### hg-pd-service + +gRPC service implementations and REST API. + +**Key Classes**: +- `ServiceGrpc`: Main gRPC service endpoint +- `PDPulseService`: Heartbeat processing +- `DiscoveryService`: Service discovery +- REST APIs: `PartitionAPI`, `StoreAPI` (Spring Boot controllers) + +**REST Endpoints** (port 8620 by default): +- `/actuator/health`: Health check +- `/actuator/metrics`: Prometheus-compatible metrics +- `/v1/partitions`: Partition management API +- `/v1/stores`: Store management API + +#### hg-pd-client + +Java client library for applications to interact with PD. + +**Features**: +- gRPC connection pooling +- Automatic leader detection and failover +- Partition routing and caching +- Store discovery and health awareness + +**Typical Usage**: +```java +PDConfig config = PDConfig.builder() + .pdServers("192.168.1.10:8686,192.168.1.11:8686,192.168.1.12:8686") + .build(); + +PDClient client = new PDClient(config); + +// Register a store +client.registerStore(storeId, storeAddress); + +// Get partition information +Partition partition = client.getPartitionByCode(graphName, partitionCode); + +// Watch for partition changes +client.watchPartitions(graphName, listener); +``` + +#### hg-pd-cli + +Command-line tools for PD administration. + +**Common Operations**: +- Store management (list, offline, online) +- Partition inspection and balancing +- Raft cluster status +- Metadata backup and restore + +#### hg-pd-test + +Integration and unit tests. + +**Test Categories**: +- Core service tests: `PartitionServiceTest`, `StoreNodeServiceTest` +- Raft integration tests: Leader election, snapshot, log replication +- gRPC API tests: Service registration, partition queries +- Metadata persistence tests: RocksDB operations, recovery + +**Location**: `hg-pd-test/src/main/java/` (non-standard location) + +#### hg-pd-dist + +Distribution packaging and deployment artifacts. + +**Structure**: +``` +src/assembly/ +├── descriptor/ +│ └── server-assembly.xml # Maven assembly configuration +└── static/ + ├── bin/ + │ ├── start-hugegraph-pd.sh + │ ├── stop-hugegraph-pd.sh + │ └── util.sh + └── conf/ + ├── application.yml.template + └── log4j2.xml +``` + +## Core Components + +### Metadata Stores + +All metadata is persisted in RocksDB via the `MetadataRocksDBStore` base class, ensuring durability and fast access. + +#### PartitionMeta + +Manages partition allocation and shard group information. + +**Key Responsibilities**: +- Partition-to-store mapping +- Shard group (replica set) management +- Partition leader tracking +- Partition splitting metadata + +**Data Structure**: +``` +Partition { + graphName: String + partitionId: Int + startKey: Long + endKey: Long + shards: List + workState: PartitionState (NORMAL, SPLITTING, OFFLINE) +} + +Shard { + storeId: Long + role: ShardRole (LEADER, FOLLOWER, LEARNER) +} +``` + +**Related Service**: `PartitionService` (hg-pd-core:712) + +#### StoreInfoMeta + +Stores information about Store nodes in the cluster. + +**Key Responsibilities**: +- Store registration and activation +- Store state management (ONLINE, OFFLINE, TOMBSTONE) +- Store labels and deployment topology +- Store capacity and load tracking + +**Data Structure**: +``` +Store { + storeId: Long + address: String (gRPC endpoint) + raftAddress: String + state: StoreState + labels: Map # rack, zone, region + stats: StoreStats (capacity, available, partitionCount) + lastHeartbeat: Timestamp +} +``` + +**Related Service**: `StoreNodeService` (hg-pd-core:589) + +#### TaskInfoMeta + +Coordinates distributed tasks across the PD cluster. + +**Task Types**: +- Partition balancing +- Partition splitting +- Store decommissioning +- Data migration + +**Related Service**: `TaskScheduleService` + +#### IdMetaStore + +Provides auto-increment ID generation for: +- Store IDs +- Partition IDs +- Task IDs +- Custom business IDs + +**Location**: `hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java:41` + +**Implementation**: Cluster ID-based ID allocation with local batching for performance. + +### Service Layer + +#### PartitionService + +The most complex service, responsible for all partition management. + +**Key Methods**: +- `getPartitionByCode(graphName, code)`: Route queries to correct partition +- `splitPartition(partitionId)`: Split partition when size exceeds threshold +- `balancePartitions()`: Rebalance partitions across stores +- `updatePartitionLeader(partitionId, shardId)`: Handle leader changes +- `transferLeader(partitionId, targetStoreId)`: Manual leader transfer + +**Balancing Algorithm**: +1. Calculate partition distribution across stores +2. Identify overloaded stores (above threshold) +3. Identify underloaded stores (below threshold) +4. Generate transfer plans (partition → target store) +5. Execute transfers sequentially with validation + +**Location**: `hg-pd-core/.../PartitionService.java` (2000+ lines) + +#### StoreNodeService + +Manages Store node lifecycle and health monitoring. + +**Key Methods**: +- `registerStore(store)`: Register new store node +- `handleStoreHeartbeat(storeId, stats)`: Process heartbeat and update state +- `setStoreState(storeId, state)`: Change store state (ONLINE/OFFLINE) +- `getStore(storeId)`: Retrieve store information +- `getStoresByGraphName(graphName)`: Get stores for specific graph + +**Heartbeat Processing**: +1. Update store last heartbeat timestamp +2. Update store statistics (disk usage, partition count) +3. Detect store failures (heartbeat timeout) +4. Trigger partition rebalancing if needed + +**Location**: `hg-pd-core/.../StoreNodeService.java` + +#### TaskScheduleService + +Automated background tasks for cluster maintenance. + +**Scheduled Tasks**: +- **Partition Patrol**: Periodically scan all partitions for health issues +- **Balance Check**: Detect imbalanced partition distribution +- **Store Monitor**: Check store health and trigger failover +- **Metrics Collection**: Aggregate cluster metrics + +**Configuration**: +- `pd.patrol-interval`: Patrol interval in seconds (default: 1800) + +#### KvService + +Distributed key-value operations backed by Raft consensus. + +**Operations**: +- `put(key, value)`: Store key-value pair +- `get(key)`: Retrieve value by key +- `delete(key)`: Remove key-value pair +- `scan(startKey, endKey)`: Range scan + +**Use Cases**: +- Configuration storage +- Graph metadata +- Custom application data + +## Raft Consensus Layer + +### Why Raft? + +PD uses Apache JRaft to ensure: +- **Strong Consistency**: All PD nodes see the same metadata +- **High Availability**: Automatic leader election on failures +- **Fault Tolerance**: Cluster survives (N-1)/2 node failures + +### Raft Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ PD Node 1 (Leader) │ +│ ┌──────────────┐ ┌─────────────┐ ┌──────────────────┐ │ +│ │ gRPC Service │→ │ RaftEngine │→ │ RaftStateMachine │ │ +│ └──────────────┘ └─────────────┘ └──────────────────┘ │ +│ ↓ ↓ │ +│ ┌──────────────────────────────┐ │ +│ │ RocksDB (Metadata) │ │ +│ └──────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ (Log Replication) + ┌─────────────────────┴─────────────────────┐ + ↓ ↓ +┌───────────────────┐ ┌───────────────────┐ +│ PD Node 2 │ │ PD Node 3 │ +│ (Follower) │ │ (Follower) │ +│ │ │ │ +│ RaftStateMachine │ │ RaftStateMachine │ +│ ↓ │ │ ↓ │ +│ RocksDB │ │ RocksDB │ +└───────────────────┘ └───────────────────┘ +``` + +### Raft Components + +#### RaftEngine + +Manages Raft group lifecycle. + +**Location**: `hg-pd-core/src/main/java/.../raft/RaftEngine.java` + +**Responsibilities**: +- Initialize Raft group on startup +- Handle leader election +- Manage Raft configuration changes (add/remove nodes) +- Snapshot creation and recovery + +**Key Methods**: +- `init()`: Initialize Raft node +- `isLeader()`: Check if current node is leader +- `submitTask(operation)`: Submit operation to Raft (leader only) + +#### RaftStateMachine + +Applies committed Raft log entries to metadata stores. + +**Location**: `hg-pd-core/src/main/java/.../raft/RaftStateMachine.java` + +**Workflow**: +1. Receive committed log entry from Raft +2. Deserialize operation (PUT, DELETE, etc.) +3. Apply operation to RocksDB +4. Return result to client (if on leader) + +**Snapshot Management**: +- Periodic snapshots to reduce log size +- Snapshots stored in `pd_data/raft/snapshot/` +- Followers recover from snapshots + incremental logs + +#### KVOperation + +Abstraction for Raft operations. + +**Types**: +- `PUT`: Write key-value pair +- `DELETE`: Remove key-value pair +- `BATCH`: Atomic batch operations + +**Serialization**: Hessian2 for compact binary encoding + +### Raft Data Flow + +**Write Operation**: +``` +1. Client → PD Leader gRPC API +2. Leader → RaftEngine.submitTask(PUT operation) +3. RaftEngine → Replicate log to followers +4. Followers → Acknowledge log entry +5. Leader → Commit log entry (quorum reached) +6. RaftStateMachine → Apply to RocksDB +7. Leader → Return success to client +``` + +**Read Operation** (default mode): +``` +1. Client → Any PD node gRPC API +2. PD Node → Read from local RocksDB +3. PD Node → Return result to client +``` + +**Linearizable Read** (optional): +``` +1. Client → PD Leader gRPC API +2. Leader → ReadIndex query to ensure leadership +3. Leader → Wait for commit index â‰Ĩ read index +4. Leader → Read from RocksDB +5. Leader → Return result to client +``` + +## Data Flow + +### Store Registration Flow + +``` +1. Store Node starts up +2. Store → gRPC RegisterStore(storeInfo) → PD Leader +3. PD Leader → Validate store info +4. PD Leader → Raft proposal (PUT store metadata) +5. Raft → Replicate and commit +6. PD Leader → Assign store ID +7. PD Leader → Return store ID to Store +8. Store → Start heartbeat loop +``` + +### Partition Query Flow + +``` +1. Server → gRPC GetPartition(graphName, key) → PD +2. PD → Hash key to partition code +3. PD → Query PartitionMeta (local RocksDB) +4. PD → Return partition info (shards, leader) +5. Server → Cache partition info +6. Server → Route query to Store (partition leader) +``` + +### Heartbeat Flow + +``` +1. Store → gRPC StoreHeartbeat(storeId, stats) → PD Leader (every 10s) +2. PD Leader → Update store last heartbeat +3. PD Leader → Update store statistics +4. PD Leader → Check for partition state changes +5. PD Leader → Return instructions (transfer leader, split partition, etc.) +6. Store → Execute instructions +``` + +### Partition Balancing Flow + +``` +1. TaskScheduleService → Periodic patrol (every 30 min by default) +2. PartitionService → Calculate partition distribution +3. PartitionService → Identify imbalanced stores +4. PartitionService → Generate balance plan +5. PartitionService → Raft proposal (update partition metadata) +6. PD → Send transfer instructions via heartbeat response +7. Store → Execute partition transfers +8. Store → Report completion via heartbeat +``` + +## Interaction with Store and Server + +### Architecture Context + +``` +┌────────────────────────────────────────────────────────────────┐ +│ HugeGraph Cluster │ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ HugeGraph │ │ HugeGraph │ │ +│ │ Server (3x) │ │ Server (3x) │ │ +│ │ - REST API │ │ - REST API │ │ +│ │ - Gremlin │ │ - Cypher │ │ +│ └────────â”Ŧ────────┘ └────────â”Ŧ────────┘ │ +│ │ │ │ +│ └──────────â”Ŧ──────────────┘ │ +│ ↓ (query routing) │ +│ ┌─────────────────────────┐ │ +│ │ HugeGraph PD Cluster │ │ +│ │ (3x or 5x nodes) │ │ +│ │ - Service Discovery │ │ +│ │ - Partition Routing │ │ +│ │ - Metadata Management │ │ +│ └─────────â”Ŧ───────────────┘ │ +│ ↓ (partition assignment) │ +│ ┌──────────────────┴───────────────────────────┐ │ +│ │ │ │ +│ ↓ ↓ ↓ │ +│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ +│ │ HugeGraph │ │ HugeGraph │ │ HugeGraph │ │ +│ │ Store Node 1 │ │ Store Node 2 │ │ Store Node 3 │ │ +│ │ - RocksDB │ │ - RocksDB │ │ - RocksDB │ │ +│ │ - Raft (data) │ │ - Raft (data) │ │ - Raft (data) │ │ +│ └───────────────┘ └───────────────┘ └───────────────┘ │ +└────────────────────────────────────────────────────────────────┘ +``` + +### PD ↔ Store Communication + +**gRPC Services Used**: +- `PDGrpc.registerStore()`: Store registration +- `PDGrpc.getStoreInfo()`: Retrieve store metadata +- `PDGrpc.reportTask()`: Task completion reporting +- `HgPdPulseGrpc.pulse()`: Heartbeat streaming + +**Store → PD** (initiated by Store): +- Store registration on startup +- Periodic heartbeat (every 10 seconds) +- Partition state updates +- Task completion reports + +**PD → Store** (via heartbeat response): +- Partition transfer instructions +- Partition split instructions +- Leadership transfer commands +- Store state changes (ONLINE/OFFLINE) + +### PD ↔ Server Communication + +**gRPC Services Used**: +- `PDGrpc.getPartition()`: Partition routing queries +- `PDGrpc.getPartitionsByGraphName()`: Batch partition queries +- `HgPdWatchGrpc.watch()`: Real-time partition change notifications + +**Server → PD**: +- Partition routing queries (on cache miss) +- Watch partition changes +- Graph metadata queries + +**PD → Server** (via watch stream): +- Partition added/removed events +- Partition leader changes +- Store online/offline events + +### Partition Assignment Example + +Scenario: A new graph "social_network" is created with 12 partitions and 3 stores. + +**Step-by-Step**: +1. Server → `CreateGraph("social_network", partitionCount=12)` → PD +2. PD → Calculate partition distribution: 4 partitions per store +3. PD → Create partition metadata: + ``` + Partition 0: [Shard(store=1, LEADER), Shard(store=2, FOLLOWER), Shard(store=3, FOLLOWER)] + Partition 1: [Shard(store=2, LEADER), Shard(store=3, FOLLOWER), Shard(store=1, FOLLOWER)] + ... + Partition 11: [Shard(store=3, LEADER), Shard(store=1, FOLLOWER), Shard(store=2, FOLLOWER)] + ``` +4. PD → Raft commit partition metadata +5. PD → Send create partition instructions to stores via heartbeat +6. Stores → Create RocksDB instances for assigned partitions +7. Stores → Form Raft groups for each partition +8. Stores → Report partition ready via heartbeat +9. PD → Return success to Server +10. Server → Cache partition routing table + +### Load Balancing Example + +Scenario: Store 3 is overloaded (8 partitions), Store 1 is underloaded (2 partitions). + +**Rebalancing Process**: +1. TaskScheduleService detects imbalance during patrol +2. PartitionService generates plan: Move 3 partitions from Store 3 to Store 1 +3. For each partition to move: + - PD → Raft commit: Add Store 1 as LEARNER to partition + - PD → Instruct Store 3 to add Store 1 replica (via heartbeat) + - Store 3 → Raft add learner and sync data + - Store 1 → Catch up with leader + - PD → Raft commit: Promote Store 1 to FOLLOWER + - PD → Raft commit: Transfer leader to Store 1 + - PD → Raft commit: Remove Store 3 from partition + - Store 3 → Delete partition RocksDB +4. Repeat for remaining partitions +5. Final state: Store 1 (5 partitions), Store 3 (5 partitions) + +## Summary + +HugeGraph PD provides a robust, highly available control plane for distributed HugeGraph deployments through: + +- **Raft Consensus**: Strong consistency and automatic failover +- **Modular Design**: Clean separation of concerns across 8 modules +- **Scalable Metadata**: RocksDB-backed persistence with efficient indexing +- **Intelligent Scheduling**: Automated partition balancing and failure recovery +- **gRPC Communication**: High-performance inter-service communication + +For configuration details, see [Configuration Guide](configuration.md). + +For API usage, see [API Reference](api-reference.md). + +For development workflows, see [Development Guide](development.md). diff --git a/hugegraph-pd/docs/configuration.md b/hugegraph-pd/docs/configuration.md new file mode 100644 index 0000000000..f66ddbd043 --- /dev/null +++ b/hugegraph-pd/docs/configuration.md @@ -0,0 +1,768 @@ +# HugeGraph PD Configuration Guide + +This document provides comprehensive configuration guidance for HugeGraph PD, including parameter descriptions, deployment scenarios, and production tuning recommendations. + +## Table of Contents + +- [Configuration File Overview](#configuration-file-overview) +- [Core Configuration Parameters](#core-configuration-parameters) +- [Deployment Scenarios](#deployment-scenarios) +- [Production Tuning](#production-tuning) +- [Logging Configuration](#logging-configuration) +- [Monitoring and Metrics](#monitoring-and-metrics) + +## Configuration File Overview + +### Configuration Files + +PD uses the following configuration files (located in `conf/` directory): + +| File | Purpose | +|------|---------| +| `application.yml` | Main PD configuration (gRPC, Raft, storage, etc.) | +| `log4j2.xml` | Logging configuration (log levels, appenders, rotation) | +| `verify-license.json` | License verification configuration (optional) | + +### Configuration Hierarchy + +``` +application.yml +├── spring # Spring Boot framework settings +├── management # Actuator endpoints and metrics +├── logging # Log configuration file location +├── license # License verification (optional) +├── grpc # gRPC server settings +├── server # REST API server settings +├── pd # PD-specific settings +├── raft # Raft consensus settings +├── store # Store node management settings +└── partition # Partition management settings +``` + +## Core Configuration Parameters + +### gRPC Settings + +Controls the gRPC server for inter-service communication. + +```yaml +grpc: + host: 127.0.0.1 # gRPC bind address + port: 8686 # gRPC server port +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `grpc.host` | String | `127.0.0.1` | **IMPORTANT**: Must be set to actual IP address (not `127.0.0.1`) for distributed deployments. Store and Server nodes connect to this address. | +| `grpc.port` | Integer | `8686` | gRPC server port. Ensure this port is accessible from Store and Server nodes. | + +**Production Notes**: +- Set `grpc.host` to the node's actual IP address (e.g., `192.168.1.10`) +- Avoid using `0.0.0.0` as it may cause service discovery issues +- Ensure firewall allows incoming connections on `grpc.port` + +### REST API Settings + +Controls the REST API server for management and monitoring. + +```yaml +server: + port: 8620 # REST API port +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `server.port` | Integer | `8620` | REST API port for health checks, metrics, and management operations. | + +**Endpoints**: +- Health check: `http://:8620/actuator/health` +- Metrics: `http://:8620/actuator/metrics` +- Prometheus: `http://:8620/actuator/prometheus` + +### Raft Consensus Settings + +Controls Raft consensus for PD cluster coordination. + +```yaml +raft: + address: 127.0.0.1:8610 # This node's Raft address + peers-list: 127.0.0.1:8610 # All PD nodes in the cluster +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `raft.address` | String | `127.0.0.1:8610` | Raft service address for this PD node. Format: `:`. Must be unique across all PD nodes. | +| `raft.peers-list` | String | `127.0.0.1:8610` | Comma-separated list of all PD nodes' Raft addresses. Used for cluster formation and leader election. | + +**Critical Rules**: +1. `raft.address` must be unique for each PD node +2. `raft.peers-list` must be **identical** on all PD nodes +3. `raft.peers-list` must contain all PD nodes (including this node) +4. Use actual IP addresses, not `127.0.0.1`, for multi-node clusters +5. Cluster size should be odd (3, 5, 7) for optimal Raft quorum + +**Example** (3-node cluster): +```yaml +# Node 1 +raft: + address: 192.168.1.10:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 + +# Node 2 +raft: + address: 192.168.1.11:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 + +# Node 3 +raft: + address: 192.168.1.12:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 +``` + +### PD Core Settings + +Controls PD-specific behavior. + +```yaml +pd: + data-path: ./pd_data # Metadata storage path + patrol-interval: 1800 # Partition rebalancing interval (seconds) + initial-store-count: 1 # Minimum stores for cluster availability + initial-store-list: 127.0.0.1:8500 # Auto-activated stores +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `pd.data-path` | String | `./pd_data` | Directory for RocksDB metadata storage and Raft logs. Ensure sufficient disk space and fast I/O (SSD recommended). | +| `pd.patrol-interval` | Integer | `1800` | Interval (in seconds) for partition health patrol and automatic rebalancing. Lower values = more frequent checks. | +| `pd.initial-store-count` | Integer | `1` | Minimum number of Store nodes required for cluster to be operational. Set to expected initial store count. | +| `pd.initial-store-list` | String | `127.0.0.1:8500` | Comma-separated list of Store gRPC addresses to auto-activate on startup. Useful for bootstrapping. | + +**Production Recommendations**: +- `pd.data-path`: Use dedicated SSD with at least 50GB free space +- `pd.patrol-interval`: + - Development: `300` (5 minutes) for fast testing + - Production: `1800` (30 minutes) to reduce overhead + - Large clusters: `3600` (1 hour) +- `pd.initial-store-count`: Set to expected initial store count (e.g., `3` for 3 stores) + +### Store Management Settings + +Controls how PD monitors and manages Store nodes. + +```yaml +store: + max-down-time: 172800 # Store permanent failure threshold (seconds) + monitor_data_enabled: true # Enable metrics collection + monitor_data_interval: 1 minute # Metrics collection interval + monitor_data_retention: 1 day # Metrics retention period +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `store.max-down-time` | Integer | `172800` | Time (in seconds) after which a Store is considered permanently offline and its partitions are reallocated. Default: 48 hours. | +| `store.monitor_data_enabled` | Boolean | `true` | Enable collection of Store metrics (CPU, memory, disk, partition count). | +| `store.monitor_data_interval` | Duration | `1 minute` | Interval for collecting Store metrics. Format: ` ` (second, minute, hour). | +| `store.monitor_data_retention` | Duration | `1 day` | Retention period for historical metrics. Format: ` ` (day, month, year). | + +**Production Recommendations**: +- `store.max-down-time`: + - Development: `300` (5 minutes) for fast failover testing + - Production: `86400` (24 hours) to avoid false positives during maintenance + - Conservative: `172800` (48 hours) for network instability +- `store.monitor_data_interval`: + - High-frequency monitoring: `10 seconds` + - Standard: `1 minute` + - Low overhead: `5 minutes` +- `store.monitor_data_retention`: + - Short-term: `1 day` + - Standard: `7 days` + - Long-term: `30 days` (requires more disk space) + +### Partition Settings + +Controls partition allocation and replication. + +```yaml +partition: + default-shard-count: 1 # Replicas per partition + store-max-shard-count: 12 # Max partitions per store +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `partition.default-shard-count` | Integer | `1` | Number of replicas per partition. Typically `3` in production for high availability. | +| `partition.store-max-shard-count` | Integer | `12` | Maximum number of partition replicas a single Store can hold. Used for initial partition allocation. | + +**Initial Partition Count Calculation**: +``` +initial_partitions = (store_count * store_max_shard_count) / default_shard_count +``` + +**Example**: +- 3 stores, `store-max-shard-count=12`, `default-shard-count=3` +- Initial partitions: `(3 * 12) / 3 = 12` partitions +- Each store hosts: `12 * 3 / 3 = 12` shards (4 partitions as leader + 8 as follower) + +**Production Recommendations**: +- `partition.default-shard-count`: + - Development/Testing: `1` (no replication) + - Production: `3` (standard HA configuration) + - Critical systems: `5` (maximum fault tolerance) +- `partition.store-max-shard-count`: + - Small deployment: `10-20` + - Medium deployment: `50-100` + - Large deployment: `200-500` + - Limit based on Store disk capacity and expected data volume + +### Management and Metrics + +Controls Spring Boot Actuator endpoints for monitoring. + +```yaml +management: + metrics: + export: + prometheus: + enabled: true # Enable Prometheus metrics export + endpoints: + web: + exposure: + include: "*" # Expose all actuator endpoints +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `management.metrics.export.prometheus.enabled` | Boolean | `true` | Enable Prometheus-compatible metrics at `/actuator/prometheus`. | +| `management.endpoints.web.exposure.include` | String | `"*"` | Actuator endpoints to expose. `"*"` = all, or specify comma-separated list (e.g., `"health,metrics"`). | + +## Deployment Scenarios + +### Single-Node Deployment (Development/Testing) + +Minimal configuration for local development. + +```yaml +grpc: + host: 127.0.0.1 + port: 8686 + +server: + port: 8620 + +raft: + address: 127.0.0.1:8610 + peers-list: 127.0.0.1:8610 + +pd: + data-path: ./pd_data + patrol-interval: 300 # Fast rebalancing for testing + initial-store-count: 1 + initial-store-list: 127.0.0.1:8500 + +store: + max-down-time: 300 # Fast failover for testing + monitor_data_enabled: true + monitor_data_interval: 10 seconds + monitor_data_retention: 1 day + +partition: + default-shard-count: 1 # No replication + store-max-shard-count: 10 +``` + +**Characteristics**: +- Single PD node (no HA) +- No replication (`default-shard-count=1`) +- Fast rebalancing for quick testing +- Suitable for development, not for production + +### 3-Node Cluster Deployment (Production Standard) + +Recommended configuration for production deployments. + +#### Node 1: 192.168.1.10 + +```yaml +grpc: + host: 192.168.1.10 + port: 8686 + +server: + port: 8620 + +raft: + address: 192.168.1.10:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 + +pd: + data-path: /data/pd/metadata + patrol-interval: 1800 + initial-store-count: 3 + initial-store-list: 192.168.1.20:8500,192.168.1.21:8500,192.168.1.22:8500 + +store: + max-down-time: 86400 # 24 hours + monitor_data_enabled: true + monitor_data_interval: 1 minute + monitor_data_retention: 7 days + +partition: + default-shard-count: 3 # Triple replication + store-max-shard-count: 50 +``` + +#### Node 2: 192.168.1.11 + +```yaml +grpc: + host: 192.168.1.11 + port: 8686 + +server: + port: 8620 + +raft: + address: 192.168.1.11:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 + +pd: + data-path: /data/pd/metadata + patrol-interval: 1800 + initial-store-count: 3 + initial-store-list: 192.168.1.20:8500,192.168.1.21:8500,192.168.1.22:8500 + +store: + max-down-time: 86400 + monitor_data_enabled: true + monitor_data_interval: 1 minute + monitor_data_retention: 7 days + +partition: + default-shard-count: 3 + store-max-shard-count: 50 +``` + +#### Node 3: 192.168.1.12 + +```yaml +grpc: + host: 192.168.1.12 + port: 8686 + +server: + port: 8620 + +raft: + address: 192.168.1.12:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 + +pd: + data-path: /data/pd/metadata + patrol-interval: 1800 + initial-store-count: 3 + initial-store-list: 192.168.1.20:8500,192.168.1.21:8500,192.168.1.22:8500 + +store: + max-down-time: 86400 + monitor_data_enabled: true + monitor_data_interval: 1 minute + monitor_data_retention: 7 days + +partition: + default-shard-count: 3 + store-max-shard-count: 50 +``` + +**Characteristics**: +- 3 PD nodes for high availability +- Tolerates 1 PD node failure +- Triple replication (`default-shard-count=3`) +- 3 Store nodes specified in `initial-store-list` +- Standard monitoring and metrics collection + +**Network Requirements**: +- Low latency (<5ms) between PD nodes for Raft +- Open ports: 8620 (REST), 8686 (gRPC), 8610 (Raft) + +### 5-Node Cluster Deployment (High Availability) + +Configuration for mission-critical deployments requiring maximum fault tolerance. + +```yaml +# Node 1: 192.168.1.10 +grpc: + host: 192.168.1.10 + port: 8686 + +raft: + address: 192.168.1.10:8610 + peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610,192.168.1.13:8610,192.168.1.14:8610 + +pd: + data-path: /data/pd/metadata + patrol-interval: 3600 # Lower frequency for large clusters + initial-store-count: 5 + initial-store-list: 192.168.1.20:8500,192.168.1.21:8500,192.168.1.22:8500,192.168.1.23:8500,192.168.1.24:8500 + +store: + max-down-time: 172800 # 48 hours (conservative) + monitor_data_enabled: true + monitor_data_interval: 1 minute + monitor_data_retention: 30 days # Long-term retention + +partition: + default-shard-count: 3 # Or 5 for extreme HA + store-max-shard-count: 100 +``` + +**Characteristics**: +- 5 PD nodes for maximum HA +- Tolerates 2 PD node failures +- 5 Store nodes for data distribution +- Lower patrol frequency to reduce overhead +- Long-term metrics retention (30 days) + +## Production Tuning + +### JVM Tuning + +JVM options are specified via the startup script (`bin/start-hugegraph-pd.sh`). + +#### Memory Configuration + +```bash +# Option 1: Via startup script flag +bin/start-hugegraph-pd.sh -j "-Xmx8g -Xms8g" + +# Option 2: Edit start-hugegraph-pd.sh directly +JAVA_OPTIONS="-Xmx8g -Xms8g -XX:+UseG1GC" +``` + +**Recommendations by Cluster Size**: + +| Cluster Size | Partitions | Heap Size | Notes | +|--------------|------------|-----------|-------| +| Small (1-3 stores, <100 partitions) | <100 | `-Xmx2g -Xms2g` | Development/testing | +| Medium (3-10 stores, 100-1000 partitions) | 100-1000 | `-Xmx4g -Xms4g` | Standard production | +| Large (10-50 stores, 1000-10000 partitions) | 1000-10000 | `-Xmx8g -Xms8g` | Large production | +| X-Large (50+ stores, 10000+ partitions) | 10000+ | `-Xmx16g -Xms16g` | Enterprise scale | + +**Key Principles**: +- Set `-Xms` equal to `-Xmx` to avoid heap resizing +- Reserve at least 2GB for OS and off-heap memory +- Monitor GC pause times and adjust accordingly + +#### Garbage Collection + +**G1GC (Default, Recommended)**: +```bash +bin/start-hugegraph-pd.sh -g g1 -j "-Xmx8g -Xms8g \ + -XX:MaxGCPauseMillis=200 \ + -XX:G1HeapRegionSize=16m \ + -XX:InitiatingHeapOccupancyPercent=45" +``` + +- **MaxGCPauseMillis**: Target GC pause time (200ms recommended) +- **G1HeapRegionSize**: Region size (16m for 8GB heap) +- **InitiatingHeapOccupancyPercent**: When to trigger concurrent GC (45% recommended) + +**ZGC (Low-Latency, Java 11+)**: +```bash +bin/start-hugegraph-pd.sh -g ZGC -j "-Xmx8g -Xms8g \ + -XX:ZCollectionInterval=30" +``` + +- Ultra-low pause times (<10ms) +- Recommended for latency-sensitive deployments +- Requires Java 11+ (Java 15+ for production) + +#### GC Logging + +```bash +-Xlog:gc*:file=logs/gc.log:time,uptime,level,tags:filecount=10,filesize=100M +``` + +### Raft Tuning + +Raft parameters are typically sufficient with defaults, but can be tuned for specific scenarios. + +#### Election Timeout + +Increase election timeout for high-latency networks. + +**Default**: 1000ms (1 second) + +**Tuning** (requires code changes in `RaftEngine.java`): +```java +// In hg-pd-core/.../raft/RaftEngine.java +nodeOptions.setElectionTimeoutMs(3000); // 3 seconds +``` + +**When to Increase**: +- Network latency >10ms between PD nodes +- Frequent false leader elections +- Cross-datacenter deployments + +#### Snapshot Interval + +Control how often Raft snapshots are created. + +**Default**: 3600 seconds (1 hour) + +**Tuning** (in `RaftEngine.java`): +```java +nodeOptions.setSnapshotIntervalSecs(7200); // 2 hours +``` + +**Recommendations**: +- **Frequent snapshots** (1800s): Faster recovery, more I/O overhead +- **Infrequent snapshots** (7200s): Less I/O, slower recovery + +### Disk I/O Optimization + +#### RocksDB Configuration + +PD uses RocksDB for metadata storage. Optimize for your workload. + +**SSD Optimization** (default, recommended): +- RocksDB uses default settings optimized for SSD +- No configuration changes needed + +**HDD Optimization** (not recommended): +If using HDD (not recommended for production): +```java +// In MetadataRocksDBStore.java, customize RocksDB options +Options options = new Options() + .setCompactionStyle(CompactionStyle.LEVEL) + .setWriteBufferSize(64 * 1024 * 1024) // 64MB + .setMaxWriteBufferNumber(3) + .setLevelCompactionDynamicLevelBytes(true); +``` + +**Key Metrics to Monitor**: +- Disk I/O utilization +- RocksDB write stalls +- Compaction backlog + +### Network Tuning + +#### gRPC Connection Pooling + +For high-throughput scenarios, tune gRPC connection pool size. + +**Client-Side** (in `PDClient`): +```java +PDConfig config = PDConfig.builder() + .pdServers("192.168.1.10:8686,192.168.1.11:8686,192.168.1.12:8686") + .maxChannels(5) // Number of gRPC channels per PD node + .build(); +``` + +**Recommendations**: +- Low traffic: `maxChannels=1` +- Medium traffic: `maxChannels=3-5` +- High traffic: `maxChannels=10+` + +#### TCP Tuning (Linux) + +Optimize OS-level TCP settings for low latency. + +```bash +# Increase TCP buffer sizes +sysctl -w net.core.rmem_max=16777216 +sysctl -w net.core.wmem_max=16777216 +sysctl -w net.ipv4.tcp_rmem="4096 87380 16777216" +sysctl -w net.ipv4.tcp_wmem="4096 65536 16777216" + +# Reduce TIME_WAIT connections +sysctl -w net.ipv4.tcp_tw_reuse=1 +sysctl -w net.ipv4.tcp_fin_timeout=30 +``` + +### Monitoring and Alerting + +#### Key Metrics to Monitor + +| Metric | Threshold | Action | +|--------|-----------|--------| +| PD Leader Changes | >2 per hour | Investigate network stability, increase election timeout | +| Raft Log Lag | >1000 entries | Check follower disk I/O, network latency | +| Store Heartbeat Failures | >5% | Check Store node health, network connectivity | +| Partition Imbalance | >20% deviation | Reduce `patrol-interval`, check rebalancing logic | +| GC Pause Time | >500ms | Tune GC settings, increase heap size | +| Disk Usage (`pd.data-path`) | >80% | Clean up old snapshots, expand disk, increase `monitor_data_retention` | + +#### Prometheus Scrape Configuration + +```yaml +scrape_configs: + - job_name: 'hugegraph-pd' + static_configs: + - targets: + - '192.168.1.10:8620' + - '192.168.1.11:8620' + - '192.168.1.12:8620' + metrics_path: '/actuator/prometheus' + scrape_interval: 15s +``` + +#### Grafana Dashboards + +Key panels to create: +- **PD Cluster Status**: Leader, follower count, Raft state +- **Store Health**: Online/offline stores, heartbeat success rate +- **Partition Distribution**: Partitions per store, leader distribution +- **Performance**: QPS, latency (p50, p95, p99) +- **System Resources**: CPU, memory, disk I/O, network + +## Logging Configuration + +### log4j2.xml + +Located at `conf/log4j2.xml`. + +#### Log Levels + +```xml + + + + + + + + + + + + + + + + + + + +``` + +**Recommendations**: +- **Development**: Set PD logger to `DEBUG` for detailed tracing +- **Production**: Use `INFO` (default) or `WARN` for lower overhead +- **Troubleshooting**: Temporarily set specific package to `DEBUG` + +#### Log Rotation + +```xml + + + %d{ISO8601} [%t] %-5level %logger{36} - %msg%n + + + + + + + +``` + +**Configuration**: +- **Size**: Rotate when log file reaches 100MB +- **Time**: Rotate daily +- **Retention**: Keep last 30 log files + +## Monitoring and Metrics + +### Health Check + +```bash +curl http://localhost:8620/actuator/health +``` + +**Response** (healthy): +```json +{ + "status": "UP" +} +``` + +### Metrics Endpoint + +```bash +curl http://localhost:8620/actuator/metrics +``` + +**Available Metrics**: +- `pd.raft.state`: Raft state (0=Follower, 1=Candidate, 2=Leader) +- `pd.store.count`: Number of stores by state +- `pd.partition.count`: Total partitions +- `jvm.memory.used`: JVM memory usage +- `jvm.gc.pause`: GC pause times + +### Prometheus Metrics + +```bash +curl http://localhost:8620/actuator/prometheus +``` + +**Sample Output**: +``` +# HELP pd_raft_state Raft state +# TYPE pd_raft_state gauge +pd_raft_state 2.0 + +# HELP pd_store_count Store count by state +# TYPE pd_store_count gauge +pd_store_count{state="Up"} 3.0 +pd_store_count{state="Offline"} 0.0 + +# HELP pd_partition_count Total partitions +# TYPE pd_partition_count gauge +pd_partition_count 36.0 +``` + +## Configuration Validation + +### Pre-Deployment Checklist + +- [ ] `grpc.host` set to actual IP address (not `127.0.0.1`) +- [ ] `raft.address` unique for each PD node +- [ ] `raft.peers-list` identical on all PD nodes +- [ ] `raft.peers-list` contains all PD node addresses +- [ ] `pd.data-path` has sufficient disk space (>50GB) +- [ ] `pd.initial-store-count` matches expected store count +- [ ] `partition.default-shard-count` = 3 (for production HA) +- [ ] Ports accessible from Store/Server nodes (8620, 8686, 8610) +- [ ] NTP synchronized across all nodes + +### Configuration Validation Tool + +```bash +# Check Raft configuration +grep -A2 "^raft:" conf/application.yml + +# Verify peers list on all nodes +for node in 192.168.1.{10,11,12}; do + echo "Node $node:" + ssh $node "grep peers-list /path/to/conf/application.yml" +done + +# Check port accessibility +nc -zv 192.168.1.10 8620 8686 8610 +``` + +## Summary + +Key configuration guidelines: +- **Single-node**: Use defaults with `127.0.0.1` addresses +- **3-node cluster**: Standard production setup with triple replication +- **5-node cluster**: Maximum HA with increased fault tolerance +- **JVM tuning**: Allocate 4-8GB heap for typical production deployments +- **Monitoring**: Enable Prometheus metrics and create Grafana dashboards + +For architecture details, see [Architecture Documentation](architecture.md). + +For API usage, see [API Reference](api-reference.md). + +For development, see [Development Guide](development.md). diff --git a/hugegraph-pd/docs/development.md b/hugegraph-pd/docs/development.md new file mode 100644 index 0000000000..691fcd9b7c --- /dev/null +++ b/hugegraph-pd/docs/development.md @@ -0,0 +1,854 @@ +# HugeGraph PD Development Guide + +This document provides comprehensive guidance for developing, testing, and contributing to HugeGraph PD. + +## Table of Contents + +- [Development Environment Setup](#development-environment-setup) +- [Building from Source](#building-from-source) +- [Testing](#testing) +- [Development Workflows](#development-workflows) +- [Code Style and Standards](#code-style-and-standards) +- [Debugging](#debugging) +- [Contributing](#contributing) + +## Development Environment Setup + +### Prerequisites + +Ensure you have the following tools installed: + +| Tool | Minimum Version | Recommended | Purpose | +|------|----------------|-------------|---------| +| **JDK** | 11 | 11 or 17 | Java runtime and compilation | +| **Maven** | 3.5.0 | 3.8+ | Build tool and dependency management | +| **Git** | 2.0+ | Latest | Version control | +| **IDE** | N/A | IntelliJ IDEA | Development environment | + +### Verify Installation + +```bash +# Check Java version +java -version +# Expected: openjdk version "11.0.x" or later + +# Check Maven version +mvn -version +# Expected: Apache Maven 3.5.0 or later + +# Check Git version +git --version +# Expected: git version 2.x +``` + +### Clone Repository + +```bash +# Clone HugeGraph repository +git clone https://github.com/apache/hugegraph.git +cd hugegraph + +# PD module location +cd hugegraph-pd +``` + +### IDE Setup (IntelliJ IDEA) + +#### Import Project + +1. Open IntelliJ IDEA +2. **File → Open** → Select `hugegraph-pd` directory +3. Wait for Maven to download dependencies (may take 5-10 minutes) + +#### Configure Code Style + +1. **File → Settings → Editor → Code Style** +2. **Import Scheme → IntelliJ IDEA code style XML** +3. Select `hugegraph-style.xml` from repository root +4. **Apply** and **OK** + +#### Enable Annotation Processing + +Required for Lombok support: + +1. **File → Settings → Build, Execution, Deployment → Compiler → Annotation Processors** +2. Check **Enable annotation processing** +3. **Apply** and **OK** + +#### Configure JDK + +1. **File → Project Structure → Project** +2. **Project SDK**: Select JDK 11 or 17 +3. **Project language level**: 11 +4. **Apply** and **OK** + +## Building from Source + +### Full Build + +Build all PD modules from the `hugegraph-pd` directory: + +```bash +cd hugegraph-pd +mvn clean install -DskipTests +``` + +**Output**: +- JARs in each module's `target/` directory +- Distribution package: `hg-pd-dist/target/hugegraph-pd-.tar.gz` + +**Build Time**: 2-5 minutes (first build may take longer for dependency download) + +### Module-Specific Builds + +Build individual modules: + +```bash +# Build gRPC module only (regenerate proto stubs) +mvn clean compile -pl hg-pd-grpc + +# Build core module only +mvn clean install -pl hg-pd-core -am -DskipTests + +# Build service module only +mvn clean install -pl hg-pd-service -am -DskipTests + +# Build distribution package only +mvn clean package -pl hg-pd-dist -am -DskipTests +``` + +**Maven Flags**: +- `-pl `: Build specific module +- `-am`: Also build required dependencies (--also-make) +- `-DskipTests`: Skip test execution (faster builds) +- `-Dmaven.test.skip=true`: Skip test compilation and execution + +### Clean Build + +Remove all build artifacts: + +```bash +mvn clean + +# This also removes: +# - *.tar, *.tar.gz files +# - .flattened-pom.xml (CI-friendly versioning) +``` + +### Build from Project Root + +Build PD from HugeGraph root directory: + +```bash +cd /path/to/hugegraph + +# Build PD and dependencies +mvn clean package -pl hugegraph-pd -am -DskipTests +``` + +## Testing + +### Test Organization + +PD tests are located in `hg-pd-test/src/main/java/` (non-standard location): + +``` +hg-pd-test/src/main/java/org/apache/hugegraph/pd/ +├── BaseTest.java # Base test class with common setup +├── core/ # Core service tests +│ ├── PartitionServiceTest.java +│ ├── StoreNodeServiceTest.java +│ └── ... +├── client/ # Client library tests +├── raft/ # Raft integration tests +└── PDCoreSuiteTest.java # Test suite (runs all tests) +``` + +### Running Tests + +#### All Tests + +```bash +# Run all PD tests +mvn test + +# Run all tests with coverage report +mvn test jacoco:report +# Coverage report: hg-pd-test/target/site/jacoco/index.html +``` + +#### Module-Specific Tests + +```bash +# Core module tests +mvn test -pl hg-pd-test -am -P pd-core-test + +# Client module tests +mvn test -pl hg-pd-test -am -P pd-client-test + +# Common module tests +mvn test -pl hg-pd-test -am -P pd-common-test + +# REST API tests +mvn test -pl hg-pd-test -am -P pd-rest-test +``` + +#### Single Test Class + +```bash +# Run specific test class +mvn test -pl hg-pd-test -am -Dtest=PartitionServiceTest + +# Run specific test method +mvn test -pl hg-pd-test -am -Dtest=PartitionServiceTest#testSplitPartition +``` + +#### Test from IDE + +**IntelliJ IDEA**: +1. Open test class (e.g., `PartitionServiceTest.java`) +2. Right-click on class name or test method +3. Select **Run 'PartitionServiceTest'** + +### Test Coverage + +View test coverage report: + +```bash +# Generate coverage report +mvn test jacoco:report + +# Open report in browser +open hg-pd-test/target/site/jacoco/index.html +``` + +**Target Coverage**: +- Core services: >80% +- Utility classes: >70% +- Generated gRPC code: Excluded from coverage + +### Integration Tests + +Integration tests start embedded PD instances and verify end-to-end functionality. + +```bash +# Run integration test suite +mvn test -pl hg-pd-test -am -Dtest=PDCoreSuiteTest +``` + +**What Integration Tests Cover**: +- Raft cluster formation and leader election +- Partition allocation and balancing +- Store registration and heartbeat processing +- Metadata persistence and recovery +- gRPC service interactions + +## Development Workflows + +### Adding a New gRPC Service + +#### 1. Define Protocol Buffers + +Create or modify `.proto` file in `hg-pd-grpc/src/main/proto/`: + +```protobuf +// example_service.proto +syntax = "proto3"; + +package org.apache.hugegraph.pd.grpc; + +service ExampleService { + rpc DoSomething(DoSomethingRequest) returns (DoSomethingResponse); +} + +message DoSomethingRequest { + string input = 1; +} + +message DoSomethingResponse { + string output = 1; +} +``` + +#### 2. Generate Java Stubs + +```bash +cd hugegraph-pd +mvn clean compile -pl hg-pd-grpc + +# Generated files location: +# hg-pd-grpc/target/generated-sources/protobuf/java/ +# hg-pd-grpc/target/generated-sources/protobuf/grpc-java/ +``` + +**Note**: Generated files are excluded from source control (`.gitignore`) + +#### 3. Implement Service + +Create service implementation in `hg-pd-service`: + +```java +// ExampleServiceImpl.java +package org.apache.hugegraph.pd.service; + +import io.grpc.stub.StreamObserver; +import org.apache.hugegraph.pd.grpc.ExampleServiceGrpc; + +public class ExampleServiceImpl extends ExampleServiceGrpc.ExampleServiceImplBase { + + @Override + public void doSomething(DoSomethingRequest request, + StreamObserver responseObserver) { + String output = processInput(request.getInput()); + + DoSomethingResponse response = DoSomethingResponse.newBuilder() + .setOutput(output) + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + + private String processInput(String input) { + // Business logic here + return "Processed: " + input; + } +} +``` + +#### 4. Register Service + +Register service in gRPC server (in `hg-pd-service`): + +```java +// In GrpcServerInitializer or similar +ExampleServiceImpl exampleService = new ExampleServiceImpl(); +grpcServer.addService(exampleService); +``` + +#### 5. Add Tests + +Create test class in `hg-pd-test`: + +```java +// ExampleServiceTest.java +package org.apache.hugegraph.pd.service; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class ExampleServiceTest extends BaseTest { + + @Test + public void testDoSomething() { + ExampleServiceImpl service = new ExampleServiceImpl(); + // Test service logic... + } +} +``` + +#### 6. Update Documentation + +Document the new API in `docs/api-reference.md`. + +### Modifying Partition Logic + +Partition logic is in `hg-pd-core/.../PartitionService.java` (2000+ lines). + +**Key Methods**: +- `splitPartition()`: Partition splitting +- `balancePartitions()`: Auto-balancing +- `updatePartitionLeader()`: Leader changes +- `getPartitionByCode()`: Partition routing + +**Development Process**: + +1. **Understand Current Logic**: + ```bash + # Read relevant methods + # File: hg-pd-core/src/main/java/.../PartitionService.java + ``` + +2. **Make Changes**: + - Modify partition allocation algorithm + - Update balancing logic + - Add new partition operations + +3. **Test Changes**: + ```bash + # Run partition service tests + mvn test -pl hg-pd-test -am -Dtest=PartitionServiceTest + + # Run integration tests + mvn test -pl hg-pd-test -am -Dtest=PDCoreSuiteTest + ``` + +4. **Submit Raft Proposals**: + All partition metadata changes must go through Raft: + ```java + // Example: Update partition metadata via Raft + KVOperation operation = KVOperation.put(key, value); + raftTaskHandler.submitTask(operation, closure); + ``` + +### Adding a New Metadata Store + +Metadata stores extend `MetadataRocksDBStore` (in `hg-pd-core/.../meta/`). + +**Example**: Creating `GraphMetaStore`: + +```java +package org.apache.hugegraph.pd.meta; + +public class GraphMetaStore extends MetadataRocksDBStore { + + private static final String GRAPH_PREFIX = "@GRAPH@"; + + public GraphMetaStore(PDConfig config) { + super(config); + } + + public void saveGraph(String graphName, Graph graph) throws PDException { + String key = GRAPH_PREFIX + graphName; + byte[] value = serialize(graph); + put(key.getBytes(), value); + } + + public Graph getGraph(String graphName) throws PDException { + String key = GRAPH_PREFIX + graphName; + byte[] value = get(key.getBytes()); + return deserialize(value, Graph.class); + } + + public List listGraphs() throws PDException { + List graphs = new ArrayList<>(); + String startKey = GRAPH_PREFIX; + String endKey = GRAPH_PREFIX + "\uffff"; + + scan(startKey.getBytes(), endKey.getBytes(), (key, value) -> { + Graph graph = deserialize(value, Graph.class); + graphs.add(graph); + return true; // Continue scanning + }); + + return graphs; + } + + private byte[] serialize(Object obj) { + // Use Hessian2 or Protocol Buffers + } + + private T deserialize(byte[] bytes, Class clazz) { + // Deserialize bytes to object + } +} +``` + +**Testing**: +```java +@Test +public void testGraphMetaStore() { + GraphMetaStore store = new GraphMetaStore(config); + + Graph graph = new Graph("test_graph", 12); + store.saveGraph("test_graph", graph); + + Graph retrieved = store.getGraph("test_graph"); + assertEquals("test_graph", retrieved.getName()); +} +``` + +### Debugging Raft Issues + +Enable detailed Raft logging in `conf/log4j2.xml`: + +```xml + + + + + + + + +``` + +**Raft State Inspection**: +```bash +# Check Raft data directory +ls -lh pd_data/raft/ + +# Raft logs +ls -lh pd_data/raft/log/ + +# Raft snapshots +ls -lh pd_data/raft/snapshot/ +``` + +**Common Raft Issues**: + +| Issue | Symptom | Solution | +|-------|---------|----------| +| **Split-brain** | Multiple leaders | Check `peers-list` consistency, network partitioning | +| **Leader election failure** | Constant candidate state | Check network latency, increase election timeout | +| **Log replication lag** | Followers behind leader | Check follower disk I/O, network bandwidth | +| **Snapshot transfer failure** | Followers can't catch up | Check snapshot directory permissions, disk space | + +## Code Style and Standards + +### Code Formatting + +HugeGraph PD follows Apache HugeGraph code style. + +**Import Code Style**: +1. IntelliJ IDEA: **File → Settings → Editor → Code Style** +2. **Import Scheme** → Select `hugegraph-style.xml` (in repository root) + +**Key Style Rules**: +- **Indentation**: 4 spaces (no tabs) +- **Line length**: 100 characters (Java), 120 characters (comments) +- **Braces**: K&R style (opening brace on same line) +- **Imports**: No wildcard imports (`import java.util.*`) + +### License Headers + +All source files must include Apache License header. + +**Check License Headers**: +```bash +mvn apache-rat:check + +# Output: target/rat.txt (lists files missing license headers) +``` + +**Add License Header**: +Manually add to new files: +```java +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +``` + +### Naming Conventions + +| Type | Convention | Example | +|------|------------|---------| +| **Classes** | PascalCase | `PartitionService`, `StoreNodeService` | +| **Interfaces** | PascalCase (prefix with `I` optional) | `MetadataStore` or `IMetadataStore` | +| **Methods** | camelCase | `getPartition()`, `registerStore()` | +| **Variables** | camelCase | `storeId`, `partitionCount` | +| **Constants** | UPPER_SNAKE_CASE | `MAX_RETRY_COUNT`, `DEFAULT_TIMEOUT` | +| **Packages** | lowercase | `org.apache.hugegraph.pd.core` | + +### JavaDoc + +Public APIs must include JavaDoc comments. + +**Example**: +```java +/** + * Get partition by partition code. + * + * @param graphName the graph name + * @param partitionId the partition ID + * @return the partition metadata + * @throws PDException if partition not found or Raft error + */ +public Partition getPartitionByCode(String graphName, int partitionId) throws PDException { + // Implementation... +} +``` + +**Required for**: +- All public classes and interfaces +- All public and protected methods +- Complex private methods (optional but recommended) + +### Error Handling + +Use custom `PDException` for PD-specific errors. + +**Example**: +```java +if (store == null) { + throw new PDException(ErrorType.STORE_NOT_FOUND, + "Store not found: " + storeId); +} +``` + +**Exception Hierarchy**: +- `PDException`: Base exception for all PD errors +- `RaftException`: Raft-related errors (from JRaft) +- `GrpcException`: gRPC communication errors + +## Debugging + +### Local Debugging in IDE + +#### Run PD from IDE + +1. Create run configuration in IntelliJ IDEA: + - **Run → Edit Configurations** + - **Add New Configuration → Application** + - **Main class**: `org.apache.hugegraph.pd.HgPdApplication` (in `hg-pd-service`) + - **Program arguments**: `--spring.config.location=file:./conf/application.yml` + - **Working directory**: `hugegraph-pd/hg-pd-dist/target/hugegraph-pd-/` + - **JRE**: 11 or 17 + +2. Set breakpoints in code + +3. Click **Debug** (Shift+F9) + +#### Debug Tests + +1. Open test class (e.g., `PartitionServiceTest.java`) +2. Set breakpoints +3. Right-click on test method → **Debug 'testMethod'** + +### Remote Debugging + +Debug PD running on a remote server. + +**Start PD with Debug Port**: +```bash +bin/start-hugegraph-pd.sh -j "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" +``` + +**Connect from IDE**: +1. **Run → Edit Configurations → Add New → Remote JVM Debug** +2. **Host**: PD server IP +3. **Port**: `5005` +4. **Debugger mode**: Attach +5. Click **Debug** + +### Logging + +Increase log verbosity for troubleshooting. + +**Edit `conf/log4j2.xml`**: +```xml + + + + + + + +``` + +**View Logs**: +```bash +# Real-time log monitoring +tail -f logs/hugegraph-pd.log + +# Search logs +grep "ERROR" logs/hugegraph-pd.log +grep "PartitionService" logs/hugegraph-pd.log +``` + +### Performance Profiling + +Use JVM profiling tools to identify performance bottlenecks. + +**Async-profiler** (recommended): +```bash +# Download async-profiler +wget https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.9/async-profiler-2.9-linux-x64.tar.gz +tar -xzf async-profiler-2.9-linux-x64.tar.gz + +# Profile running PD process +./profiler.sh -d 60 -f /tmp/pd-profile.svg + +# View flamegraph +open /tmp/pd-profile.svg +``` + +**JProfiler**: +1. Download JProfiler from https://www.ej-technologies.com/products/jprofiler/overview.html +2. Attach to running PD process +3. Analyze CPU, memory, and thread usage + +## Contributing + +### Contribution Workflow + +1. **Fork Repository**: + - Fork https://github.com/apache/hugegraph on GitHub + +2. **Clone Your Fork**: + ```bash + git clone https://github.com/YOUR_USERNAME/hugegraph.git + cd hugegraph + ``` + +3. **Create Feature Branch**: + ```bash + git checkout -b feature/your-feature-name + ``` + +4. **Make Changes**: + - Write code + - Add tests + - Update documentation + +5. **Run Tests**: + ```bash + mvn test -pl hugegraph-pd/hg-pd-test -am + ``` + +6. **Check Code Style**: + ```bash + mvn apache-rat:check + ``` + +7. **Commit Changes**: + ```bash + git add . + git commit -m "feat(pd): add new feature description" + ``` + + **Commit Message Format**: + ``` + (): + + + +