diff --git a/.github/workflows/_publish_image_reusable.yml b/.github/workflows/_publish_image_reusable.yml new file mode 100644 index 0000000..10dea1f --- /dev/null +++ b/.github/workflows/_publish_image_reusable.yml @@ -0,0 +1,292 @@ +name: "Publish image (reusable)" + +on: + workflow_call: + inputs: + mode: + description: "publish mode: latest or release" + required: true + type: string + component: + description: "component name for cache scope" + required: true + type: string + repository_url: + description: "source repository in owner/name format" + required: true + type: string + branch: + description: "source branch name" + required: true + type: string + build_matrix_json: + description: "build matrix JSON array" + required: true + type: string + use_mvn_args: + description: "whether to pass mvn args to build-args" + required: false + default: false + type: boolean + mvn_args: + description: "mvn build args" + required: false + default: '' + type: string + enable_hash_gate: + description: "whether to skip latest publish if source hash unchanged" + required: false + default: false + type: boolean + last_hash_value: + description: "last published source hash" + required: false + default: '' + type: string + last_hash_name: + description: "repo variable name for latest hash" + required: false + default: '' + type: string + hash_repo_owner: + description: "owner of repo storing LAST_* hash variable" + required: false + default: 'hugegraph' + type: string + hash_repo_name: + description: "repo name storing LAST_* hash variable" + required: false + default: 'actions' + type: string + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_PASSWORD: + required: true + PERSONAL_ACCESS_TOKEN: + required: false + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + need_update: ${{ steps.prepare.outputs.need_update }} + source_sha: ${{ steps.prepare.outputs.source_sha }} + checkout_ref: ${{ steps.prepare.outputs.checkout_ref }} + version_tag: ${{ steps.prepare.outputs.version_tag }} + steps: + - name: Resolve mode and source ref + id: prepare + env: + MODE: ${{ inputs.mode }} + REPOSITORY_URL: ${{ inputs.repository_url }} + BRANCH: ${{ inputs.branch }} + ENABLE_HASH_GATE: ${{ inputs.enable_hash_gate }} + LAST_HASH_VALUE: ${{ inputs.last_hash_value }} + run: | + set -euo pipefail + + if [ "$MODE" != "latest" ] && [ "$MODE" != "release" ]; then + echo "Invalid mode: $MODE. Expected latest or release." + exit 1 + fi + + source_sha="$(git ls-remote "https://github.com/${REPOSITORY_URL}.git" "refs/heads/${BRANCH}" | awk '{print $1}')" + if [ -z "$source_sha" ]; then + echo "Failed to resolve source SHA for ${REPOSITORY_URL}@${BRANCH}" + exit 1 + fi + + checkout_ref="$source_sha" + + if [ "$MODE" = "latest" ]; then + version_tag="latest" + need_update="true" + if [ "$ENABLE_HASH_GATE" = "true" ] && [ "$source_sha" = "$LAST_HASH_VALUE" ]; then + need_update="false" + fi + else + version_tag="$(echo "$BRANCH" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n 1)" + if [ -z "$version_tag" ]; then + echo "Branch name does not contain a valid version number (x.x.x): $BRANCH" + exit 1 + fi + need_update="true" + fi + + { + echo "source_sha=$source_sha" + echo "checkout_ref=$checkout_ref" + echo "version_tag=$version_tag" + echo "need_update=$need_update" + } >> "$GITHUB_OUTPUT" + + publish: + needs: prepare + if: ${{ needs.prepare.outputs.need_update == 'true' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.build_matrix_json) }} + steps: + - name: Resolve build parameters (${{ matrix.module }}) + id: params + env: + MODE: ${{ inputs.mode }} + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + COMPONENT: ${{ inputs.component }} + MODULE: ${{ matrix.module }} + IMAGE_REPO_LATEST: ${{ matrix.image_repo_latest }} + IMAGE_REPO_RELEASE: ${{ matrix.image_repo_release }} + PLATFORMS_LATEST: ${{ matrix.platforms_latest }} + PLATFORMS_RELEASE: ${{ matrix.platforms_release }} + run: | + set -euo pipefail + + image_repo="$IMAGE_REPO_LATEST" + platforms="$PLATFORMS_LATEST" + if [ "$MODE" = "release" ]; then + image_repo="$IMAGE_REPO_RELEASE" + platforms="$PLATFORMS_RELEASE" + fi + + if [ -z "$image_repo" ] || [ -z "$platforms" ]; then + echo "Missing image repo or platforms for module: $MODULE" + exit 1 + fi + + image_url="${image_repo}:${VERSION_TAG}" + cache_scope="${COMPONENT}-${MODULE}" + + { + echo "image_url=$image_url" + echo "platforms=$platforms" + echo "cache_scope=$cache_scope" + } >> "$GITHUB_OUTPUT" + + - name: Checkout source (${{ matrix.module }}) + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository_url }} + ref: ${{ needs.prepare.outputs.checkout_ref }} + fetch-depth: 2 + + - name: Set up QEMU (${{ matrix.module }}) + if: ${{ contains(steps.params.outputs.platforms, 'arm64') }} + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build x86 image for smoke test with mvn args (${{ matrix.module }}) + if: ${{ matrix.smoke_test && inputs.use_mvn_args }} + uses: docker/build-push-action@v7 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + tags: ${{ steps.params.outputs.image_url }} + cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }} + cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=min + build-args: ${{ inputs.mvn_args }} + + - name: Build x86 image for smoke test (${{ matrix.module }}) + if: ${{ matrix.smoke_test && !inputs.use_mvn_args }} + uses: docker/build-push-action@v7 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + tags: ${{ steps.params.outputs.image_url }} + cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }} + cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=min + + - name: Run smoke test (${{ matrix.module }}) + if: ${{ matrix.smoke_test }} + env: + IMAGE_URL: ${{ steps.params.outputs.image_url }} + SMOKE_TEST_CMD: ${{ matrix.smoke_test_cmd }} + run: | + set -euo pipefail + if [ -z "$SMOKE_TEST_CMD" ]; then + echo "smoke_test_cmd is empty while smoke_test=true" + exit 1 + fi + bash -euo pipefail -c "$SMOKE_TEST_CMD" + + - name: Cleanup smoke test container (${{ matrix.module }}) + if: ${{ always() && matrix.smoke_test }} + run: | + docker rm -f graph || true + + - name: Build and push image with mvn args (${{ matrix.module }}) + if: ${{ inputs.use_mvn_args }} + uses: docker/build-push-action@v7 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: ${{ steps.params.outputs.platforms }} + push: true + tags: ${{ steps.params.outputs.image_url }} + cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }} + cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=max + build-args: ${{ inputs.mvn_args }} + + - name: Build and push image (${{ matrix.module }}) + if: ${{ !inputs.use_mvn_args }} + uses: docker/build-push-action@v7 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: ${{ steps.params.outputs.platforms }} + push: true + tags: ${{ steps.params.outputs.image_url }} + cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }} + cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=max + + update_latest_hash: + needs: [prepare, publish] + if: ${{ inputs.mode == 'latest' && inputs.enable_hash_gate && needs.prepare.outputs.need_update == 'true' && needs.publish.result == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Validate hash update inputs + env: + LAST_HASH_NAME: ${{ inputs.last_hash_name }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + run: | + set -euo pipefail + if [ -z "$LAST_HASH_NAME" ]; then + echo "last_hash_name is required when enable_hash_gate=true" + exit 1 + fi + if [ -z "$PERSONAL_ACCESS_TOKEN" ]; then + echo "PERSONAL_ACCESS_TOKEN is required to update latest hash" + exit 1 + fi + + - name: Update latest source hash variable + env: + OWNER: ${{ inputs.hash_repo_owner }} + REPO: ${{ inputs.hash_repo_name }} + LAST_HASH_NAME: ${{ inputs.last_hash_name }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + run: | + set -euo pipefail + curl --fail-with-body -sS -L -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "Authorization: Bearer $PERSONAL_ACCESS_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/variables/$LAST_HASH_NAME" \ + -d '{"name":"'"$LAST_HASH_NAME"'","value":"'"$SOURCE_SHA"'"}' diff --git a/.github/workflows/publish_hugegraph_hubble.yml b/.github/workflows/publish_hugegraph_hubble.yml index 7f5c4fa..08a2d86 100644 --- a/.github/workflows/publish_hugegraph_hubble.yml +++ b/.github/workflows/publish_hugegraph_hubble.yml @@ -7,7 +7,7 @@ on: inputs: repository_url: required: true - default: 'apache/incubator-hugegraph-toolchain' + default: 'apache/hugegraph-toolchain' repository_branch: required: true default: 'master' diff --git a/.github/workflows/publish_latest_ai_image.yml b/.github/workflows/publish_latest_ai_image.yml index ae95770..baae8d4 100644 --- a/.github/workflows/publish_latest_ai_image.yml +++ b/.github/workflows/publish_latest_ai_image.yml @@ -4,83 +4,44 @@ on: schedule: - cron: '0 23 * * *' workflow_dispatch: - -jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/incubator-hugegraph-ai - BRANCH: main - IMAGE_URL: hugegraph/rag:latest - IMAGE_URL_NK: hugegraph/rag-bin:latest - GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - OWNER: hugegraph - REPO: actions - LAST_AI_HASH: ${{vars.LAST_AI_HASH}} - - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Get current commit-hash - run: | - current_commit_hash=$(git rev-parse HEAD) - echo "CURRENT_COMMIT_HASH=$current_commit_hash" >> $GITHUB_ENV - - name: Check if an update is needed - run: | - need_update='false' - if [[ "$CURRENT_COMMIT_HASH" != "$LAST_AI_HASH" ]]; then - need_update='true' - fi - echo "NEED_UPDATE=$need_update" >> $GITHUB_ENV - # TODO: replace the `if` statements with a graceful exit once GitHub Actions supports it. - # See https://github.com/actions/runner/issues/662 for more details. - - - name: Build X86 & ARM And Push All - uses: docker/build-push-action@v5 - if: ${{ env.NEED_UPDATE == 'true' }} - with: - context: . - file: ./docker/Dockerfile.llm - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build Binary X86 And Push All - uses: docker/build-push-action@v5 - if: ${{ env.NEED_UPDATE == 'true' }} - with: - context: . - file: ./docker/Dockerfile.nk - # platforms: linux/amd64,linux/arm64 - platforms: linux/amd64 - tags: ${{ env.IMAGE_URL_NK }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Update last commit-hash - if: ${{ env.NEED_UPDATE == 'true' }} - run: | - curl -L -X PATCH \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - https://api.github.com/repos/$OWNER/$REPO/actions/variables/LAST_AI_HASH \ - -d '{"name":"LAST_AI_HASH","value":"'"$CURRENT_COMMIT_HASH"'"}' +jobs: + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: latest + component: ai + repository_url: apache/hugegraph-ai + branch: main + # IMPORTANT: + # - Keep ARM enabled for normal RAG image (Dockerfile.llm). + # - Keep rag-bin (Dockerfile.nk) on amd64 only due to arm incompatibility. + build_matrix_json: | + [ + { + "module": "rag", + "context": ".", + "dockerfile": "./docker/Dockerfile.llm", + "image_repo_latest": "hugegraph/rag", + "image_repo_release": "hugegraph/rag", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + }, + { + "module": "rag-bin", + "context": ".", + "dockerfile": "./docker/Dockerfile.nk", + "image_repo_latest": "hugegraph/rag-bin", + "image_repo_release": "hugegraph/rag-bin", + "platforms_latest": "linux/amd64", + "platforms_release": "linux/amd64", + "smoke_test": false + } + ] + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_AI_HASH }} + last_hash_name: LAST_AI_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_latest_hubble_image.yml b/.github/workflows/publish_latest_hubble_image.yml index 0029aeb..35af9d1 100644 --- a/.github/workflows/publish_latest_hubble_image.yml +++ b/.github/workflows/publish_latest_hubble_image.yml @@ -7,73 +7,35 @@ on: inputs: mvn_args: required: false - default: 'MAVEN_ARGS=-P stage' + default: '' description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph-toolchain - BRANCH: master - IMAGE_URL: hugegraph/hubble:latest - GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - OWNER: hugegraph - REPO: actions - LAST_HUBBLE_HASH: ${{vars.LAST_HUBBLE_HASH}} - MVN_ARGS: ${{ github.event.inputs.mvn_args || 'MAVEN_ARGS=-P stage' }} - - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Get current commit-hash - run: | - current_commit_hash=$(git rev-parse HEAD) - echo "CURRENT_COMMIT_HASH=$current_commit_hash" >> $GITHUB_ENV - - name: Check if an update is needed - run: | - need_update='false' - if [[ "$CURRENT_COMMIT_HASH" != "$LAST_HUBBLE_HASH" ]]; then - need_update='true' - fi - echo "NEED_UPDATE=$need_update" >> $GITHUB_ENV - # TODO: replace `if` statements for exit if github provide support for exit gracefully, - # see https://github.com/actions/runner/issues/662 for more details - - - name: Build X86 & ARM And Push All - uses: docker/build-push-action@v5 - if: ${{ env.NEED_UPDATE == 'true' }} - with: - context: . - file: ./hugegraph-hubble/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} - - - name: Updata last commit-hash - if: ${{ env.NEED_UPDATE == 'true' }} - run: | - curl -L -X PATCH \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - https://api.github.com/repos/$OWNER/$REPO/actions/variables/LAST_HUBBLE_HASH \ - -d '{"name":"LAST_HUBBLE_HASH","value":"'"$CURRENT_COMMIT_HASH"'"}' +jobs: + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: latest + component: hubble + repository_url: apache/hugegraph-toolchain + branch: master + build_matrix_json: | + [ + { + "module": "hubble", + "context": ".", + "dockerfile": "./hugegraph-hubble/Dockerfile", + "image_repo_latest": "hugegraph/hubble", + "image_repo_release": "hugegraph/hubble", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + } + ] + use_mvn_args: true + mvn_args: ${{ github.event.inputs.mvn_args || '' }} + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_HUBBLE_HASH }} + last_hash_name: LAST_HUBBLE_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_latest_loader_image.yml b/.github/workflows/publish_latest_loader_image.yml index 2f6ec49..e41ea84 100644 --- a/.github/workflows/publish_latest_loader_image.yml +++ b/.github/workflows/publish_latest_loader_image.yml @@ -7,73 +7,35 @@ on: inputs: mvn_args: required: false - default: 'MAVEN_ARGS=-P stage' + default: '' description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph-toolchain - BRANCH: master - IMAGE_URL: hugegraph/loader:latest - GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - OWNER: hugegraph - REPO: actions - LAST_LOADER_HASH: ${{vars.LAST_LOADER_HASH}} - MVN_ARGS: ${{ github.event.inputs.mvn_args || 'MAVEN_ARGS=-P stage' }} - - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - name: Get current commit-hash - run: | - current_commit_hash=$(git rev-parse HEAD) - echo "CURRENT_COMMIT_HASH=$current_commit_hash" >> $GITHUB_ENV - - - name: Check if an update is needed - run: | - need_update='false' - if [[ "$CURRENT_COMMIT_HASH" != "$LAST_LOADER_HASH" ]]; then - need_update='true' - fi - echo "NEED_UPDATE=$need_update" >> $GITHUB_ENV - # TODO: replace `if` statements for exit if github provide support for exit gracefully, - # see https://github.com/actions/runner/issues/662 for more details - - - name: Build X86 & ARM And Push All - uses: docker/build-push-action@v5 - if: ${{ env.NEED_UPDATE == 'true' }} - with: - context: . - file: ./hugegraph-loader/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} - - - name: Updata last commit-hash - if: ${{ env.NEED_UPDATE == 'true' }} - run: | - curl -L -X PATCH \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - https://api.github.com/repos/$OWNER/$REPO/actions/variables/LAST_LOADER_HASH \ - -d '{"name":"LAST_LOADER_HASH","value":"'"$CURRENT_COMMIT_HASH"'"}' +jobs: + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: latest + component: loader + repository_url: apache/hugegraph-toolchain + branch: master + build_matrix_json: | + [ + { + "module": "loader", + "context": ".", + "dockerfile": "./hugegraph-loader/Dockerfile", + "image_repo_latest": "hugegraph/loader", + "image_repo_release": "hugegraph/loader", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + } + ] + use_mvn_args: true + mvn_args: ${{ github.event.inputs.mvn_args || '' }} + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_LOADER_HASH }} + last_hash_name: LAST_LOADER_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_latest_server_image.yml b/.github/workflows/publish_latest_server_image.yml index 1f98ac9..1935bf5 100644 --- a/.github/workflows/publish_latest_server_image.yml +++ b/.github/workflows/publish_latest_server_image.yml @@ -9,97 +9,34 @@ on: required: false default: '' description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph - BRANCH: master - IMAGE_URL: hugegraph/hugegraph:latest - GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - OWNER: hugegraph - REPO: actions - LAST_SERVER_HASH: ${{vars.LAST_SERVER_HASH}} - MVN_ARGS: ${{ github.event.inputs.mvn_args || 'MAVEN_ARGS=-P stage' }} - - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Get current commit-hash - run: | - current_commit_hash=$(git rev-parse HEAD) - echo "CURRENT_COMMIT_HASH=$current_commit_hash" >> $GITHUB_ENV - - name: Check if an update is needed - run: | - need_update='false' - if [[ "$CURRENT_COMMIT_HASH" != "$LAST_SERVER_HASH" ]]; then - need_update='true' - else - docker pull hugegraph/hugegraph - fi - echo "NEED_UPDATE=$need_update" >> $GITHUB_ENV - # TODO: replace `if` statements for exit if github provide support for exit gracefully, - # see https://github.com/actions/runner/issues/662 for more details - - - name: Build X86 Image - if: ${{ env.NEED_UPDATE == 'true' }} - uses: docker/build-push-action@v5 - with: - context: . - file: ./hugegraph-server/Dockerfile - load: true - tags: ${{ env.IMAGE_URL }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} - - - name: Test X86 Image - if: ${{ env.NEED_UPDATE == 'true' }} - run: | - docker images - docker run -itd --name=graph -p 18080:8080 $IMAGE_URL - sleep 20s - curl 0.0.0.0:18080 || exit - docker ps -a - sleep 20s - curl 0.0.0.0:18080 || exit - docker ps -a - - - name: Build ARM & Push all - if: ${{ env.NEED_UPDATE == 'true' }} - uses: docker/build-push-action@v5 - with: - context: . - file: ./hugegraph-server/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ env.IMAGE_URL }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} - - - name: Updata last commit-hash - if: ${{ env.NEED_UPDATE == 'true' }} - run: | - curl -L -X PATCH \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - https://api.github.com/repos/$OWNER/$REPO/actions/variables/LAST_SERVER_HASH \ - -d '{"name":"LAST_SERVER_HASH","value":"'"$CURRENT_COMMIT_HASH"'"}' +jobs: + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: latest + component: server + repository_url: apache/hugegraph + branch: master + build_matrix_json: | + [ + { + "module": "server", + "context": ".", + "dockerfile": "./hugegraph-server/Dockerfile", + "image_repo_latest": "hugegraph/hugegraph", + "image_repo_release": "hugegraph/hugegraph", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": true, + "smoke_test_cmd": "docker rm -f graph >/dev/null 2>&1 || true; docker run -itd --name=graph -p 18080:8080 $IMAGE_URL; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a" + } + ] + use_mvn_args: true + mvn_args: ${{ github.event.inputs.mvn_args || '' }} + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_SERVER_HASH }} + last_hash_name: LAST_SERVER_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_latest_vermeer_image.yml b/.github/workflows/publish_latest_vermeer_image.yml index 2db11c3..6de4eeb 100644 --- a/.github/workflows/publish_latest_vermeer_image.yml +++ b/.github/workflows/publish_latest_vermeer_image.yml @@ -4,69 +4,31 @@ on: schedule: - cron: '0 23 * * *' workflow_dispatch: - -jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/incubator-hugegraph-computer - BRANCH: master - IMAGE_URL: hugegraph/vermeer:latest - GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - OWNER: hugegraph - REPO: actions - LAST_VERMEER_HASH: ${{vars.LAST_VERMEER_HASH}} - - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Get current commit-hash - run: | - current_commit_hash=$(git rev-parse HEAD) - echo "CURRENT_COMMIT_HASH=$current_commit_hash" >> $GITHUB_ENV - - name: Check if an update is needed - run: | - need_update='false' - if [[ "$CURRENT_COMMIT_HASH" != "$LAST_VERMEER_HASH" ]]; then - need_update='true' - fi - echo "NEED_UPDATE=$need_update" >> $GITHUB_ENV - # TODO: replace the `if` statements with a graceful exit once GitHub Actions supports it. - # See https://github.com/actions/runner/issues/662 for more details. - - - name: Build X86 & ARM And Push All - uses: docker/build-push-action@v5 - if: ${{ env.NEED_UPDATE == 'true' }} - with: - context: ./vermeer - file: ./vermeer/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Update last commit-hash - if: ${{ env.NEED_UPDATE == 'true' }} - run: | - curl -L -X PATCH \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - https://api.github.com/repos/$OWNER/$REPO/actions/variables/LAST_VERMEER_HASH \ - -d '{"name":"LAST_VERMEER_HASH","value":"'"$CURRENT_COMMIT_HASH"'"}' +jobs: + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: latest + component: vermeer + repository_url: apache/hugegraph-computer + branch: master + build_matrix_json: | + [ + { + "module": "vermeer", + "context": "./vermeer", + "dockerfile": "./vermeer/Dockerfile", + "image_repo_latest": "hugegraph/vermeer", + "image_repo_release": "hugegraph/vermeer", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + } + ] + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_VERMEER_HASH }} + last_hash_name: LAST_VERMEER_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_release_ai_image.yml b/.github/workflows/publish_release_ai_image.yml index 9e197d1..18bc638 100644 --- a/.github/workflows/publish_release_ai_image.yml +++ b/.github/workflows/publish_release_ai_image.yml @@ -1,4 +1,5 @@ name: "Publish AI Image(release)" + on: workflow_dispatch: inputs: @@ -8,56 +9,37 @@ on: description: 'The branch name should be like *-x.x.x, for example release-1.0.0' jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/incubator-hugegraph-ai - BRANCH: ${{inputs.branch}} - - steps: - - name: Set image_url - run: | - version=$(echo "${{ inputs.branch }}" | grep -oP '(\d+\.\d+\.\d+)') || exit 1 - echo "IMAGE_URL=hugegraph/rag:$version" >> $GITHUB_ENV - - name: Set image_url_nk - run: | - image_url_nk=hugegraph/rag-bin:$(echo "${{ inputs.branch }}" | grep -oP '(\d+\.\d+\.\d+)') - echo $image_url_nk && echo "IMAGE_URL_NK=$image_url_nk" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Build X86 & ARM and Push All - uses: docker/build-push-action@v5 - with: - context: . - file: ./docker/Dockerfile.llm - platforms: linux/amd64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build Binary X86 & ARM and Push All - uses: docker/build-push-action@v5 - with: - context: . - file: ./docker/Dockerfile.nk - platforms: linux/amd64 - tags: ${{ env.IMAGE_URL_NK }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: release + component: ai + repository_url: apache/hugegraph-ai + branch: ${{ inputs.branch }} + # IMPORTANT: + # - Keep ARM enabled for normal RAG image (Dockerfile.llm). + # - Keep rag-bin (Dockerfile.nk) on amd64 only due to arm incompatibility. + build_matrix_json: | + [ + { + "module": "rag", + "context": ".", + "dockerfile": "./docker/Dockerfile.llm", + "image_repo_latest": "hugegraph/rag", + "image_repo_release": "hugegraph/rag", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + }, + { + "module": "rag-bin", + "context": ".", + "dockerfile": "./docker/Dockerfile.nk", + "image_repo_latest": "hugegraph/rag-bin", + "image_repo_release": "hugegraph/rag-bin", + "platforms_latest": "linux/amd64", + "platforms_release": "linux/amd64", + "smoke_test": false + } + ] + secrets: inherit diff --git a/.github/workflows/publish_release_hubble_image.yml b/.github/workflows/publish_release_hubble_image.yml index fc75b28..329bb99 100644 --- a/.github/workflows/publish_release_hubble_image.yml +++ b/.github/workflows/publish_release_hubble_image.yml @@ -1,4 +1,5 @@ name: "Publish hubble image(release)" + on: workflow_dispatch: inputs: @@ -6,49 +7,32 @@ on: required: true default: '' description: 'The branch name should be like *-x.x.x, for example release-1.0.0' - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' + mvn_args: + required: false + default: '' + description: 'mvn build args, like "MAVEN_ARGS=-P stage"' jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph-toolchain - BRANCH: ${{inputs.branch}} - MVN_ARGS: ${{inputs.mvn_args}} - - steps: - - name: Set image_url - run: | - image_url=hugegraph/hubble:$(echo "${{ inputs.branch }}" | grep -oP '(\d+\.\d+\.\d+)') - echo $image_url && echo "IMAGE_URL=$image_url" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Build X86 &ARM And Push All - uses: docker/build-push-action@v5 - with: - context: . - file: ./hugegraph-hubble/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: release + component: hubble + repository_url: apache/hugegraph-toolchain + branch: ${{ inputs.branch }} + build_matrix_json: | + [ + { + "module": "hubble", + "context": ".", + "dockerfile": "./hugegraph-hubble/Dockerfile", + "image_repo_latest": "hugegraph/hubble", + "image_repo_release": "hugegraph/hubble", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + } + ] + use_mvn_args: true + mvn_args: ${{ inputs.mvn_args }} + secrets: inherit diff --git a/.github/workflows/publish_release_loader_image.yml b/.github/workflows/publish_release_loader_image.yml index 6fbd2eb..e923f37 100644 --- a/.github/workflows/publish_release_loader_image.yml +++ b/.github/workflows/publish_release_loader_image.yml @@ -1,4 +1,5 @@ name: "Publish loader image(release)" + on: workflow_dispatch: inputs: @@ -6,49 +7,32 @@ on: required: true default: '' description: 'The branch name should be like *-x.x.x, for example release-1.0.0' - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' + mvn_args: + required: false + default: '' + description: 'mvn build args, like "MAVEN_ARGS=-P stage"' jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph-toolchain - BRANCH: ${{inputs.branch}} - MVN_ARGS: ${{inputs.mvn_args}} - - steps: - - name: Set image_url - run: | - image_url=hugegraph/loader:$(echo "${{ inputs.branch }}" | grep -oP '(\d+\.\d+\.\d+)') - echo $image_url && echo "IMAGE_URL=$image_url" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Build X86 &ARM And Push All - uses: docker/build-push-action@v5 - with: - context: . - file: ./hugegraph-loader/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: release + component: loader + repository_url: apache/hugegraph-toolchain + branch: ${{ inputs.branch }} + build_matrix_json: | + [ + { + "module": "loader", + "context": ".", + "dockerfile": "./hugegraph-loader/Dockerfile", + "image_repo_latest": "hugegraph/loader", + "image_repo_release": "hugegraph/loader", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + } + ] + use_mvn_args: true + mvn_args: ${{ inputs.mvn_args }} + secrets: inherit diff --git a/.github/workflows/publish_release_server_image.yml b/.github/workflows/publish_release_server_image.yml index 30bbc58..aa39dd7 100644 --- a/.github/workflows/publish_release_server_image.yml +++ b/.github/workflows/publish_release_server_image.yml @@ -1,4 +1,5 @@ name: "Publish server image(release)" + on: workflow_dispatch: inputs: @@ -6,71 +7,33 @@ on: required: true default: '' description: 'The branch name should be like *-x.x.x, for example release-1.0.0' - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' + mvn_args: + required: false + default: '' + description: 'mvn build args, like "MAVEN_ARGS=-P stage"' jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph - BRANCH: ${{inputs.branch}} - MVN_ARGS: ${{inputs.mvn_args}} - - steps: - - name: Set image_url - run: | - image_url=hugegraph/hugegraph:$(echo "${{ inputs.branch }}" | grep -oP '(\d+\.\d+\.\d+)') - echo $image_url && echo "image_url=$image_url" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Build X86 Image - uses: docker/build-push-action@v5 - with: - context: . - file: ./hugegraph-server/Dockerfile - load: true - tags: ${{ env.image_url }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Test X86 Image - run: | - echo "$image_url" - docker images - docker run -itd --name=graph -p 18080:8080 $image_url - sleep 20s - curl 0.0.0.0:18080 || exit - docker ps -a - sleep 20s - curl 0.0.0.0:18080 || exit - docker ps -a - - - name: Build ARM & Push all - uses: docker/build-push-action@v5 - with: - context: . - file: ./hugegraph-server/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ env.image_url }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ env.MVN_ARGS }} + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: release + component: server + repository_url: apache/hugegraph + branch: ${{ inputs.branch }} + build_matrix_json: | + [ + { + "module": "server", + "context": ".", + "dockerfile": "./hugegraph-server/Dockerfile", + "image_repo_latest": "hugegraph/hugegraph", + "image_repo_release": "hugegraph/hugegraph", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": true, + "smoke_test_cmd": "docker rm -f graph >/dev/null 2>&1 || true; docker run -itd --name=graph -p 18080:8080 $IMAGE_URL; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a" + } + ] + use_mvn_args: true + mvn_args: ${{ inputs.mvn_args }} + secrets: inherit diff --git a/.github/workflows/publish_release_vermeer_image.yml b/.github/workflows/publish_release_vermeer_image.yml index e6cef28..43838d2 100644 --- a/.github/workflows/publish_release_vermeer_image.yml +++ b/.github/workflows/publish_release_vermeer_image.yml @@ -7,48 +7,26 @@ on: required: true default: '' description: 'The branch name should be like *-x.x.x, for example release-1.0.0' - -jobs: - build_latest: - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/incubator-hugegraph-computer - BRANCH: ${{inputs.branch}} - - steps: - - name: Set image_url - run: | - version=$(echo "${{ inputs.branch }}" | grep -oP '(\d+\.\d+\.\d+)') - if [ -z "$version" ]; then - echo "Error: Branch name does not contain a valid version number (x.x.x)." >&2 - exit 1 - fi - image_url=apache/vermeer:$version - echo $image_url && echo "IMAGE_URL=$image_url" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.BRANCH }} - fetch-depth: 2 - - - name: Build X86 & ARM And Push All - uses: docker/build-push-action@v5 - with: - context: ./vermeer - file: ./vermeer/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_URL }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max +jobs: + publish: + uses: ./.github/workflows/_publish_image_reusable.yml + with: + mode: release + component: vermeer + repository_url: apache/hugegraph-computer + branch: ${{ inputs.branch }} + build_matrix_json: | + [ + { + "module": "vermeer", + "context": "./vermeer", + "dockerfile": "./vermeer/Dockerfile", + "image_repo_latest": "hugegraph/vermeer", + "image_repo_release": "hugegraph/vermeer", + "platforms_latest": "linux/amd64,linux/arm64", + "platforms_release": "linux/amd64,linux/arm64", + "smoke_test": false + } + ] + secrets: inherit