From d12541233cf2484c54220a48e7bfe00bfcf4a9c9 Mon Sep 17 00:00:00 2001 From: "zhoujiahui.01" Date: Wed, 8 Apr 2026 11:57:52 +0800 Subject: [PATCH] fix(docker): restore venv tooling for ragfs build fix(docker): preserve ragfs wheel extraction script indentation fix(docker): keep heredoc cleanup inside run shell fix(docker): terminate ragfs extraction heredoc cleanly fix(ci): split docker manifest digests by registry --- .github/workflows/build-docker-image.yml | 69 +++++++++++++++---- .github/workflows/release.yml | 68 ++++++++++++++---- Dockerfile | 49 +++++++------ .../test_docker_workflow_native_multiarch.py | 30 ++++++++ 4 files changed, 168 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index e9240f0c4..de86e1cf2 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -75,31 +75,57 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Build and push Docker image - id: push + - name: Build and push Docker image to GHCR + id: push-ghcr uses: docker/build-push-action@v7 with: context: . platforms: ${{ matrix.platform }} outputs: | type=image,name=${{ env.REGISTRY }}/${{ steps.image-name.outputs.image }},push-by-digest=true,name-canonical=true,push=true + labels: ${{ steps.meta.outputs.labels }} + build-args: | + # fallback to 0.0.0 if no version is provided + OPENVIKING_VERSION=${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.version) || (github.ref_type == 'tag' && github.ref_name) || '0.0.0' }} + + - name: Build and push Docker image to Docker Hub + id: push-dockerhub + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + outputs: | type=image,name=docker.io/${{ secrets.DOCKERHUB_USERNAME }}/openviking,push-by-digest=true,name-canonical=true,push=true labels: ${{ steps.meta.outputs.labels }} build-args: | # fallback to 0.0.0 if no version is provided OPENVIKING_VERSION=${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.version) || (github.ref_type == 'tag' && github.ref_name) || '0.0.0' }} - - name: Export image digest + - name: Export GHCR image digest + run: | + mkdir -p /tmp/digests-ghcr + ghcr_digest="${{ steps.push-ghcr.outputs.digest }}" + touch "/tmp/digests-ghcr/${ghcr_digest#sha256:}" + + - name: Upload GHCR image digest + uses: actions/upload-artifact@v7 + with: + name: docker-digests-ghcr-${{ matrix.arch }} + path: /tmp/digests-ghcr/* + if-no-files-found: error + retention-days: 1 + + - name: Export Docker Hub image digest run: | - mkdir -p /tmp/digests - digest="${{ steps.push.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" + mkdir -p /tmp/digests-dockerhub + dockerhub_digest="${{ steps.push-dockerhub.outputs.digest }}" + touch "/tmp/digests-dockerhub/${dockerhub_digest#sha256:}" - - name: Upload image digest + - name: Upload Docker Hub image digest uses: actions/upload-artifact@v7 with: - name: docker-digests-${{ matrix.arch }} - path: /tmp/digests/* + name: docker-digests-dockerhub-${{ matrix.arch }} + path: /tmp/digests-dockerhub/* if-no-files-found: error retention-days: 1 @@ -152,11 +178,18 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Download image digests + - name: Download GHCR image digests uses: actions/download-artifact@v8 with: - pattern: docker-digests-* - path: /tmp/digests + pattern: docker-digests-ghcr-* + path: /tmp/digests-ghcr + merge-multiple: true + + - name: Download Docker Hub image digests + uses: actions/download-artifact@v8 + with: + pattern: docker-digests-dockerhub-* + path: /tmp/digests-dockerhub merge-multiple: true - name: Create multi-arch manifests @@ -166,15 +199,23 @@ jobs: # Collect image references for both registries ghcr_image_refs=() dockerhub_image_refs=() - for digest_file in /tmp/digests/*; do + for digest_file in /tmp/digests-ghcr/*; do [ -e "$digest_file" ] || continue digest="sha256:$(basename "$digest_file")" ghcr_image_refs+=("${{ env.REGISTRY }}/${{ steps.image-name.outputs.image }}@${digest}") + done + for digest_file in /tmp/digests-dockerhub/*; do + [ -e "$digest_file" ] || continue + digest="sha256:$(basename "$digest_file")" dockerhub_image_refs+=("docker.io/${{ secrets.DOCKERHUB_USERNAME }}/openviking@${digest}") done [ ${#ghcr_image_refs[@]} -gt 0 ] || { - echo "No image digests found" >&2 + echo "No GHCR image digests found" >&2 + exit 1 + } + [ ${#dockerhub_image_refs[@]} -gt 0 ] || { + echo "No Docker Hub image digests found" >&2 exit 1 } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 045de4d58..8f1de676b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -217,30 +217,55 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Build and push Docker image - id: push + - name: Build and push Docker image to GHCR + id: push-ghcr uses: docker/build-push-action@v7 with: context: . platforms: ${{ matrix.platform }} outputs: | type=image,name=ghcr.io/${{ steps.image-name.outputs.image }},push-by-digest=true,name-canonical=true,push=true + labels: ${{ steps.meta.outputs.labels }} + build-args: | + OPENVIKING_VERSION=${{ github.event.release.tag_name }} + + - name: Build and push Docker image to Docker Hub + id: push-dockerhub + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + outputs: | type=image,name=docker.io/${{ secrets.DOCKERHUB_USERNAME }}/openviking,push-by-digest=true,name-canonical=true,push=true labels: ${{ steps.meta.outputs.labels }} build-args: | OPENVIKING_VERSION=${{ github.event.release.tag_name }} - - name: Export image digest + - name: Export GHCR image digest run: | - mkdir -p /tmp/digests - digest="${{ steps.push.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" + mkdir -p /tmp/digests-ghcr + ghcr_digest="${{ steps.push-ghcr.outputs.digest }}" + touch "/tmp/digests-ghcr/${ghcr_digest#sha256:}" - - name: Upload image digest + - name: Upload GHCR image digest uses: actions/upload-artifact@v7 with: - name: docker-digests-${{ matrix.arch }} - path: /tmp/digests/* + name: docker-digests-ghcr-${{ matrix.arch }} + path: /tmp/digests-ghcr/* + if-no-files-found: error + retention-days: 1 + + - name: Export Docker Hub image digest + run: | + mkdir -p /tmp/digests-dockerhub + dockerhub_digest="${{ steps.push-dockerhub.outputs.digest }}" + touch "/tmp/digests-dockerhub/${dockerhub_digest#sha256:}" + + - name: Upload Docker Hub image digest + uses: actions/upload-artifact@v7 + with: + name: docker-digests-dockerhub-${{ matrix.arch }} + path: /tmp/digests-dockerhub/* if-no-files-found: error retention-days: 1 @@ -294,11 +319,18 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Download image digests + - name: Download GHCR image digests + uses: actions/download-artifact@v8 + with: + pattern: docker-digests-ghcr-* + path: /tmp/digests-ghcr + merge-multiple: true + + - name: Download Docker Hub image digests uses: actions/download-artifact@v8 with: - pattern: docker-digests-* - path: /tmp/digests + pattern: docker-digests-dockerhub-* + path: /tmp/digests-dockerhub merge-multiple: true - name: Create multi-arch manifests @@ -308,15 +340,23 @@ jobs: # Collect image references for both registries ghcr_image_refs=() dockerhub_image_refs=() - for digest_file in /tmp/digests/*; do + for digest_file in /tmp/digests-ghcr/*; do [ -e "$digest_file" ] || continue digest="sha256:$(basename "$digest_file")" ghcr_image_refs+=("ghcr.io/${{ steps.image-name.outputs.image }}@${digest}") + done + for digest_file in /tmp/digests-dockerhub/*; do + [ -e "$digest_file" ] || continue + digest="sha256:$(basename "$digest_file")" dockerhub_image_refs+=("docker.io/${{ secrets.DOCKERHUB_USERNAME }}/openviking@${digest}") done [ ${#ghcr_image_refs[@]} -gt 0 ] || { - echo "No image digests found" >&2 + echo "No GHCR image digests found" >&2 + exit 1 + } + [ ${#dockerhub_image_refs[@]} -gt 0 ] || { + echo "No Docker Hub image digests found" >&2 exit 1 } diff --git a/Dockerfile b/Dockerfile index 0f683d4e8..f8ff0901f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY --from=rust-toolchain /usr/local/cargo /usr/local/cargo COPY --from=rust-toolchain /usr/local/rustup /usr/local/rustup ENV CARGO_HOME=/usr/local/cargo ENV RUSTUP_HOME=/usr/local/rustup -ENV PATH="/usr/local/cargo/bin:/usr/local/go/bin:${PATH}" +ENV PATH="/app/.venv/bin:/usr/local/cargo/bin:/usr/local/go/bin:${PATH}" ARG OPENVIKING_VERSION=0.0.0 ARG TARGETPLATFORM ARG UV_LOCK_STRATEGY=auto @@ -71,28 +71,37 @@ RUN --mount=type=cache,target=/root/.cache/uv,id=uv-${TARGETPLATFORM} \ RUN --mount=type=cache,target=/root/.cache/uv,id=uv-${TARGETPLATFORM} \ uv pip install maturin && \ export _TMPDIR=$(mktemp -d) && \ + trap 'rm -rf "$_TMPDIR"' EXIT && \ cd crates/ragfs-python && \ - maturin build --release --out "$_TMPDIR" && \ + python -m maturin build --release --out "$_TMPDIR" && \ cd ../.. && \ - export _OV_LIB=$(/app/.venv/bin/python -c "import openviking; from pathlib import Path; print(Path(openviking.__file__).resolve().parent / 'lib')") && \ + export _OV_LIB=$(python -c "import openviking; from pathlib import Path; print(Path(openviking.__file__).resolve().parent / 'lib')") && \ mkdir -p "$_OV_LIB" && \ - /app/.venv/bin/python -c " \ -import zipfile, glob, os, sys; \ -tmpdir, ov_lib = os.environ['_TMPDIR'], os.environ['_OV_LIB']; \ -whls = glob.glob(os.path.join(tmpdir, 'ragfs_python-*.whl')); \ -assert whls, 'maturin produced no wheel'; \ -with zipfile.ZipFile(whls[0]) as zf: \ - for name in zf.namelist(): \ - bn = os.path.basename(name); \ - if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd')): \ - dst = os.path.join(ov_lib, bn); \ - with zf.open(name) as src, open(dst, 'wb') as f: f.write(src.read()); \ - os.chmod(dst, 0o755); \ - print(f'ragfs-python: extracted {bn} -> {dst}'); \ - sys.exit(0); \ -print('WARNING: No ragfs_python .so/.pyd in wheel'); sys.exit(1) \ - " && \ - rm -rf "$_TMPDIR" + python - <<'PY' +import glob +import os +import sys +import zipfile + +tmpdir = os.environ["_TMPDIR"] +ov_lib = os.environ["_OV_LIB"] +whls = glob.glob(os.path.join(tmpdir, "ragfs_python-*.whl")) +assert whls, "maturin produced no wheel" + +with zipfile.ZipFile(whls[0]) as zf: + for name in zf.namelist(): + bn = os.path.basename(name) + if bn.startswith("ragfs_python") and (bn.endswith(".so") or bn.endswith(".pyd")): + dst = os.path.join(ov_lib, bn) + with zf.open(name) as src, open(dst, "wb") as f: + f.write(src.read()) + os.chmod(dst, 0o755) + print(f"ragfs-python: extracted {bn} -> {dst}") + sys.exit(0) + +print("WARNING: No ragfs_python .so/.pyd in wheel") +sys.exit(1) +PY # Stage 4: runtime FROM python:3.13-slim-trixie diff --git a/tests/misc/test_docker_workflow_native_multiarch.py b/tests/misc/test_docker_workflow_native_multiarch.py index e5b390e36..94f5a70f9 100644 --- a/tests/misc/test_docker_workflow_native_multiarch.py +++ b/tests/misc/test_docker_workflow_native_multiarch.py @@ -48,3 +48,33 @@ def test_docker_workflows_normalize_image_names_to_lowercase(): assert "steps.image-name.outputs.image" in build_workflow assert "tr '[:upper:]' '[:lower:]'" in release_workflow assert "steps.image-name.outputs.image" in release_workflow + + +def test_build_docker_workflow_tracks_registry_specific_digests_for_manifests(): + workflow = _read_text(".github/workflows/build-docker-image.yml") + + assert "docker-digests-ghcr-${{ matrix.arch }}" in workflow + assert "docker-digests-dockerhub-${{ matrix.arch }}" in workflow + assert 'ghcr_digest="${{ steps.push-ghcr.outputs.digest }}"' in workflow + assert 'dockerhub_digest="${{ steps.push-dockerhub.outputs.digest }}"' in workflow + assert "pattern: docker-digests-ghcr-*" in workflow + assert "pattern: docker-digests-dockerhub-*" in workflow + assert ( + 'ghcr_image_refs+=("${{ env.REGISTRY }}/${{ steps.image-name.outputs.image }}@${digest}")' + in workflow + ) + assert ( + 'dockerhub_image_refs+=("docker.io/${{ secrets.DOCKERHUB_USERNAME }}/openviking@${digest}")' + in workflow + ) + + +def test_release_workflow_tracks_registry_specific_digests_for_manifests(): + workflow = _read_text(".github/workflows/release.yml") + + assert "docker-digests-ghcr-${{ matrix.arch }}" in workflow + assert "docker-digests-dockerhub-${{ matrix.arch }}" in workflow + assert 'ghcr_digest="${{ steps.push-ghcr.outputs.digest }}"' in workflow + assert 'dockerhub_digest="${{ steps.push-dockerhub.outputs.digest }}"' in workflow + assert "pattern: docker-digests-ghcr-*" in workflow + assert "pattern: docker-digests-dockerhub-*" in workflow