Entlein/adaptive write perf

.github/workflows/perf_clickhouse.yaml

@@ -0,0 +1,158 @@
+---
+name: perf-eval-clickhouse
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Branch or commit'
+        required: false
+        type: string
+      tags:
+        description: 'Tags (comma separated)'
+        required: false
+        type: string
+permissions:
+  contents: read
+  packages: write
+jobs:
+  get-dev-image-with-extras:
+    uses: ./.github/workflows/get_image.yaml
+    with:
+      image-base-name: "dev_image_with_extras"
+      ref: ${{ inputs.ref }}
+
+  clickhouse-export-perf:
+    name: ClickHouse export perf eval
+    needs: get-dev-image-with-extras
+    runs-on: oracle-vm-16cpu-64gb-x86-64
+    container:
+      image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
+      options: --cap-add=NET_ADMIN --device=/dev/net/tun
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      with:
+        ref: ${{ inputs.ref }}
+        fetch-depth: 0
+    - name: Add pwd to git safe dir
+      run: git config --global --add safe.directory `pwd`
+    - id: get-commit-sha
+      run: echo "commit-sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+    # TODO(ddelnano): swap TAILSCALE_AUTH_KEY for an OAuth client once one is
+    # provisioned in the k8sstormcenter tailnet. Use
+    # `tailscale/github-action@v2` with `oauth-client-id` and `oauth-secret`
+    # inputs (`TS_OAUTH_CLIENT_ID` / `TS_OAUTH_CLIENT_SECRET` secrets) so
+    # credentials rotate automatically instead of expiring on a fixed cadence.
+    - name: Start Tailscale sidecar
+      env:
+        TS_AUTHKEY: ${{ secrets.TAILSCALE_AUTH_KEY }}
+      run: |
+        curl -fsSL https://tailscale.com/install.sh | sh
+        mkdir -p /var/run/tailscale /var/lib/tailscale
+        tailscaled \
+          --socket=/var/run/tailscale/tailscaled.sock \
+          --state=/var/lib/tailscale/tailscaled.state &
+        until tailscale status --json >/dev/null 2>&1; do sleep 1; done
+        tailscale up \
+          --authkey="${TS_AUTHKEY}" \
+          --accept-routes \
+          --hostname="pixie-perf-ci-${GITHUB_RUN_ID}"
+
+    - name: Write kubeconfig
+      env:
+        KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
+      run: |
+        mkdir -p "${RUNNER_TEMP}"
+        echo "${KUBECONFIG_B64}" | base64 -d > "${RUNNER_TEMP}/kubeconfig"
+        chmod 600 "${RUNNER_TEMP}/kubeconfig"
+
+    # Fail fast if Tailscale can't reach the cluster API, before the 2+ minute
+    # bazel/skaffold build wastes time.
+    - name: Tailscale connectivity probe
+      env:
+        KUBECONFIG: ${{ runner.temp }}/kubeconfig
+      run: |
+        tailscale status
+        tailscale netcheck
+        api_host="$(kubectl --kubeconfig="$KUBECONFIG" config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed -E 's|https?://||; s|/.*||')"
+        api_ip="${api_host%%:*}"
+        api_port="${api_host##*:}"
+        echo "--- tailscale ping ${api_ip} ---"
+        tailscale ping --c 3 --until-direct=false "${api_ip}" || true
+        echo "--- tcp probe ${api_ip}:${api_port} ---"
+        timeout 5 bash -c "</dev/tcp/${api_ip}/${api_port}" \
+          && echo "API port reachable" \
+          || { echo "API port UNREACHABLE"; exit 1; }
+        echo "--- kubectl get nodes ---"
+        kubectl --kubeconfig="$KUBECONFIG" get nodes
+
+    - name: Use github bazel config
+      uses: ./.github/actions/bazelrc
+      with:
+        download_toplevel: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
+
+    # TODO(ddelnano): revert to `./.github/actions/gcloud_creds` once GCP_SA_KEY
+    # is re-uploaded with `base64 -w0`. The shared composite uses plain
+    # `base64 --decode` which rejects the wrapped (multi-line/CRLF) value
+    # currently stored in the secret.
+    - id: gcloud-creds
+      env:
+        SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SA_KEY }}
+      run: |
+        printf '%s' "$SERVICE_ACCOUNT_KEY" | base64 -di > /tmp/gcloud.json
+        chmod 600 /tmp/gcloud.json
+        echo "gcloud-creds=/tmp/gcloud.json" >> $GITHUB_OUTPUT
+    - name: Activate gcloud service account
+      env:
+        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
+      run: |
+        service_account="$(jq -r '.client_email' "$GOOGLE_APPLICATION_CREDENTIALS")"
+        gcloud auth activate-service-account "${service_account}" --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
+        gcloud auth configure-docker
+
+    - name: Log in to GHCR
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: echo "${GH_TOKEN}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+    - name: Build and install px CLI
+      run: |
+        bazel build --config=x86_64_sysroot //src/pixie_cli:px
+        install -m 0755 bazel-bin/src/pixie_cli/px_/px /usr/local/bin/px
+        px version
+
+    - name: Run clickhouse-export perf
+      env:
+        PX_API_KEY: ${{ secrets.PX_API_KEY }}
+        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
+        KUBECONFIG: ${{ runner.temp }}/kubeconfig
+      run: |
+        bazel run //src/e2e_test/perf_tool:perf_tool -- run \
+          --api_key="${PX_API_KEY}" \
+          --cloud_addr=pixie.austrianopencloudcommunity.org:443 \
+          --commit_sha="${{ steps.get-commit-sha.outputs.commit-sha }}" \
+          --experiment_name=clickhouse-export \
+          --suite=clickhouse-exec \
+          --use_local_cluster \
+          --export_backend=parquet-gcs \
+          --gcs_bucket=k8sstormcenter-soc-perf \
+          --container_repo=ghcr.io/k8sstormcenter \
+          --prom_recorder_override 'clickhouse-operator=:k8ss-forensic' \
+          --tags "${{ inputs.tags }}"
+
+    - name: Upload skaffold stderr log
+      if: always()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+      with:
+        name: skaffold-stderr-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ runner.temp }}/skaffold-stderr.log
+        if-no-files-found: ignore
+
+    - name: Deactivate gcloud service account
+      if: always()
+      run: gcloud auth revoke || true
+
+    - name: Tailscale logout
+      if: always()
+      run: tailscale logout || true

.github/workflows/perf_soc_attack.yaml

@@ -0,0 +1,159 @@
+---
+name: perf-eval-soc-attack
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Branch or commit'
+        required: false
+        type: string
+      tags:
+        description: 'Tags (comma separated)'
+        required: false
+        type: string
+permissions:
+  contents: read
+  packages: write
+jobs:
+  get-dev-image-with-extras:
+    uses: ./.github/workflows/get_image.yaml
+    with:
+      image-base-name: "dev_image_with_extras"
+      ref: ${{ inputs.ref }}
+
+  soc-attack-perf:
+    name: Sovereign SOC redis-attack perf eval
+    needs: get-dev-image-with-extras
+    runs-on: oracle-vm-16cpu-64gb-x86-64
+    container:
+      image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
+      options: --cap-add=NET_ADMIN --device=/dev/net/tun
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      with:
+        ref: ${{ inputs.ref }}
+        fetch-depth: 0
+    - name: Add pwd to git safe dir
+      run: git config --global --add safe.directory `pwd`
+    - id: get-commit-sha
+      run: echo "commit-sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+    # TODO(ddelnano): swap TAILSCALE_AUTH_KEY for an OAuth client once one is
+    # provisioned in the k8sstormcenter tailnet. Use
+    # `tailscale/github-action@v2` with `oauth-client-id` and `oauth-secret`
+    # inputs (`TS_OAUTH_CLIENT_ID` / `TS_OAUTH_CLIENT_SECRET` secrets) so
+    # credentials rotate automatically instead of expiring on a fixed cadence.
+    - name: Start Tailscale sidecar
+      env:
+        TS_AUTHKEY: ${{ secrets.TAILSCALE_AUTH_KEY }}
+      run: |
+        curl -fsSL https://tailscale.com/install.sh | sh
+        mkdir -p /var/run/tailscale /var/lib/tailscale
+        tailscaled \
+          --socket=/var/run/tailscale/tailscaled.sock \
+          --state=/var/lib/tailscale/tailscaled.state &
+        until tailscale status --json >/dev/null 2>&1; do sleep 1; done
+        tailscale up \
+          --authkey="${TS_AUTHKEY}" \
+          --accept-routes \
+          --hostname="pixie-perf-ci-${GITHUB_RUN_ID}"
+
+    - name: Write kubeconfig
+      env:
+        KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
+      run: |
+        mkdir -p "${RUNNER_TEMP}"
+        echo "${KUBECONFIG_B64}" | base64 -d > "${RUNNER_TEMP}/kubeconfig"
+        chmod 600 "${RUNNER_TEMP}/kubeconfig"
+
+    # Fail fast if Tailscale can't reach the cluster API, before the 2+ minute
+    # bazel/skaffold build wastes time.
+    - name: Tailscale connectivity probe
+      env:
+        KUBECONFIG: ${{ runner.temp }}/kubeconfig
+      run: |
+        tailscale status
+        tailscale netcheck
+        api_host="$(kubectl --kubeconfig="$KUBECONFIG" config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed -E 's|https?://||; s|/.*||')"
+        api_ip="${api_host%%:*}"
+        api_port="${api_host##*:}"
+        echo "--- tailscale ping ${api_ip} ---"
+        tailscale ping --c 3 --until-direct=false "${api_ip}" || true
+        echo "--- tcp probe ${api_ip}:${api_port} ---"
+        timeout 5 bash -c "</dev/tcp/${api_ip}/${api_port}" \
+          && echo "API port reachable" \
+          || { echo "API port UNREACHABLE"; exit 1; }
+        echo "--- kubectl get nodes ---"
+        kubectl --kubeconfig="$KUBECONFIG" get nodes
+
+    - name: Use github bazel config
+      uses: ./.github/actions/bazelrc
+      with:
+        download_toplevel: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
+
+    # TODO(ddelnano): revert to `./.github/actions/gcloud_creds` once GCP_SA_KEY
+    # is re-uploaded with `base64 -w0`. The shared composite uses plain
+    # `base64 --decode` which rejects the wrapped (multi-line/CRLF) value
+    # currently stored in the secret.
+    - id: gcloud-creds
+      env:
+        SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SA_KEY }}
+      run: |
+        printf '%s' "$SERVICE_ACCOUNT_KEY" | base64 -di > /tmp/gcloud.json
+        chmod 600 /tmp/gcloud.json
+        echo "gcloud-creds=/tmp/gcloud.json" >> $GITHUB_OUTPUT
+    - name: Activate gcloud service account
+      env:
+        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
+      run: |
+        service_account="$(jq -r '.client_email' "$GOOGLE_APPLICATION_CREDENTIALS")"
+        gcloud auth activate-service-account "${service_account}" --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
+        gcloud auth configure-docker
+
+    - name: Log in to GHCR
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: echo "${GH_TOKEN}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+    - name: Build and install px CLI
+      run: |
+        bazel build --config=x86_64_sysroot //src/pixie_cli:px
+        install -m 0755 bazel-bin/src/pixie_cli/px_/px /usr/local/bin/px
+        px version
+
+    # The sovereign-soc suite installs Kubescape + Vector on the experiment
+    # cluster as part of the run (see KubescapeVectorWorkload). The
+    # kubescape-operator chart is pre-rendered under
+    # src/e2e_test/perf_tool/pkg/suites/k8s/sovereign-soc/helm-rendered/
+    # and applied via PrerenderedDeploy, so no extra ./scripts step is needed.
+    #
+    # ClickHouse operator metrics are scraped on the forensic cluster via
+    # the prom_recorder_override; the kubescape node-agent prom recorder
+    # is intentionally NOT overridden — kubescape runs on the experiment
+    # cluster (where redis+bobctl drive traffic), so the recorder uses the
+    # default kubeconfig.
+    - name: Run sovereign-soc redis-attack perf
+      env:
+        PX_API_KEY: ${{ secrets.PX_API_KEY }}
+        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
+        KUBECONFIG: ${{ runner.temp }}/kubeconfig
+        SOC_VIZIER_EXISTING: "1"
+      run: |
+        bazel run //src/e2e_test/perf_tool:perf_tool -- run \
+          --api_key="${PX_API_KEY}" \
+          --cloud_addr=pixie.austrianopencloudcommunity.org:443 \
+          --commit_sha="${{ steps.get-commit-sha.outputs.commit-sha }}" \
+          --experiment_name=redis-attack \
+          --suite=sovereign-soc \
+          --use_local_cluster \
+          --export_backend=parquet-gcs \
+          --gcs_bucket=k8sstormcenter-soc-perf \
+          --container_repo=ghcr.io/k8sstormcenter \
+          --prom_recorder_override 'clickhouse-operator=:k8ss-forensic' \
+          --max_retries=1 \
+          --tags "${{ inputs.tags }}"
+
+    - name: Tailscale logout
+      if: always()
+      run: tailscale logout || true

.github/workflows/trivy_fs.yaml

@@ -23,7 +23,9 @@ jobs:
       security-events: write
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-    - uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0  # v0.29.0
+    # v0.36.0 released 2026-04-22 (post-incident). Internally SHA-pins
+    # setup-trivy@3fb12ec = Aqua's safe v0.2.6 per GHSA-69fq-xp46-6x23.
+    - uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25  # v0.36.0
       with:
         scan-type: 'fs'
         ignore-unfixed: true

chained-sweep.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# chained-sweep.sh — wait for an in-flight perf-sweep to finish, then kick
+# off a second (independent) sweep into a fresh /tmp/perf-sweep-<ts>/ dir
+# with its own watcher. Use this when you want a clean before/after pair
+# without having to be at the keyboard when the first one ends.
+#
+# Usage:
+#   ./chained-sweep.sh <first-sweep-dir>
+#   ./chained-sweep.sh /tmp/perf-sweep-20260514-114224
+set -euo pipefail
+
+FIRST="${1:?need path to first sweep dir}"
+LOG=/tmp/chained-sweep.log
+exec > >(tee -a "$LOG") 2>&1
+
+echo "$(date -Is) waiting for first sweep to finish: $FIRST"
+# perf-sweep.sh writes "sweep complete in N s — <dir>" as the last line
+# of sweep.log when all multipliers landed.
+while ! grep -q "sweep complete" "$FIRST/sweep.log" 2>/dev/null; do
+  sleep 30
+done
+echo "$(date -Is) first sweep finished"
+
+# Kick off second sweep (perf-sweep.sh creates its own timestamped dir).
+# Tag the sweep.log with a header so it's obvious in the watcher output
+# that this is the "after" run.
+echo "$(date -Is) launching second sweep"
+/home/constanze/code/pixie/perf-sweep.sh > /tmp/perf-sweep-second.stdout 2>&1 &
+SWEEP_PID=$!
+
+# Give perf-sweep.sh a moment to create its dir + sweep.log.
+sleep 8
+NEW=$(ls -dt /tmp/perf-sweep-2*/ 2>/dev/null | head -1)
+NEW="${NEW%/}"
+if [[ -z "$NEW" || "$NEW" == "$FIRST" ]]; then
+  echo "$(date -Is) ERROR: second sweep dir not detected"
+  exit 1
+fi
+echo "$(date -Is) second sweep dir: $NEW"
+
+# Watcher for the new sweep (auto-exits when its sweep.log shows complete).
+setsid bash /home/constanze/code/pixie/render-sweep-watch.sh "$NEW" \
+  </dev/null > /tmp/render-watch-second.log 2>&1 &
+disown
+echo "$(date -Is) watcher launched for $NEW"
+
+wait "$SWEEP_PID"
+echo "$(date -Is) second sweep done"

docker.properties

@@ -1,4 +1,4 @@
-DOCKER_IMAGE_TAG=202512082352
-LINTER_IMAGE_DIGEST=441fc5a65697dab0b38627d5afde9e38da6812f1a5b98732b224161c23238e73
-DEV_IMAGE_DIGEST=cac2e8a1c3e70dde4e5089b2383b2e11cc022af467ee430c12416eb42066fbb7
-DEV_IMAGE_WITH_EXTRAS_DIGEST=e84f82d62540e1ca72650f8f7c9c4fe0b32b64a33f04cf0b913b9961527c9e30
+DOCKER_IMAGE_TAG=202604270358
+LINTER_IMAGE_DIGEST=af984e837756bce44089d0f977146aee989b24a12884ba2366b4e6eaf19d9acb
+DEV_IMAGE_DIGEST=e4aec14294cff907e7dc3c4835950a4e166e503d32cae082418971e7f70d86bc
+DEV_IMAGE_WITH_EXTRAS_DIGEST=331a2391941c589d2b6536ae49794460b1097c482a45a11029d96a7d0d8d8030