diff --git a/.github/workflows/README.md b/.github/workflows/README.md index ebc0dc8e9..5c23eb80d 100755 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -207,14 +207,19 @@ Calls the reusable `_publish.yml` workflow with: 4. Upserts plugin entry into `catalog.json` and preserves existing entries 5. Signs artifact with cosign keyless OIDC identity 6. Verifies signature in CI -7. Builds plugins catalog site -8. Deploys to Cloudflare Pages (configurable) +7. Uploads immutable artifacts + mutable metadata to Cloudflare R2 (atomic order) +8. Verifies uploaded catalog integrity and public Worker endpoints +9. Optionally builds/deploys legacy plugins catalog site to Cloudflare Pages -**Required secrets/vars for deployment**: +**Required secrets/vars for R2 publishing**: - `CLOUDFLARE_API_TOKEN` - `CLOUDFLARE_ACCOUNT_ID` -- `CLOUDFLARE_PAGES_PROJECT_NAME` (repository variable recommended) +- `CLOUDFLARE_R2_BUCKET_NAME` (repository variable recommended) + +**Optional legacy Pages deployment**: + +- `CLOUDFLARE_PAGES_PROJECT_NAME` (used only when `deploy_cloudflare_pages=true`) --- diff --git a/.github/workflows/publish-plugins.yml b/.github/workflows/publish-plugins.yml index d8d74706e..ca5375c69 100644 --- a/.github/workflows/publish-plugins.yml +++ b/.github/workflows/publish-plugins.yml @@ -27,11 +27,15 @@ on: description: "Deploy plugins catalog app to Cloudflare Pages" required: false type: boolean - default: true + default: false cloudflare_project_name: description: "Cloudflare Pages project name" required: false default: "" + cloudflare_r2_bucket_name: + description: "Cloudflare R2 bucket for plugin distribution assets" + required: false + default: "" permissions: contents: read @@ -54,6 +58,8 @@ jobs: INPUT_DEPLOY_CF: ${{ inputs.deploy_cloudflare_pages }} INPUT_CF_PROJECT_NAME: ${{ inputs.cloudflare_project_name }} DEFAULT_CF_PROJECT_NAME: ${{ vars.CLOUDFLARE_PAGES_PROJECT_NAME }} + INPUT_CF_R2_BUCKET_NAME: ${{ inputs.cloudflare_r2_bucket_name }} + DEFAULT_CF_R2_BUCKET_NAME: ${{ vars.CLOUDFLARE_R2_BUCKET_NAME }} steps: - name: ✈ Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -88,16 +94,25 @@ jobs: .rstrip("/") ) input_oci_repository = os.environ.get("INPUT_OCI_REPOSITORY", "").strip() - input_deploy_cf = os.environ.get("INPUT_DEPLOY_CF", "true").strip().lower() + input_deploy_cf = os.environ.get("INPUT_DEPLOY_CF", "false").strip().lower() input_cf_project_name = os.environ.get("INPUT_CF_PROJECT_NAME", "").strip() default_cf_project_name = os.environ.get("DEFAULT_CF_PROJECT_NAME", "").strip() effective_cf_project_name = input_cf_project_name or default_cf_project_name + input_cf_r2_bucket_name = os.environ.get("INPUT_CF_R2_BUCKET_NAME", "").strip() + default_cf_r2_bucket_name = os.environ.get("DEFAULT_CF_R2_BUCKET_NAME", "").strip() + effective_cf_r2_bucket_name = input_cf_r2_bucket_name or default_cf_r2_bucket_name if not input_catalog_base_url: input_catalog_base_url = "https://plugins.corvus.profiletailors.com" - if not input_deploy_cf: - input_deploy_cf = "true" + if input_deploy_cf not in {"true", "false"}: + input_deploy_cf = "false" + + if not effective_cf_r2_bucket_name: + raise SystemExit( + "Missing Cloudflare R2 bucket name. Set workflow input cloudflare_r2_bucket_name " + "or repository variable CLOUDFLARE_R2_BUCKET_NAME.", + ) if not input_catalog_base_url.startswith("https://"): raise SystemExit("catalog_base_url must start with https://") @@ -215,6 +230,7 @@ jobs: fh.write(f"oci_repository={input_oci_repository}\n") fh.write(f"deploy_cloudflare_pages={input_deploy_cf}\n") fh.write(f"cloudflare_project_name={effective_cf_project_name}\n") + fh.write(f"cloudflare_r2_bucket_name={effective_cf_r2_bucket_name}\n") fh.write(f"artifact_relative_path={artifact_relative_path}\n") fh.write(f"signature_relative_path={signature_relative_path}\n") fh.write(f"certificate_relative_path={certificate_relative_path}\n") @@ -264,6 +280,8 @@ jobs: env: PLUGIN_ID: ${{ steps.meta.outputs.plugin_id }} PLUGIN_VERSION: ${{ steps.meta.outputs.plugin_version }} + CATALOG_BASE_URL: ${{ steps.meta.outputs.catalog_base_url }} + ALLOW_LOCAL_CATALOG_FALLBACK: ${{ vars.ALLOW_LOCAL_CATALOG_FALLBACK }} PLUGIN_DIR: ${{ steps.meta.outputs.plugin_dir }} WASM_BINARY_NAME: ${{ steps.meta.outputs.wasm_binary_name }} RUNTIME_VERSION: ${{ steps.meta.outputs.runtime_version }} @@ -291,6 +309,8 @@ jobs: import os import pathlib import shutil + import urllib.error + import urllib.request dist_dir = pathlib.Path(os.environ["DIST_DIR"]) dist_dir.mkdir(parents=True, exist_ok=True) @@ -350,10 +370,49 @@ jobs: ) source_catalog = pathlib.Path("clients/web/apps/plugins/public/catalog.json") - if source_catalog.exists(): - catalog_payload = json.loads(source_catalog.read_text(encoding="utf-8")) + catalog_payload = None + fallback_allowed = False + + catalog_base_url = os.environ.get("CATALOG_BASE_URL", "").strip().rstrip("/") + allow_local_fallback = os.environ.get("ALLOW_LOCAL_CATALOG_FALLBACK", "").strip().lower() in { + "1", + "true", + "yes", + } + + if catalog_base_url: + remote_catalog_url = f"{catalog_base_url}/catalog.json" + try: + with urllib.request.urlopen(remote_catalog_url, timeout=10) as response: + catalog_payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + if error.code == 404 or allow_local_fallback: + fallback_allowed = True + else: + raise SystemExit( + f"failed to fetch remote catalog {remote_catalog_url}: HTTP {error.code}", + ) + except (urllib.error.URLError, json.JSONDecodeError, UnicodeDecodeError) as error: + if allow_local_fallback: + fallback_allowed = True + else: + raise SystemExit( + f"failed to fetch remote catalog {remote_catalog_url}: {error}", + ) else: - catalog_payload = {"plugins": []} + fallback_allowed = True + + if catalog_payload is None and fallback_allowed: + if source_catalog.exists(): + try: + catalog_payload = json.loads(source_catalog.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise SystemExit(f"local catalog.json malformed: {error}") + else: + catalog_payload = {"plugins": []} + + if catalog_payload is None: + raise SystemExit("catalog source unavailable and local fallback is not allowed") existing_plugins = catalog_payload.get("plugins", []) if not isinstance(existing_plugins, list): @@ -512,7 +571,170 @@ jobs: cache: "pnpm" cache-dependency-path: clients/web/pnpm-lock.yaml + - name: 🧰 Install Wrangler CLI + run: | + set -euo pipefail + npm install --global wrangler@4.65.0 + wrangler --version + + - name: ⬆ Upload plugin bundle and metadata to Cloudflare R2 + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_R2_BUCKET: ${{ steps.meta.outputs.cloudflare_r2_bucket_name }} + DIST_DIR: ${{ env.DIST_DIR }} + ARTIFACT_RELATIVE_PATH: ${{ steps.meta.outputs.artifact_relative_path }} + SIGNATURE_RELATIVE_PATH: ${{ steps.meta.outputs.signature_relative_path }} + CERTIFICATE_RELATIVE_PATH: ${{ steps.meta.outputs.certificate_relative_path }} + MANIFEST_RELATIVE_PATH: ${{ steps.meta.outputs.manifest_relative_path }} + run: | + set -euo pipefail + + if [ -z "${CLOUDFLARE_API_TOKEN:-}" ] || [ -z "${CLOUDFLARE_ACCOUNT_ID:-}" ]; then + echo "Missing required Cloudflare secrets: CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID" + exit 1 + fi + + if [ -z "${CLOUDFLARE_R2_BUCKET:-}" ]; then + echo "Missing Cloudflare R2 bucket name" + exit 1 + fi + + wrangler_put() { + local key="$1" + local file="$2" + local content_type="$3" + test -f "$file" || { echo "missing file: $file" >&2; exit 1; } + wrangler r2 object put "${CLOUDFLARE_R2_BUCKET}/${key}" \ + --file "$file" \ + --content-type "$content_type" + } + + # Upload immutable artifacts first + wrangler_put "$ARTIFACT_RELATIVE_PATH" "$DIST_DIR/$ARTIFACT_RELATIVE_PATH" "application/wasm" + wrangler_put "$SIGNATURE_RELATIVE_PATH" "$DIST_DIR/$SIGNATURE_RELATIVE_PATH" "application/octet-stream" + wrangler_put "$CERTIFICATE_RELATIVE_PATH" "$DIST_DIR/$CERTIFICATE_RELATIVE_PATH" "application/x-pem-file" + wrangler_put "$MANIFEST_RELATIVE_PATH" "$DIST_DIR/$MANIFEST_RELATIVE_PATH" "application/json" + + # Upload mutable metadata last for an atomic catalog view + wrangler_put "catalog/catalog.json" "$DIST_DIR/catalog.json" "application/json" + wrangler_put "catalog/revocations.json" "$DIST_DIR/revocations.json" "application/json" + + - name: ✅ Verify R2 objects and catalog integrity + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_R2_BUCKET: ${{ steps.meta.outputs.cloudflare_r2_bucket_name }} + PLUGIN_ID: ${{ steps.meta.outputs.plugin_id }} + PLUGIN_VERSION: ${{ steps.meta.outputs.plugin_version }} + ARTIFACT_URL: ${{ steps.meta.outputs.artifact_url }} + SIGNATURE_URL: ${{ steps.meta.outputs.signature_url }} + CERTIFICATE_URL: ${{ steps.meta.outputs.certificate_url }} + shell: bash + run: | + set -euo pipefail + + tmp_dir="$(mktemp -d)" + trap 'rm -rf "$tmp_dir"' EXIT + catalog_path="$tmp_dir/catalog.json" + revocations_path="$tmp_dir/revocations.json" + + wrangler r2 object get "${CLOUDFLARE_R2_BUCKET}/catalog/catalog.json" --file "$catalog_path" + wrangler r2 object get "${CLOUDFLARE_R2_BUCKET}/catalog/revocations.json" --file "$revocations_path" + + export TMP_CATALOG_PATH="$catalog_path" + export TMP_REVOCATIONS_PATH="$revocations_path" + + python - <<'PY' + import json + import os + import pathlib + + plugin_id = os.environ["PLUGIN_ID"] + plugin_version = os.environ["PLUGIN_VERSION"] + artifact_url = os.environ["ARTIFACT_URL"] + signature_url = os.environ["SIGNATURE_URL"] + certificate_url = os.environ["CERTIFICATE_URL"] + catalog = json.loads(pathlib.Path(os.environ["TMP_CATALOG_PATH"]).read_text(encoding="utf-8")) + revocations = json.loads(pathlib.Path(os.environ["TMP_REVOCATIONS_PATH"]).read_text(encoding="utf-8")) + + plugins = catalog.get("plugins", []) + if not isinstance(plugins, list): + raise SystemExit("catalog/plugins must be a list") + + entry = None + for item in plugins: + if item.get("id") == plugin_id: + entry = item + break + + if entry is None: + raise SystemExit(f"catalog missing plugin entry for {plugin_id}") + if entry.get("version") != plugin_version: + raise SystemExit( + f"catalog version mismatch: expected {plugin_version}, found {entry.get('version')}", + ) + + artifact = entry.get("artifact", {}) + if artifact.get("url") != artifact_url: + raise SystemExit("catalog artifact.url mismatch") + + signature = entry.get("signature", {}) + if signature.get("url") != signature_url: + raise SystemExit("catalog signature.url mismatch") + if signature.get("certificate_url") != certificate_url: + raise SystemExit("catalog signature.certificate_url mismatch") + + if not isinstance(revocations, dict): + raise SystemExit("revocations payload must be a JSON object") + if "revoked" not in revocations: + raise SystemExit("revocations payload missing required key: revoked") + if not isinstance(revocations.get("revoked"), list): + raise SystemExit("revocations/revoked must be a list") + if "updated_at" not in revocations: + raise SystemExit("revocations payload missing required key: updated_at") + + updated_at = revocations.get("updated_at") + is_valid_string_timestamp = isinstance(updated_at, str) and bool(updated_at.strip()) + is_valid_numeric_timestamp = isinstance(updated_at, (int, float)) + if not (is_valid_string_timestamp or is_valid_numeric_timestamp): + raise SystemExit("revocations/updated_at must be a valid timestamp") + + print("Catalog entry integrity check passed") + PY + + - name: 🌐 Smoke check public worker endpoints + env: + CATALOG_URL: ${{ steps.meta.outputs.catalog_base_url }}/catalog.json + REVOCATIONS_URL: ${{ steps.meta.outputs.catalog_base_url }}/revocations.json + ARTIFACT_URL: ${{ steps.meta.outputs.artifact_url }} + SIGNATURE_URL: ${{ steps.meta.outputs.signature_url }} + CERTIFICATE_URL: ${{ steps.meta.outputs.certificate_url }} + run: | + set -euo pipefail + + check_url() { + local url="$1" + local attempts=0 + until [ "$attempts" -ge 5 ]; do + if curl -fsSIL "$url" >/dev/null 2>&1; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 2 + done + echo "Public URL check failed: $url" >&2 + return 1 + } + + check_url "$CATALOG_URL" + check_url "$REVOCATIONS_URL" + check_url "$ARTIFACT_URL" + check_url "$SIGNATURE_URL" + check_url "$CERTIFICATE_URL" + - name: 🧩 Stage generated catalog assets + if: ${{ steps.meta.outputs.deploy_cloudflare_pages == 'true' }} env: DIST_DIR: ${{ env.DIST_DIR }} ARTIFACT_RELATIVE_PATH: ${{ steps.meta.outputs.artifact_relative_path }} @@ -534,10 +756,12 @@ jobs: cp "$DIST_DIR/revocations.json" "$catalog_public/revocations.json" - name: 📥 Install web dependencies + if: ${{ steps.meta.outputs.deploy_cloudflare_pages == 'true' }} working-directory: clients/web run: pnpm install --frozen-lockfile --filter @corvus/plugins-catalog... - name: 🏗️ Build plugins catalog site + if: ${{ steps.meta.outputs.deploy_cloudflare_pages == 'true' }} working-directory: clients/web env: PLUGINS_URL: ${{ steps.meta.outputs.catalog_base_url }} @@ -565,7 +789,7 @@ jobs: exit 1 fi - pnpm dlx wrangler@4.44.0 pages deploy ./dist \ + wrangler pages deploy ./dist \ --project-name "$CLOUDFLARE_PROJECT_NAME" \ --branch main \ --commit-hash "$GITHUB_SHA" \ @@ -586,6 +810,7 @@ jobs: if-no-files-found: error - name: ⬆ Upload plugins catalog site artifact + if: ${{ steps.meta.outputs.deploy_cloudflare_pages == 'true' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: plugins-catalog-site-${{ steps.meta.outputs.plugin_id }}-${{ steps.meta.outputs.plugin_version }} diff --git a/clients/web/apps/docs/src/content/docs/en/guides/plugins.md b/clients/web/apps/docs/src/content/docs/en/guides/plugins.md index 913e49f78..aff75c6a8 100644 --- a/clients/web/apps/docs/src/content/docs/en/guides/plugins.md +++ b/clients/web/apps/docs/src/content/docs/en/guides/plugins.md @@ -126,8 +126,11 @@ Current workflow behavior: 5. Sign artifact with cosign keyless OIDC identity. 6. Verify signature in CI. 7. Optionally push artifact bundle to OCI (`oci_repository`). -8. Build plugins catalog app and deploy to Cloudflare Pages (enabled by default for release tags). -9. Upload build + bundle artifacts as workflow artifacts for traceability. +8. Upload immutable artifacts and mutable metadata to Cloudflare R2 in atomic order + (artifacts first, `catalog.json`/`revocations.json` last). +9. Verify catalog integrity from R2 and smoke-check public Worker endpoints. +10. Optionally build/deploy legacy Cloudflare Pages catalog UI (`deploy_cloudflare_pages=true`). +11. Upload build + bundle artifacts as workflow artifacts for traceability. :::important To onboard a new plugin into automated releases: @@ -147,11 +150,13 @@ git tag plugin/memory.surreal.graphs/v0.1.0 git push origin plugin/memory.surreal.graphs/v0.1.0 ``` -Cloudflare deployment configuration expected by the workflow: +Cloudflare configuration expected by the workflow: - Secret: `CLOUDFLARE_API_TOKEN` - Secret: `CLOUDFLARE_ACCOUNT_ID` -- Repository variable: `CLOUDFLARE_PAGES_PROJECT_NAME` +- Repository variable: `CLOUDFLARE_R2_BUCKET_NAME` +- Optional repository variable (legacy Pages deployment only): + `CLOUDFLARE_PAGES_PROJECT_NAME` ## 6. Operator Commands (Runtime) diff --git a/clients/web/apps/docs/src/content/docs/es/guides/plugins.md b/clients/web/apps/docs/src/content/docs/es/guides/plugins.md index 2b5f65438..ceef96ccf 100644 --- a/clients/web/apps/docs/src/content/docs/es/guides/plugins.md +++ b/clients/web/apps/docs/src/content/docs/es/guides/plugins.md @@ -127,9 +127,14 @@ Comportamiento actual del workflow: 5. Firma el artefacto con identidad OIDC keyless de cosign. 6. Verifica la firma en CI. 7. Push opcional del bundle a OCI (`oci_repository`). -8. Hace build de la app de catálogo y deploy a Cloudflare Pages (habilitado por defecto para tags de - release). -9. Sube artefactos del build y del bundle para trazabilidad. +8. Sube artefactos inmutables y metadatos mutables a Cloudflare R2 en orden atómico + (artefactos primero, `catalog.json`/`revocations.json` al final). + +9. Verifica integridad del catálogo en R2 y smoke-check de endpoints públicos del Worker. + +10. Opcionalmente hace build/deploy del catálogo legacy en Cloudflare Pages (`deploy_cloudflare_pages=true`). + +11. Sube artefactos del build y del bundle para trazabilidad. :::important Para integrar un nuevo plugin al release automático: @@ -153,7 +158,8 @@ Configuración de Cloudflare esperada por el workflow: - Secret: `CLOUDFLARE_API_TOKEN` - Secret: `CLOUDFLARE_ACCOUNT_ID` -- Variable de repositorio: `CLOUDFLARE_PAGES_PROJECT_NAME` +- Variable de repositorio: `CLOUDFLARE_R2_BUCKET_NAME` +- Variable opcional de repositorio (solo deploy legacy de Pages): `CLOUDFLARE_PAGES_PROJECT_NAME` ## 6. Comandos de Operador (Runtime)