From e0923537bf987ae9bc6edb0d8efa6cba6ed27ee2 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 29 Apr 2026 15:46:07 +0200 Subject: [PATCH 1/7] feat: add CLI compatibility manifest for template version pinning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add cli-compat.json that maps CLI versions to compatible AppKit template and Agent Skills versions. This enables decoupling template delivery from CLI release cycles — older CLI versions automatically receive a compatible template version. Also add tools/check-cli-compat.ts validation script and CI step to prevent malformed manifests from being merged. Signed-off-by: Pawel Kosiec --- .github/workflows/ci.yml | 2 ++ cli-compat.json | 7 ++++ tools/check-cli-compat.ts | 74 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 cli-compat.json create mode 100644 tools/check-cli-compat.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cd0af7d..273d9782 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,8 @@ jobs: run: pnpm run check:licenses - name: Check template deps are pinned run: pnpm exec tsx tools/check-template-deps.ts + - name: Check CLI compat manifest + run: pnpm exec tsx tools/check-cli-compat.ts test: name: Unit Tests diff --git a/cli-compat.json b/cli-compat.json new file mode 100644 index 00000000..fd13c5f9 --- /dev/null +++ b/cli-compat.json @@ -0,0 +1,7 @@ +{ + "next": { "appkit": "0.27.0", "skills": "0.1.5" }, + "0.296.0": { "appkit": "0.27.0", "skills": "0.1.5" }, + "0.295.0": { "appkit": "0.27.0", "skills": "0.1.5" }, + "0.290.0": { "appkit": "0.24.0", "skills": "0.1.5" }, + "0.288.0": { "appkit": "0.24.0", "skills": "0.1.4" } +} diff --git a/tools/check-cli-compat.ts b/tools/check-cli-compat.ts new file mode 100644 index 00000000..f18c1981 --- /dev/null +++ b/tools/check-cli-compat.ts @@ -0,0 +1,74 @@ +#!/usr/bin/env tsx +/** + * Validates that cli-compat.json is well-formed: + * - Valid JSON + * - "next" key is required + * - All keys are "next" or valid semver (X.Y.Z) + * - All values have "appkit" and "skills" fields with valid semver strings + */ + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const SEMVER = /^\d+\.\d+\.\d+(-[\w.]+)?$/; + +const raw = readFileSync( + join(import.meta.dirname, "../cli-compat.json"), + "utf-8", +); + +let manifest: Record; +try { + manifest = JSON.parse(raw); +} catch { + console.error("cli-compat.json is not valid JSON"); + process.exit(1); +} + +if (typeof manifest !== "object" || manifest === null || Array.isArray(manifest)) { + console.error("cli-compat.json must be a JSON object"); + process.exit(1); +} + +if (!("next" in manifest)) { + console.error('cli-compat.json must contain a "next" key'); + process.exit(1); +} + +const errors: string[] = []; + +for (const [key, value] of Object.entries(manifest)) { + // Validate key + if (key !== "next" && !SEMVER.test(key)) { + errors.push(`Invalid key "${key}": must be "next" or a semver string (X.Y.Z)`); + continue; + } + + // Validate value shape + if (typeof value !== "object" || value === null || Array.isArray(value)) { + errors.push(`Value for "${key}" must be an object`); + continue; + } + + const entry = value as Record; + + if (typeof entry.appkit !== "string" || !SEMVER.test(entry.appkit)) { + errors.push( + `"${key}.appkit" must be a valid semver string, got: ${JSON.stringify(entry.appkit)}`, + ); + } + + if (typeof entry.skills !== "string" || !SEMVER.test(entry.skills)) { + errors.push( + `"${key}.skills" must be a valid semver string, got: ${JSON.stringify(entry.skills)}`, + ); + } +} + +if (errors.length) { + console.error("cli-compat.json validation failed:"); + for (const e of errors) { + console.error(` - ${e}`); + } + process.exit(1); +} From e03d89276c3742b231c28af4dbd115a80b8afa60 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 29 Apr 2026 16:38:26 +0200 Subject: [PATCH 2/7] fix: add next>=max validation and document check-cli-compat tool - Validate that "next" appkit/skills versions are >= all versioned entries to prevent accidental downgrades for newest CLIs - Add check-cli-compat.ts to CLAUDE.md tools listing Signed-off-by: Pawel Kosiec --- CLAUDE.md | 1 + tools/check-cli-compat.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index eaf50479..17beb069 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,6 +50,7 @@ Examples: - check-licenses.ts - License compliance checks - build-notice.ts - Build NOTICE.md from dependencies - check-template-deps.ts - Validate template package.json dependencies are pinned + - check-cli-compat.ts - Validate cli-compat.json manifest structure and semver values - finalize-release.ts - Apply release changes (changelog, versions, tags) for secure repo ``` diff --git a/tools/check-cli-compat.ts b/tools/check-cli-compat.ts index f18c1981..259c21fa 100644 --- a/tools/check-cli-compat.ts +++ b/tools/check-cli-compat.ts @@ -5,6 +5,7 @@ * - "next" key is required * - All keys are "next" or valid semver (X.Y.Z) * - All values have "appkit" and "skills" fields with valid semver strings + * - "next" appkit/skills versions are >= all versioned entries */ import { readFileSync } from "node:fs"; @@ -12,6 +13,15 @@ import { join } from "node:path"; const SEMVER = /^\d+\.\d+\.\d+(-[\w.]+)?$/; +function compareSemver(a: string, b: string): number { + const pa = a.split(".").map(Number); + const pb = b.split(".").map(Number); + for (let i = 0; i < 3; i++) { + if (pa[i] !== pb[i]) return pa[i] - pb[i]; + } + return 0; +} + const raw = readFileSync( join(import.meta.dirname, "../cli-compat.json"), "utf-8", @@ -65,6 +75,22 @@ for (const [key, value] of Object.entries(manifest)) { } } +// Validate that "next" versions are >= all versioned entries +const nextEntry = manifest.next as Record; +for (const [key, value] of Object.entries(manifest)) { + if (key === "next") continue; + const entry = value as Record; + if (!entry.appkit || !nextEntry.appkit) continue; + + for (const field of ["appkit", "skills"] as const) { + if (entry[field] && nextEntry[field] && compareSemver(nextEntry[field], entry[field]) < 0) { + errors.push( + `"next.${field}" (${nextEntry[field]}) must be >= "${key}.${field}" (${entry[field]})`, + ); + } + } +} + if (errors.length) { console.error("cli-compat.json validation failed:"); for (const e of errors) { From cccb77854a7c1949748bebb22b8040b144a6c029 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 30 Apr 2026 11:35:28 +0200 Subject: [PATCH 3/7] fix: tighten cli-compat.json validation - Reject prerelease suffixes in SEMVER regex (compareSemver produces NaN for them, silently bypassing next>=max check) - Check appkit and skills fields independently in next>=max validation (previously a missing appkit skipped skills too) - Require at least one versioned CLI entry besides "next" Signed-off-by: Pawel Kosiec --- tools/check-cli-compat.ts | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tools/check-cli-compat.ts b/tools/check-cli-compat.ts index 259c21fa..c4c4263e 100644 --- a/tools/check-cli-compat.ts +++ b/tools/check-cli-compat.ts @@ -11,7 +11,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; -const SEMVER = /^\d+\.\d+\.\d+(-[\w.]+)?$/; +const SEMVER = /^\d+\.\d+\.\d+$/; function compareSemver(a: string, b: string): number { const pa = a.split(".").map(Number); @@ -35,7 +35,11 @@ try { process.exit(1); } -if (typeof manifest !== "object" || manifest === null || Array.isArray(manifest)) { +if ( + typeof manifest !== "object" || + manifest === null || + Array.isArray(manifest) +) { console.error("cli-compat.json must be a JSON object"); process.exit(1); } @@ -45,12 +49,22 @@ if (!("next" in manifest)) { process.exit(1); } +const versionedKeys = Object.keys(manifest).filter((k) => k !== "next"); +if (versionedKeys.length === 0) { + console.error( + 'cli-compat.json must contain at least one versioned CLI entry besides "next"', + ); + process.exit(1); +} + const errors: string[] = []; for (const [key, value] of Object.entries(manifest)) { // Validate key if (key !== "next" && !SEMVER.test(key)) { - errors.push(`Invalid key "${key}": must be "next" or a semver string (X.Y.Z)`); + errors.push( + `Invalid key "${key}": must be "next" or a semver string (X.Y.Z)`, + ); continue; } @@ -80,10 +94,13 @@ const nextEntry = manifest.next as Record; for (const [key, value] of Object.entries(manifest)) { if (key === "next") continue; const entry = value as Record; - if (!entry.appkit || !nextEntry.appkit) continue; for (const field of ["appkit", "skills"] as const) { - if (entry[field] && nextEntry[field] && compareSemver(nextEntry[field], entry[field]) < 0) { + if ( + entry[field] && + nextEntry[field] && + compareSemver(nextEntry[field], entry[field]) < 0 + ) { errors.push( `"next.${field}" (${nextEntry[field]}) must be >= "${key}.${field}" (${entry[field]})`, ); From 0443a623d1ce173ffeb27b65393da41118f86e9d Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 30 Apr 2026 13:25:15 +0200 Subject: [PATCH 4/7] docs: document CLI compat manifest and simplify initial entries - Add "Updating the CLI compatibility manifest" section to CONTRIBUTING.md - Reduce cli-compat.json to two initial entries (next + 0.299.0) Signed-off-by: Pawel Kosiec --- CONTRIBUTING.md | 33 +++++++++++++++++++++++++++++++++ cli-compat.json | 7 ++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ada5188a..5c961283 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,6 +128,39 @@ pnpm docs:serve # Serve built docs See [docs/README.md](./docs/README.md) for more details. +## Updating the CLI compatibility manifest + +`cli-compat.json` maps Databricks CLI versions to compatible AppKit and Agent Skills versions. The CLI fetches this file at runtime to determine which template version to use for `apps init` and `aitools install`. See the [design doc](https://docs.google.com/document/d/1qOchR4vnYAo_5a_oHyFdIdySBayzRc5bBmVZ0u-Mvus/edit) for full context. + +### Manifest format + +```json +{ + "next": { "appkit": "0.24.0", "skills": "0.1.4" }, + "0.299.0": { "appkit": "0.24.0", "skills": "0.1.4" } +} +``` + +- Each key is a CLI version (`X.Y.Z`) or `"next"`. +- Each value specifies the compatible `appkit` and `skills` versions. +- `"next"` is used for CLI versions newer than any listed entry. + +### How the CLI resolves versions + +1. **Exact match** on CLI version → use that entry. +2. **No exact match**, between two entries → use the nearest lower version's entry. +3. **Newer than all entries** → use `"next"`. + +### When to update + +Update `cli-compat.json` via a pull request after each AppKit release: + +- **No template changes** (just an AppKit version bump): update all entries to the new version. +- **Template changes that don't require new CLI features**: test the last few CLI versions and update matching entries. +- **Template changes that require new CLI features**: add a new entry for the minimum CLI version that supports them; older entries keep pointing to the previous template version. + +CI validates the manifest structure and invariants via `tools/check-cli-compat.ts`. + ## Adding or changing a resource type Resource types and their permissions are defined once in the plugin-manifest schema; the CLI (create, add-resource, validate) and the appkit registry types are derived from it. diff --git a/cli-compat.json b/cli-compat.json index fd13c5f9..b7cff3b8 100644 --- a/cli-compat.json +++ b/cli-compat.json @@ -1,7 +1,4 @@ { - "next": { "appkit": "0.27.0", "skills": "0.1.5" }, - "0.296.0": { "appkit": "0.27.0", "skills": "0.1.5" }, - "0.295.0": { "appkit": "0.27.0", "skills": "0.1.5" }, - "0.290.0": { "appkit": "0.24.0", "skills": "0.1.5" }, - "0.288.0": { "appkit": "0.24.0", "skills": "0.1.4" } + "next": { "appkit": "0.24.0", "skills": "0.1.4" }, + "0.299.0": { "appkit": "0.24.0", "skills": "0.1.4" } } From f45d86f3b599f040c2a9e683fd81cbd466c41c55 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 30 Apr 2026 14:24:07 +0200 Subject: [PATCH 5/7] fix: tighten cli-compat validator and add unit tests - Bail out of cross-entry comparison when structure validation fails, preventing NaN-based semver comparisons on invalid entries - Reject unexpected fields in manifest entries - Validate versioned keys are in ascending semver order - Extract validateCliCompat() for testability - Add 20 unit tests covering all validation branches Signed-off-by: Pawel Kosiec --- tools/check-cli-compat.test.ts | 174 +++++++++++++++++++++++++++++++++ tools/check-cli-compat.ts | 162 +++++++++++++++++------------- vitest.config.ts | 7 ++ 3 files changed, 273 insertions(+), 70 deletions(-) create mode 100644 tools/check-cli-compat.test.ts diff --git a/tools/check-cli-compat.test.ts b/tools/check-cli-compat.test.ts new file mode 100644 index 00000000..0a176552 --- /dev/null +++ b/tools/check-cli-compat.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { compareSemver, validateCliCompat } from "./check-cli-compat"; + +describe("compareSemver", () => { + it("returns 0 for equal versions", () => { + expect(compareSemver("1.2.3", "1.2.3")).toBe(0); + }); + + it("compares major versions", () => { + expect(compareSemver("2.0.0", "1.0.0")).toBeGreaterThan(0); + expect(compareSemver("1.0.0", "2.0.0")).toBeLessThan(0); + }); + + it("compares minor versions", () => { + expect(compareSemver("1.2.0", "1.1.0")).toBeGreaterThan(0); + expect(compareSemver("1.1.0", "1.2.0")).toBeLessThan(0); + }); + + it("compares patch versions", () => { + expect(compareSemver("1.0.2", "1.0.1")).toBeGreaterThan(0); + expect(compareSemver("1.0.1", "1.0.2")).toBeLessThan(0); + }); + + it("handles large version numbers", () => { + expect(compareSemver("0.299.0", "0.288.0")).toBeGreaterThan(0); + }); +}); + +describe("validateCliCompat", () => { + const valid = { + next: { appkit: "0.24.0", skills: "0.1.4" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }; + + it("accepts a valid manifest", () => { + expect(validateCliCompat(valid)).toEqual([]); + }); + + it("rejects non-object input", () => { + expect(validateCliCompat(null)).toEqual([ + "cli-compat.json must be a JSON object", + ]); + expect(validateCliCompat([])).toEqual([ + "cli-compat.json must be a JSON object", + ]); + expect(validateCliCompat("string")).toEqual([ + "cli-compat.json must be a JSON object", + ]); + }); + + it("rejects manifest without next key", () => { + expect(validateCliCompat({ "0.299.0": valid["0.299.0"] })).toEqual([ + 'cli-compat.json must contain a "next" key', + ]); + }); + + it("rejects manifest with only next key", () => { + expect(validateCliCompat({ next: valid.next })).toEqual([ + 'cli-compat.json must contain at least one versioned CLI entry besides "next"', + ]); + }); + + it("rejects invalid semver keys", () => { + const errors = validateCliCompat({ + next: valid.next, + "not-semver": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('Invalid key "not-semver"'), + ]); + }); + + it("rejects non-object values", () => { + const errors = validateCliCompat({ + next: valid.next, + "1.0.0": "bad", + }); + expect(errors).toEqual([ + expect.stringContaining('Value for "1.0.0" must be an object'), + ]); + }); + + it("rejects invalid appkit semver", () => { + const errors = validateCliCompat({ + next: { appkit: "bad", skills: "0.1.0" }, + "1.0.0": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next.appkit" must be a valid semver string'), + ]); + }); + + it("rejects invalid skills semver", () => { + const errors = validateCliCompat({ + next: { appkit: "0.1.0", skills: "not-valid" }, + "1.0.0": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next.skills" must be a valid semver string'), + ]); + }); + + it("rejects missing appkit field", () => { + const errors = validateCliCompat({ + next: { skills: "0.1.0" }, + "1.0.0": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next.appkit" must be a valid semver string'), + ]); + }); + + it("rejects extra fields in entries", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.4", extra: "bad" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next" has unexpected fields: extra'), + ]); + }); + + it("rejects next.appkit lower than a versioned entry", () => { + const errors = validateCliCompat({ + next: { appkit: "0.23.0", skills: "0.1.4" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([ + expect.stringContaining( + '"next.appkit" (0.23.0) must be >= "0.299.0.appkit" (0.24.0)', + ), + ]); + }); + + it("rejects next.skills lower than a versioned entry", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.0" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([ + expect.stringContaining( + '"next.skills" (0.1.0) must be >= "0.299.0.skills" (0.1.4)', + ), + ]); + }); + + it("rejects non-monotonic versioned keys", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.4" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + "0.288.0": { appkit: "0.23.0", skills: "0.1.3" }, + }); + expect(errors).toEqual([expect.stringContaining("ascending semver order")]); + }); + + it("accepts multiple versioned entries in ascending order", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.4" }, + "0.288.0": { appkit: "0.23.0", skills: "0.1.3" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([]); + }); + + it("returns structure errors before cross-entry checks", () => { + const errors = validateCliCompat({ + next: { appkit: "bad", skills: "0.1.0" }, + "1.0.0": { appkit: "0.2.0", skills: "0.2.0" }, + }); + // Should get the structure error but NOT a "next >= versioned" error + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"next.appkit" must be a valid semver string'); + }); +}); diff --git a/tools/check-cli-compat.ts b/tools/check-cli-compat.ts index c4c4263e..204964cd 100644 --- a/tools/check-cli-compat.ts +++ b/tools/check-cli-compat.ts @@ -4,8 +4,10 @@ * - Valid JSON * - "next" key is required * - All keys are "next" or valid semver (X.Y.Z) - * - All values have "appkit" and "skills" fields with valid semver strings + * - All values have exactly "appkit" and "skills" fields with valid semver strings + * - No extra fields in entries * - "next" appkit/skills versions are >= all versioned entries + * - Versioned keys are in ascending semver order */ import { readFileSync } from "node:fs"; @@ -13,7 +15,7 @@ import { join } from "node:path"; const SEMVER = /^\d+\.\d+\.\d+$/; -function compareSemver(a: string, b: string): number { +export function compareSemver(a: string, b: string): number { const pa = a.split(".").map(Number); const pb = b.split(".").map(Number); for (let i = 0; i < 3; i++) { @@ -22,92 +24,112 @@ function compareSemver(a: string, b: string): number { return 0; } -const raw = readFileSync( - join(import.meta.dirname, "../cli-compat.json"), - "utf-8", -); +export function validateCliCompat(manifest: unknown): string[] { + if ( + typeof manifest !== "object" || + manifest === null || + Array.isArray(manifest) + ) { + return ["cli-compat.json must be a JSON object"]; + } -let manifest: Record; -try { - manifest = JSON.parse(raw); -} catch { - console.error("cli-compat.json is not valid JSON"); - process.exit(1); -} + const obj = manifest as Record; -if ( - typeof manifest !== "object" || - manifest === null || - Array.isArray(manifest) -) { - console.error("cli-compat.json must be a JSON object"); - process.exit(1); -} + if (!("next" in obj)) { + return ['cli-compat.json must contain a "next" key']; + } -if (!("next" in manifest)) { - console.error('cli-compat.json must contain a "next" key'); - process.exit(1); -} + const versionedKeys = Object.keys(obj).filter((k) => k !== "next"); + if (versionedKeys.length === 0) { + return [ + 'cli-compat.json must contain at least one versioned CLI entry besides "next"', + ]; + } -const versionedKeys = Object.keys(manifest).filter((k) => k !== "next"); -if (versionedKeys.length === 0) { - console.error( - 'cli-compat.json must contain at least one versioned CLI entry besides "next"', - ); - process.exit(1); -} + const errors: string[] = []; -const errors: string[] = []; + for (const [key, value] of Object.entries(obj)) { + if (key !== "next" && !SEMVER.test(key)) { + errors.push( + `Invalid key "${key}": must be "next" or a semver string (X.Y.Z)`, + ); + continue; + } -for (const [key, value] of Object.entries(manifest)) { - // Validate key - if (key !== "next" && !SEMVER.test(key)) { - errors.push( - `Invalid key "${key}": must be "next" or a semver string (X.Y.Z)`, - ); - continue; - } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + errors.push(`Value for "${key}" must be an object`); + continue; + } - // Validate value shape - if (typeof value !== "object" || value === null || Array.isArray(value)) { - errors.push(`Value for "${key}" must be an object`); - continue; - } + const entry = value as Record; + + if (typeof entry.appkit !== "string" || !SEMVER.test(entry.appkit)) { + errors.push( + `"${key}.appkit" must be a valid semver string, got: ${JSON.stringify(entry.appkit)}`, + ); + } - const entry = value as Record; + if (typeof entry.skills !== "string" || !SEMVER.test(entry.skills)) { + errors.push( + `"${key}.skills" must be a valid semver string, got: ${JSON.stringify(entry.skills)}`, + ); + } - if (typeof entry.appkit !== "string" || !SEMVER.test(entry.appkit)) { - errors.push( - `"${key}.appkit" must be a valid semver string, got: ${JSON.stringify(entry.appkit)}`, + const extraFields = Object.keys(entry).filter( + (k) => k !== "appkit" && k !== "skills", ); + if (extraFields.length > 0) { + errors.push(`"${key}" has unexpected fields: ${extraFields.join(", ")}`); + } } - if (typeof entry.skills !== "string" || !SEMVER.test(entry.skills)) { - errors.push( - `"${key}.skills" must be a valid semver string, got: ${JSON.stringify(entry.skills)}`, - ); + // Stop early if structure validation failed — cross-entry checks assume valid entries + if (errors.length > 0) { + return errors; } -} -// Validate that "next" versions are >= all versioned entries -const nextEntry = manifest.next as Record; -for (const [key, value] of Object.entries(manifest)) { - if (key === "next") continue; - const entry = value as Record; - - for (const field of ["appkit", "skills"] as const) { - if ( - entry[field] && - nextEntry[field] && - compareSemver(nextEntry[field], entry[field]) < 0 - ) { - errors.push( - `"next.${field}" (${nextEntry[field]}) must be >= "${key}.${field}" (${entry[field]})`, - ); + // Validate that "next" versions are >= all versioned entries + const nextEntry = obj.next as Record; + for (const [key, value] of Object.entries(obj)) { + if (key === "next") continue; + const entry = value as Record; + + for (const field of ["appkit", "skills"] as const) { + if (compareSemver(nextEntry[field], entry[field]) < 0) { + errors.push( + `"next.${field}" (${nextEntry[field]}) must be >= "${key}.${field}" (${entry[field]})`, + ); + } + } + } + + // Validate versioned keys are in ascending semver order + const sorted = [...versionedKeys].sort(compareSemver); + for (let i = 0; i < sorted.length; i++) { + if (sorted[i] !== versionedKeys[i]) { + errors.push("Versioned keys must be in ascending semver order"); + break; } } + + return errors; +} + +// CLI entrypoint +const raw = readFileSync( + join(import.meta.dirname, "../cli-compat.json"), + "utf-8", +); + +let manifest: unknown; +try { + manifest = JSON.parse(raw); +} catch { + console.error("cli-compat.json is not valid JSON"); + process.exit(1); } +const errors = validateCliCompat(manifest); if (errors.length) { console.error("cli-compat.json validation failed:"); for (const e of errors) { diff --git a/vitest.config.ts b/vitest.config.ts index 8c2893b0..5e272f03 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -58,6 +58,13 @@ export default defineConfig({ environment: "node", }, }, + { + test: { + name: "tools", + root: "./tools", + environment: "node", + }, + }, ], }, }); From 5afbe63dba3a14ff639276ffc0bf5e3051fc8c2c Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 30 Apr 2026 15:12:55 +0200 Subject: [PATCH 6/7] docs: align CLI compat update docs with design doc and add bump skill - Rewrite "When to update" section in CONTRIBUTING.md to match design doc (evals step, "last 3 CLI versions" detail, future automation note) - Add bump-cli-compat Claude Code skill for updating the manifest Signed-off-by: Pawel Kosiec --- .claude/skills/bump-cli-compat.md | 134 ++++++++++++++++++++++++++++++ CONTRIBUTING.md | 14 ++-- 2 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 .claude/skills/bump-cli-compat.md diff --git a/.claude/skills/bump-cli-compat.md b/.claude/skills/bump-cli-compat.md new file mode 100644 index 00000000..ffd41e79 --- /dev/null +++ b/.claude/skills/bump-cli-compat.md @@ -0,0 +1,134 @@ +--- +name: bump-cli-compat +description: "Bump cli-compat.json with new AppKit and Agent Skills versions, then create a PR. Use when the user says 'bump cli-compat', 'update cli-compat', 'bump compatibility manifest', 'new appkit release cli-compat', or wants to update the CLI compatibility manifest after an AppKit or Agent Skills release." +user-invocable: true +allowed-tools: Read, Edit, Write, Bash, Glob, Grep, AskUserQuestion +--- + +# Bump CLI Compatibility Manifest + +Updates `cli-compat.json` with new AppKit and Agent Skills versions, validates the result, and creates a PR. + +## Arguments + +Parse the user's input for optional version arguments: + +- `--appkit ` or first positional arg → AppKit version (e.g. `0.28.0`) +- `--skills ` or second positional arg → Agent Skills version (e.g. `0.1.6`) +- No args → auto-detect latest versions from GitHub tags + +Versions should be provided **without** the `v` prefix (e.g. `0.28.0`, not `v0.28.0`). If provided with the prefix, strip it. + +## Workflow + +### Step 1: Resolve versions + +If both `appkit` and `skills` versions were provided as arguments, skip to Step 2. + +Otherwise, fetch the latest tags from GitHub: + +```bash +# Latest appkit version (strip leading 'v') +gh api repos/databricks/appkit/tags --jq '.[0].name' | sed 's/^v//' + +# Latest skills version (strip leading 'v') +gh api repos/databricks/databricks-agent-skills/tags --jq '.[0].name' | sed 's/^v//' +``` + +Show the resolved versions to the user and ask: + +> The latest versions are: +> - AppKit: `{appkit_version}` +> - Agent Skills: `{skills_version}` +> +> Have these versions been evaluated (evals passed with no regressions)? + +**Do NOT proceed until the user confirms.** If the user says no or wants different versions, ask them to provide the correct versions. + +### Step 2: Validate tags exist + +Verify that the corresponding Git tags exist on GitHub: + +```bash +gh api repos/databricks/appkit/git/ref/tags/v{appkit_version} --jq '.ref' 2>&1 +gh api repos/databricks/databricks-agent-skills/git/ref/tags/v{skills_version} --jq '.ref' 2>&1 +``` + +If either tag doesn't exist, report the error and stop. + +### Step 3: Read current manifest + +Read `cli-compat.json` from the repo root. Note the current versions and the list of versioned entries. + +### Step 4: Update the manifest + +Update **all entries** (both `next` and all versioned CLI entries) to the new appkit and skills versions. This is the "no template changes" scenario from CONTRIBUTING.md — a simple search & replace. + +Write the updated `cli-compat.json`. + +### Step 5: Validate + +Run the CI validator to ensure the manifest is well-formed: + +```bash +pnpm exec tsx tools/check-cli-compat.ts +``` + +If validation fails, show the errors and fix them before proceeding. + +### Step 6: Format + +Run the formatter to ensure consistent style: + +```bash +pnpm exec biome check --write cli-compat.json +``` + +### Step 7: Create branch, commit, and PR + +```bash +# Create a new branch from the current branch (or main) +git checkout -b bump-cli-compat-appkit-{appkit_version}-skills-{skills_version} + +# Stage and commit +git add cli-compat.json +git commit -s -m "chore: bump cli-compat to appkit {appkit_version}, skills {skills_version}" + +# Push and create PR +git push -u origin HEAD +gh pr create \ + --title "chore: bump cli-compat to appkit {appkit_version}, skills {skills_version}" \ + --body "$(cat <<'EOF' +## Summary +Bump `cli-compat.json` to use: +- AppKit `{appkit_version}` +- Agent Skills `{skills_version}` + +## Checklist +- [ ] Evals passed with no regressions +- [ ] `pnpm exec tsx tools/check-cli-compat.ts` passes +EOF +)" +``` + +Show the PR URL to the user when done. + +## Examples + +### Example: With explicit versions +``` +/bump-cli-compat 0.28.0 0.1.6 +``` +Validates tags exist, updates manifest, creates PR. + +### Example: Auto-detect latest +``` +/bump-cli-compat +``` +Fetches latest tags, asks for eval confirmation, then updates and creates PR. + +### Example: With flags +``` +/bump-cli-compat --appkit 0.28.0 --skills 0.1.6 +``` +Same as positional args. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c961283..15a1a087 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,7 +130,7 @@ See [docs/README.md](./docs/README.md) for more details. ## Updating the CLI compatibility manifest -`cli-compat.json` maps Databricks CLI versions to compatible AppKit and Agent Skills versions. The CLI fetches this file at runtime to determine which template version to use for `apps init` and `aitools install`. See the [design doc](https://docs.google.com/document/d/1qOchR4vnYAo_5a_oHyFdIdySBayzRc5bBmVZ0u-Mvus/edit) for full context. +`cli-compat.json` maps Databricks CLI versions to compatible AppKit and Agent Skills versions. The CLI fetches this file at runtime to determine which template version to use for `apps init` and `aitools install`. ### Manifest format @@ -153,11 +153,15 @@ See [docs/README.md](./docs/README.md) for more details. ### When to update -Update `cli-compat.json` via a pull request after each AppKit release: +After each AppKit release: -- **No template changes** (just an AppKit version bump): update all entries to the new version. -- **Template changes that don't require new CLI features**: test the last few CLI versions and update matching entries. -- **Template changes that require new CLI features**: add a new entry for the minimum CLI version that supports them; older entries keep pointing to the previous template version. +1. **Run evals** on the new AppKit version. If there is no regression, proceed. +2. **Open a PR** to update `cli-compat.json`. The change depends on the type of release: + - **No template changes** (just an AppKit/skills version bump): search & replace all version occurrences in the manifest and update `next`. + - **Template changes that don't require new CLI features**: test the last 3 CLI versions with the new template and update matching entries. + - **Template changes that require new CLI features**: add a new entry for the minimum CLI version that supports them; older entries keep pointing to the previous template version. + +This process is manual for now but can be automated as part of the release workflow in the future. Use the `/bump-cli-compat` Claude Code skill to automate the update and PR creation. CI validates the manifest structure and invariants via `tools/check-cli-compat.ts`. From 9b4308586c90609d3af906cf4347db8066f4c788 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 30 Apr 2026 15:43:56 +0200 Subject: [PATCH 7/7] docs: document CLI resolution fallback for older versions Signed-off-by: Pawel Kosiec --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15a1a087..1ecf7b8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,6 +150,7 @@ See [docs/README.md](./docs/README.md) for more details. 1. **Exact match** on CLI version → use that entry. 2. **No exact match**, between two entries → use the nearest lower version's entry. 3. **Newer than all entries** → use `"next"`. +4. **Older than all entries** → use the oldest versioned entry. ### When to update