diff --git a/.claude/skills/bump-cli-compat.md b/.claude/skills/bump-cli-compat.md new file mode 100644 index 000000000..ffd41e798 --- /dev/null +++ b/.claude/skills/bump-cli-compat.md @@ -0,0 +1,134 @@ +--- +name: bump-cli-compat +description: "Bump cli-compat.json with new AppKit and Agent Skills versions, then create a PR. Use when the user says 'bump cli-compat', 'update cli-compat', 'bump compatibility manifest', 'new appkit release cli-compat', or wants to update the CLI compatibility manifest after an AppKit or Agent Skills release." +user-invocable: true +allowed-tools: Read, Edit, Write, Bash, Glob, Grep, AskUserQuestion +--- + +# Bump CLI Compatibility Manifest + +Updates `cli-compat.json` with new AppKit and Agent Skills versions, validates the result, and creates a PR. + +## Arguments + +Parse the user's input for optional version arguments: + +- `--appkit ` or first positional arg → AppKit version (e.g. `0.28.0`) +- `--skills ` or second positional arg → Agent Skills version (e.g. `0.1.6`) +- No args → auto-detect latest versions from GitHub tags + +Versions should be provided **without** the `v` prefix (e.g. `0.28.0`, not `v0.28.0`). If provided with the prefix, strip it. + +## Workflow + +### Step 1: Resolve versions + +If both `appkit` and `skills` versions were provided as arguments, skip to Step 2. + +Otherwise, fetch the latest tags from GitHub: + +```bash +# Latest appkit version (strip leading 'v') +gh api repos/databricks/appkit/tags --jq '.[0].name' | sed 's/^v//' + +# Latest skills version (strip leading 'v') +gh api repos/databricks/databricks-agent-skills/tags --jq '.[0].name' | sed 's/^v//' +``` + +Show the resolved versions to the user and ask: + +> The latest versions are: +> - AppKit: `{appkit_version}` +> - Agent Skills: `{skills_version}` +> +> Have these versions been evaluated (evals passed with no regressions)? + +**Do NOT proceed until the user confirms.** If the user says no or wants different versions, ask them to provide the correct versions. + +### Step 2: Validate tags exist + +Verify that the corresponding Git tags exist on GitHub: + +```bash +gh api repos/databricks/appkit/git/ref/tags/v{appkit_version} --jq '.ref' 2>&1 +gh api repos/databricks/databricks-agent-skills/git/ref/tags/v{skills_version} --jq '.ref' 2>&1 +``` + +If either tag doesn't exist, report the error and stop. + +### Step 3: Read current manifest + +Read `cli-compat.json` from the repo root. Note the current versions and the list of versioned entries. + +### Step 4: Update the manifest + +Update **all entries** (both `next` and all versioned CLI entries) to the new appkit and skills versions. This is the "no template changes" scenario from CONTRIBUTING.md — a simple search & replace. + +Write the updated `cli-compat.json`. + +### Step 5: Validate + +Run the CI validator to ensure the manifest is well-formed: + +```bash +pnpm exec tsx tools/check-cli-compat.ts +``` + +If validation fails, show the errors and fix them before proceeding. + +### Step 6: Format + +Run the formatter to ensure consistent style: + +```bash +pnpm exec biome check --write cli-compat.json +``` + +### Step 7: Create branch, commit, and PR + +```bash +# Create a new branch from the current branch (or main) +git checkout -b bump-cli-compat-appkit-{appkit_version}-skills-{skills_version} + +# Stage and commit +git add cli-compat.json +git commit -s -m "chore: bump cli-compat to appkit {appkit_version}, skills {skills_version}" + +# Push and create PR +git push -u origin HEAD +gh pr create \ + --title "chore: bump cli-compat to appkit {appkit_version}, skills {skills_version}" \ + --body "$(cat <<'EOF' +## Summary +Bump `cli-compat.json` to use: +- AppKit `{appkit_version}` +- Agent Skills `{skills_version}` + +## Checklist +- [ ] Evals passed with no regressions +- [ ] `pnpm exec tsx tools/check-cli-compat.ts` passes +EOF +)" +``` + +Show the PR URL to the user when done. + +## Examples + +### Example: With explicit versions +``` +/bump-cli-compat 0.28.0 0.1.6 +``` +Validates tags exist, updates manifest, creates PR. + +### Example: Auto-detect latest +``` +/bump-cli-compat +``` +Fetches latest tags, asks for eval confirmation, then updates and creates PR. + +### Example: With flags +``` +/bump-cli-compat --appkit 0.28.0 --skills 0.1.6 +``` +Same as positional args. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cd0af7dc..273d97827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,8 @@ jobs: run: pnpm run check:licenses - name: Check template deps are pinned run: pnpm exec tsx tools/check-template-deps.ts + - name: Check CLI compat manifest + run: pnpm exec tsx tools/check-cli-compat.ts test: name: Unit Tests diff --git a/CLAUDE.md b/CLAUDE.md index eaf504796..17beb0691 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,6 +50,7 @@ Examples: - check-licenses.ts - License compliance checks - build-notice.ts - Build NOTICE.md from dependencies - check-template-deps.ts - Validate template package.json dependencies are pinned + - check-cli-compat.ts - Validate cli-compat.json manifest structure and semver values - finalize-release.ts - Apply release changes (changelog, versions, tags) for secure repo ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ada5188a9..1ecf7b8a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,6 +128,44 @@ pnpm docs:serve # Serve built docs See [docs/README.md](./docs/README.md) for more details. +## Updating the CLI compatibility manifest + +`cli-compat.json` maps Databricks CLI versions to compatible AppKit and Agent Skills versions. The CLI fetches this file at runtime to determine which template version to use for `apps init` and `aitools install`. + +### Manifest format + +```json +{ + "next": { "appkit": "0.24.0", "skills": "0.1.4" }, + "0.299.0": { "appkit": "0.24.0", "skills": "0.1.4" } +} +``` + +- Each key is a CLI version (`X.Y.Z`) or `"next"`. +- Each value specifies the compatible `appkit` and `skills` versions. +- `"next"` is used for CLI versions newer than any listed entry. + +### How the CLI resolves versions + +1. **Exact match** on CLI version → use that entry. +2. **No exact match**, between two entries → use the nearest lower version's entry. +3. **Newer than all entries** → use `"next"`. +4. **Older than all entries** → use the oldest versioned entry. + +### When to update + +After each AppKit release: + +1. **Run evals** on the new AppKit version. If there is no regression, proceed. +2. **Open a PR** to update `cli-compat.json`. The change depends on the type of release: + - **No template changes** (just an AppKit/skills version bump): search & replace all version occurrences in the manifest and update `next`. + - **Template changes that don't require new CLI features**: test the last 3 CLI versions with the new template and update matching entries. + - **Template changes that require new CLI features**: add a new entry for the minimum CLI version that supports them; older entries keep pointing to the previous template version. + +This process is manual for now but can be automated as part of the release workflow in the future. Use the `/bump-cli-compat` Claude Code skill to automate the update and PR creation. + +CI validates the manifest structure and invariants via `tools/check-cli-compat.ts`. + ## Adding or changing a resource type Resource types and their permissions are defined once in the plugin-manifest schema; the CLI (create, add-resource, validate) and the appkit registry types are derived from it. diff --git a/cli-compat.json b/cli-compat.json new file mode 100644 index 000000000..b7cff3b83 --- /dev/null +++ b/cli-compat.json @@ -0,0 +1,4 @@ +{ + "next": { "appkit": "0.24.0", "skills": "0.1.4" }, + "0.299.0": { "appkit": "0.24.0", "skills": "0.1.4" } +} diff --git a/tools/check-cli-compat.test.ts b/tools/check-cli-compat.test.ts new file mode 100644 index 000000000..0a1765521 --- /dev/null +++ b/tools/check-cli-compat.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { compareSemver, validateCliCompat } from "./check-cli-compat"; + +describe("compareSemver", () => { + it("returns 0 for equal versions", () => { + expect(compareSemver("1.2.3", "1.2.3")).toBe(0); + }); + + it("compares major versions", () => { + expect(compareSemver("2.0.0", "1.0.0")).toBeGreaterThan(0); + expect(compareSemver("1.0.0", "2.0.0")).toBeLessThan(0); + }); + + it("compares minor versions", () => { + expect(compareSemver("1.2.0", "1.1.0")).toBeGreaterThan(0); + expect(compareSemver("1.1.0", "1.2.0")).toBeLessThan(0); + }); + + it("compares patch versions", () => { + expect(compareSemver("1.0.2", "1.0.1")).toBeGreaterThan(0); + expect(compareSemver("1.0.1", "1.0.2")).toBeLessThan(0); + }); + + it("handles large version numbers", () => { + expect(compareSemver("0.299.0", "0.288.0")).toBeGreaterThan(0); + }); +}); + +describe("validateCliCompat", () => { + const valid = { + next: { appkit: "0.24.0", skills: "0.1.4" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }; + + it("accepts a valid manifest", () => { + expect(validateCliCompat(valid)).toEqual([]); + }); + + it("rejects non-object input", () => { + expect(validateCliCompat(null)).toEqual([ + "cli-compat.json must be a JSON object", + ]); + expect(validateCliCompat([])).toEqual([ + "cli-compat.json must be a JSON object", + ]); + expect(validateCliCompat("string")).toEqual([ + "cli-compat.json must be a JSON object", + ]); + }); + + it("rejects manifest without next key", () => { + expect(validateCliCompat({ "0.299.0": valid["0.299.0"] })).toEqual([ + 'cli-compat.json must contain a "next" key', + ]); + }); + + it("rejects manifest with only next key", () => { + expect(validateCliCompat({ next: valid.next })).toEqual([ + 'cli-compat.json must contain at least one versioned CLI entry besides "next"', + ]); + }); + + it("rejects invalid semver keys", () => { + const errors = validateCliCompat({ + next: valid.next, + "not-semver": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('Invalid key "not-semver"'), + ]); + }); + + it("rejects non-object values", () => { + const errors = validateCliCompat({ + next: valid.next, + "1.0.0": "bad", + }); + expect(errors).toEqual([ + expect.stringContaining('Value for "1.0.0" must be an object'), + ]); + }); + + it("rejects invalid appkit semver", () => { + const errors = validateCliCompat({ + next: { appkit: "bad", skills: "0.1.0" }, + "1.0.0": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next.appkit" must be a valid semver string'), + ]); + }); + + it("rejects invalid skills semver", () => { + const errors = validateCliCompat({ + next: { appkit: "0.1.0", skills: "not-valid" }, + "1.0.0": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next.skills" must be a valid semver string'), + ]); + }); + + it("rejects missing appkit field", () => { + const errors = validateCliCompat({ + next: { skills: "0.1.0" }, + "1.0.0": { appkit: "0.1.0", skills: "0.1.0" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next.appkit" must be a valid semver string'), + ]); + }); + + it("rejects extra fields in entries", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.4", extra: "bad" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([ + expect.stringContaining('"next" has unexpected fields: extra'), + ]); + }); + + it("rejects next.appkit lower than a versioned entry", () => { + const errors = validateCliCompat({ + next: { appkit: "0.23.0", skills: "0.1.4" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([ + expect.stringContaining( + '"next.appkit" (0.23.0) must be >= "0.299.0.appkit" (0.24.0)', + ), + ]); + }); + + it("rejects next.skills lower than a versioned entry", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.0" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([ + expect.stringContaining( + '"next.skills" (0.1.0) must be >= "0.299.0.skills" (0.1.4)', + ), + ]); + }); + + it("rejects non-monotonic versioned keys", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.4" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + "0.288.0": { appkit: "0.23.0", skills: "0.1.3" }, + }); + expect(errors).toEqual([expect.stringContaining("ascending semver order")]); + }); + + it("accepts multiple versioned entries in ascending order", () => { + const errors = validateCliCompat({ + next: { appkit: "0.24.0", skills: "0.1.4" }, + "0.288.0": { appkit: "0.23.0", skills: "0.1.3" }, + "0.299.0": { appkit: "0.24.0", skills: "0.1.4" }, + }); + expect(errors).toEqual([]); + }); + + it("returns structure errors before cross-entry checks", () => { + const errors = validateCliCompat({ + next: { appkit: "bad", skills: "0.1.0" }, + "1.0.0": { appkit: "0.2.0", skills: "0.2.0" }, + }); + // Should get the structure error but NOT a "next >= versioned" error + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('"next.appkit" must be a valid semver string'); + }); +}); diff --git a/tools/check-cli-compat.ts b/tools/check-cli-compat.ts new file mode 100644 index 000000000..204964cdc --- /dev/null +++ b/tools/check-cli-compat.ts @@ -0,0 +1,139 @@ +#!/usr/bin/env tsx +/** + * Validates that cli-compat.json is well-formed: + * - Valid JSON + * - "next" key is required + * - All keys are "next" or valid semver (X.Y.Z) + * - All values have exactly "appkit" and "skills" fields with valid semver strings + * - No extra fields in entries + * - "next" appkit/skills versions are >= all versioned entries + * - Versioned keys are in ascending semver order + */ + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const SEMVER = /^\d+\.\d+\.\d+$/; + +export function compareSemver(a: string, b: string): number { + const pa = a.split(".").map(Number); + const pb = b.split(".").map(Number); + for (let i = 0; i < 3; i++) { + if (pa[i] !== pb[i]) return pa[i] - pb[i]; + } + return 0; +} + +export function validateCliCompat(manifest: unknown): string[] { + if ( + typeof manifest !== "object" || + manifest === null || + Array.isArray(manifest) + ) { + return ["cli-compat.json must be a JSON object"]; + } + + const obj = manifest as Record; + + if (!("next" in obj)) { + return ['cli-compat.json must contain a "next" key']; + } + + const versionedKeys = Object.keys(obj).filter((k) => k !== "next"); + if (versionedKeys.length === 0) { + return [ + 'cli-compat.json must contain at least one versioned CLI entry besides "next"', + ]; + } + + const errors: string[] = []; + + for (const [key, value] of Object.entries(obj)) { + if (key !== "next" && !SEMVER.test(key)) { + errors.push( + `Invalid key "${key}": must be "next" or a semver string (X.Y.Z)`, + ); + continue; + } + + if (typeof value !== "object" || value === null || Array.isArray(value)) { + errors.push(`Value for "${key}" must be an object`); + continue; + } + + const entry = value as Record; + + if (typeof entry.appkit !== "string" || !SEMVER.test(entry.appkit)) { + errors.push( + `"${key}.appkit" must be a valid semver string, got: ${JSON.stringify(entry.appkit)}`, + ); + } + + if (typeof entry.skills !== "string" || !SEMVER.test(entry.skills)) { + errors.push( + `"${key}.skills" must be a valid semver string, got: ${JSON.stringify(entry.skills)}`, + ); + } + + const extraFields = Object.keys(entry).filter( + (k) => k !== "appkit" && k !== "skills", + ); + if (extraFields.length > 0) { + errors.push(`"${key}" has unexpected fields: ${extraFields.join(", ")}`); + } + } + + // Stop early if structure validation failed — cross-entry checks assume valid entries + if (errors.length > 0) { + return errors; + } + + // Validate that "next" versions are >= all versioned entries + const nextEntry = obj.next as Record; + for (const [key, value] of Object.entries(obj)) { + if (key === "next") continue; + const entry = value as Record; + + for (const field of ["appkit", "skills"] as const) { + if (compareSemver(nextEntry[field], entry[field]) < 0) { + errors.push( + `"next.${field}" (${nextEntry[field]}) must be >= "${key}.${field}" (${entry[field]})`, + ); + } + } + } + + // Validate versioned keys are in ascending semver order + const sorted = [...versionedKeys].sort(compareSemver); + for (let i = 0; i < sorted.length; i++) { + if (sorted[i] !== versionedKeys[i]) { + errors.push("Versioned keys must be in ascending semver order"); + break; + } + } + + return errors; +} + +// CLI entrypoint +const raw = readFileSync( + join(import.meta.dirname, "../cli-compat.json"), + "utf-8", +); + +let manifest: unknown; +try { + manifest = JSON.parse(raw); +} catch { + console.error("cli-compat.json is not valid JSON"); + process.exit(1); +} + +const errors = validateCliCompat(manifest); +if (errors.length) { + console.error("cli-compat.json validation failed:"); + for (const e of errors) { + console.error(` - ${e}`); + } + process.exit(1); +} diff --git a/vitest.config.ts b/vitest.config.ts index 8c2893b01..5e272f031 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -58,6 +58,13 @@ export default defineConfig({ environment: "node", }, }, + { + test: { + name: "tools", + root: "./tools", + environment: "node", + }, + }, ], }, });