From 69eb2f70fb49965dfb198bc7b92e9e49d83ab604 Mon Sep 17 00:00:00 2001 From: Dhruv Gupta Date: Thu, 30 Apr 2026 04:12:58 +0000 Subject: [PATCH 1/5] Tests: uv_sync --frozen so the runner doesn't mutate the checked-in lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #201 switched templates from `requirements.txt` to checked-in `uv.lock` files (and removed `uv.lock` from .gitignore). The Databricks Apps runtime now runs `uv sync --locked` against that lockfile during the build step. The integration-test runner sets `UV_EXCLUDE_NEWER` to pin third-party churn in the runner's own deps. That env var also applied to the runner's `setup:uv-sync` step on the template subdir — `uv sync` re-resolved under the cutoff and rewrote the template's `uv.lock` with `excluded-newer = 2026-03-19T...` baked into its metadata. Bundle deploy then uploaded the contaminated lock, and the Apps runtime — which doesn't have UV_EXCLUDE_NEWER set — detected the cutoff was removed, ignored the lock, re-resolved to a different package set, and failed with: Ignoring existing lockfile due to removal of global exclude newer The lockfile at `uv.lock` needs to be updated, but `--locked` was provided. Repro'd in nightly run 25145086773 (agent-non-conversational, 2026-04-30). Switching to `uv sync --frozen` installs straight from the lock without re-resolving, so the on-disk lockfile stays byte-identical to what's checked in. Deploy then uploads the same lock that every end user gets via `bundle deploy`, and the Apps runtime build succeeds against an unmodified lock — matching the post-#201 flow exactly. Co-authored-by: Isaac --- .scripts/agent-integration-tests/helpers.py | 23 ++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.scripts/agent-integration-tests/helpers.py b/.scripts/agent-integration-tests/helpers.py index a4ae2605..b71d5c58 100644 --- a/.scripts/agent-integration-tests/helpers.py +++ b/.scripts/agent-integration-tests/helpers.py @@ -261,14 +261,31 @@ def clean_template(template_dir: Path): def uv_sync(template_dir: Path, max_attempts: int = 3): - """Run `uv sync` to create/update the venv before quickstart. + """Run `uv sync --frozen` to create the venv from the checked-in lockfile. + + `--frozen` matters: the runner workflow sets `UV_EXCLUDE_NEWER` to pin + third-party churn in the runner's own deps. That env var also leaks + into `uv sync` invocations in the template subdir — uv re-resolves + under the cutoff and rewrites the template's `uv.lock` with + `excluded-newer` baked into its metadata. Bundle deploy then uploads + the contaminated lockfile, and the Databricks Apps runtime — which + runs `uv sync --locked` *without* the env var — detects the cutoff + was removed, ignores the lock, re-resolves to a different package + set, and fails: + Ignoring existing lockfile due to removal of global exclude newer + The lockfile at `uv.lock` needs to be updated, but `--locked` + was provided. + `--frozen` installs straight from the existing lock without + re-resolving, leaving the on-disk lockfile byte-identical so deploy + uploads the version the template's authors checked in (matching the + end-user `bundle deploy` flow exactly). Retries up to ``max_attempts`` times with a short backoff to absorb transient PyPI / proxy hiccups, then falls back to UV_OFFLINE=true one more time as a last resort. """ for attempt in range(1, max_attempts + 1): - result = _run_cmd(["uv", "sync"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT) + result = _run_cmd(["uv", "sync", "--frozen"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT) if result.returncode == 0: return if attempt < max_attempts: @@ -278,7 +295,7 @@ def uv_sync(template_dir: Path, max_attempts: int = 3): _log(f" uv sync failed online; falling back to UV_OFFLINE=true (cache-only)...") env = os.environ.copy() env["UV_OFFLINE"] = "true" - result = _run_cmd(["uv", "sync"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT, env=env) + result = _run_cmd(["uv", "sync", "--frozen"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT, env=env) assert result.returncode == 0, ( f"uv sync failed in {template_dir.name}:\n" f"stdout: {result.stdout}\n" From 09a8be442e5aa039e24300a0f51feb71de527e95 Mon Sep 17 00:00:00 2001 From: Dhruv Gupta Date: Thu, 30 Apr 2026 04:35:02 +0000 Subject: [PATCH 2/5] Scrub UV_EXCLUDE_NEWER in _run_cmd for uv subprocess calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iterating on the previous --frozen-only fix. `uv sync --frozen` keeps the lockfile pristine, but every other uv invocation — `uv run quickstart`, `uv run start-server`, `uv run agent-evaluate` — does its own auto-sync by default and STILL re-resolves under UV_EXCLUDE_NEWER, rewriting the template's uv.lock with `excluded-newer` baked in. CI run 25147080149 confirmed: setup:uv-sync ran cleanly with --frozen (10.6s), then setup:quickstart's `uv run quickstart` (13.6s) re-synced and contaminated the lock again. Bundle deploy uploaded the contaminated lockfile. Apps runtime's `uv sync --locked` rejected it identically to the original failure. Fix: scrub UV_EXCLUDE_NEWER from the environment of every `uv` subprocess call in `_run_cmd`. The runner's own deps still get the cutoff applied via the workflow's `Sync test dependencies` shell step (which is the only place the cutoff is meant to live). Template-side uv calls now run with whatever cutoff end users have — none — matching the end-user `bundle deploy` flow exactly. Keep `--frozen` on uv_sync as belt-and-suspenders: even if a future change forgets to scrub, --frozen makes uv error loudly instead of silently rewriting the lock. Co-authored-by: Isaac --- .scripts/agent-integration-tests/helpers.py | 37 ++++++++++++--------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/.scripts/agent-integration-tests/helpers.py b/.scripts/agent-integration-tests/helpers.py index b71d5c58..7bfb9a9e 100644 --- a/.scripts/agent-integration-tests/helpers.py +++ b/.scripts/agent-integration-tests/helpers.py @@ -115,9 +115,25 @@ def _run_cmd(cmd: list[str], *, verbose: bool = False, **kwargs) -> subprocess.C — ``✓ ()`` — and full detail on failure. * Pass ``verbose=True`` to force full output on both paths (useful when the command's output is itself the test signal). + + For ``uv`` subprocess calls, ``UV_EXCLUDE_NEWER`` is scrubbed from + the environment. The runner workflow sets that variable to pin the + runner's own deps, but it leaks into every ``uv`` invocation — and + when it applies to a template subdir's ``uv run`` / ``uv sync``, uv + rewrites the template's ``uv.lock`` with ``excluded-newer`` baked + into its metadata. Bundle deploy then uploads that contaminated lock + and the Apps runtime ``uv sync --locked`` rejects it because the + deploy environment doesn't have the cutoff. End users running + ``bundle deploy`` from their own shell don't have the env var + either, so scrubbing it for template-side uv calls makes the test + match the end-user flow. """ kwargs.setdefault("capture_output", True) kwargs.setdefault("text", True) + if cmd and cmd[0] == "uv": + env = kwargs.get("env") or os.environ.copy() + env.pop("UV_EXCLUDE_NEWER", None) + kwargs["env"] = env cmd_str = " ".join(cmd) short_cmd = " ".join(cmd[:3]) + ("..." if len(cmd) > 3 else "") t0 = time.monotonic() @@ -263,22 +279,11 @@ def clean_template(template_dir: Path): def uv_sync(template_dir: Path, max_attempts: int = 3): """Run `uv sync --frozen` to create the venv from the checked-in lockfile. - `--frozen` matters: the runner workflow sets `UV_EXCLUDE_NEWER` to pin - third-party churn in the runner's own deps. That env var also leaks - into `uv sync` invocations in the template subdir — uv re-resolves - under the cutoff and rewrites the template's `uv.lock` with - `excluded-newer` baked into its metadata. Bundle deploy then uploads - the contaminated lockfile, and the Databricks Apps runtime — which - runs `uv sync --locked` *without* the env var — detects the cutoff - was removed, ignores the lock, re-resolves to a different package - set, and fails: - Ignoring existing lockfile due to removal of global exclude newer - The lockfile at `uv.lock` needs to be updated, but `--locked` - was provided. - `--frozen` installs straight from the existing lock without - re-resolving, leaving the on-disk lockfile byte-identical so deploy - uploads the version the template's authors checked in (matching the - end-user `bundle deploy` flow exactly). + `--frozen` belt-and-suspenders: the underlying scrub of + ``UV_EXCLUDE_NEWER`` in ``_run_cmd`` already prevents the template's + lockfile from being mutated, but ``--frozen`` makes it impossible + even if a future change forgets to scrub — uv refuses to update the + lock and errors loudly instead of silently rewriting it. Retries up to ``max_attempts`` times with a short backoff to absorb transient PyPI / proxy hiccups, then falls back to UV_OFFLINE=true From c0b5df17e4d2ee53dc64b8c4c48a7d66d80ed06c Mon Sep 17 00:00:00 2001 From: Dhruv Gupta Date: Thu, 30 Apr 2026 04:49:25 +0000 Subject: [PATCH 3/5] Scrub UV_EXCLUDE_NEWER at module-import time (not per uv call) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iterating again. CI run 25147595689 confirmed the previous fix was incomplete: scrubbing UV_EXCLUDE_NEWER in `_run_cmd` only covered uv calls that went through that helper. `_start_server_once` uses `subprocess.Popen` directly with `["uv", "run", "start-server", ...]` and bypasses _run_cmd, so the local-server thread inherited the workflow's env unchanged and re-synced the lockfile in parallel with the deploy thread. Deploy then uploaded the freshly-contaminated lock and Apps runtime rejected it identically. Move the scrub to module-import time: `os.environ.pop("UV_EXCLUDE_NEWER", None)` at the top of helpers.py. Every subprocess pytest spawns from that point on inherits the cleaned env, regardless of whether it goes through _run_cmd, subprocess.Popen, or anywhere else. Safe to do at import time: pytest imports helpers AFTER the workflow's "Sync test dependencies" step has already run, so the runner's own deps were pinned with UV_EXCLUDE_NEWER applied. From this point on the env var only mattered as accidental contamination of template-side calls — which is exactly what we don't want. Also drop the --frozen flag from uv_sync. With the env var scrubbed, uv has no cutoff to bake in, so the lockfile stays clean naturally. The --frozen / scrub-in-_run_cmd combination was inconsistent (--frozen only on uv_sync, not on the dozen `uv run` calls elsewhere); the import-time scrub covers them all uniformly. Co-authored-by: Isaac --- .scripts/agent-integration-tests/helpers.py | 52 +++++++++++---------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/.scripts/agent-integration-tests/helpers.py b/.scripts/agent-integration-tests/helpers.py index 7bfb9a9e..4da56e4a 100644 --- a/.scripts/agent-integration-tests/helpers.py +++ b/.scripts/agent-integration-tests/helpers.py @@ -16,6 +16,30 @@ from databricks_ai_bridge.lakebase import LakebaseClient from template_config import FileEdit +# --------------------------------------------------------------------------- +# Environment scrub: UV_EXCLUDE_NEWER +# --------------------------------------------------------------------------- +# The runner workflow sets UV_EXCLUDE_NEWER workflow-wide to pin the +# runner's *own* deps to a fixed release date (the workflow's "Sync +# test dependencies" step is where that's actually meant to apply). +# But the variable leaks into every subprocess pytest spawns — and any +# `uv run` / `uv sync` invocation against a template subdir then +# re-resolves under the cutoff and rewrites the template's `uv.lock` +# with `excluded-newer` baked into its metadata. Bundle deploy uploads +# the contaminated lock; the Databricks Apps runtime runs +# `uv sync --locked` *without* the env var, sees the cutoff was +# removed, ignores the lock, and fails: +# Ignoring existing lockfile due to removal of global exclude newer +# The lockfile at `uv.lock` needs to be updated, but `--locked` +# was provided. +# End users running `bundle deploy` from their own shell don't have the +# env var either, so scrubbing it from this process's environment makes +# every uv subprocess (whether started via _run_cmd, subprocess.Popen +# in start_server, or anywhere else) behave like the end-user flow. +# Safe to do at import time: pytest imports helpers AFTER the workflow's +# "Sync test dependencies" step has already run. +os.environ.pop("UV_EXCLUDE_NEWER", None) + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -115,25 +139,9 @@ def _run_cmd(cmd: list[str], *, verbose: bool = False, **kwargs) -> subprocess.C — ``✓ ()`` — and full detail on failure. * Pass ``verbose=True`` to force full output on both paths (useful when the command's output is itself the test signal). - - For ``uv`` subprocess calls, ``UV_EXCLUDE_NEWER`` is scrubbed from - the environment. The runner workflow sets that variable to pin the - runner's own deps, but it leaks into every ``uv`` invocation — and - when it applies to a template subdir's ``uv run`` / ``uv sync``, uv - rewrites the template's ``uv.lock`` with ``excluded-newer`` baked - into its metadata. Bundle deploy then uploads that contaminated lock - and the Apps runtime ``uv sync --locked`` rejects it because the - deploy environment doesn't have the cutoff. End users running - ``bundle deploy`` from their own shell don't have the env var - either, so scrubbing it for template-side uv calls makes the test - match the end-user flow. """ kwargs.setdefault("capture_output", True) kwargs.setdefault("text", True) - if cmd and cmd[0] == "uv": - env = kwargs.get("env") or os.environ.copy() - env.pop("UV_EXCLUDE_NEWER", None) - kwargs["env"] = env cmd_str = " ".join(cmd) short_cmd = " ".join(cmd[:3]) + ("..." if len(cmd) > 3 else "") t0 = time.monotonic() @@ -277,20 +285,14 @@ def clean_template(template_dir: Path): def uv_sync(template_dir: Path, max_attempts: int = 3): - """Run `uv sync --frozen` to create the venv from the checked-in lockfile. - - `--frozen` belt-and-suspenders: the underlying scrub of - ``UV_EXCLUDE_NEWER`` in ``_run_cmd`` already prevents the template's - lockfile from being mutated, but ``--frozen`` makes it impossible - even if a future change forgets to scrub — uv refuses to update the - lock and errors loudly instead of silently rewriting it. + """Run `uv sync` to create/update the venv before quickstart. Retries up to ``max_attempts`` times with a short backoff to absorb transient PyPI / proxy hiccups, then falls back to UV_OFFLINE=true one more time as a last resort. """ for attempt in range(1, max_attempts + 1): - result = _run_cmd(["uv", "sync", "--frozen"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT) + result = _run_cmd(["uv", "sync"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT) if result.returncode == 0: return if attempt < max_attempts: @@ -300,7 +302,7 @@ def uv_sync(template_dir: Path, max_attempts: int = 3): _log(f" uv sync failed online; falling back to UV_OFFLINE=true (cache-only)...") env = os.environ.copy() env["UV_OFFLINE"] = "true" - result = _run_cmd(["uv", "sync", "--frozen"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT, env=env) + result = _run_cmd(["uv", "sync"], cwd=template_dir, timeout=QUICKSTART_TIMEOUT, env=env) assert result.returncode == 0, ( f"uv sync failed in {template_dir.name}:\n" f"stdout: {result.stdout}\n" From 9860586582b75da246ba694d0ca3fd0ab41f9364 Mon Sep 17 00:00:00 2001 From: Dhruv Gupta Date: Thu, 30 Apr 2026 05:03:16 +0000 Subject: [PATCH 4/5] Also scrub UV_CONFIG_FILE + set UV_NO_CONFIG=1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous scrub of UV_EXCLUDE_NEWER alone wasn't sufficient. CI run 25147991251 deploy logs show two distinct contamination messages, not one: [BUILD] Ignoring existing lockfile due to removal of global exclude newer [BUILD] Ignoring existing lockfile due to removal of exclude newer for package `databricks-openai` The first is UV_EXCLUDE_NEWER (already scrubbed). The second comes from the runner repo's `uv.toml` `[exclude-newer-package]` overrides for databricks-openai / databricks-ai-bridge / databricks-langchain / databricks-agents. uv discovers that config file by walking up from the template subdir to the workspace root — so popping UV_CONFIG_FILE alone doesn't help. Fix: pop UV_CONFIG_FILE *and* set UV_NO_CONFIG=1 to disable walk-up discovery entirely. Now both contamination sources are blocked at the process level for any subprocess helpers.py spawns, and the template's checked-in lockfile reaches the Apps runtime byte-identical to what Bryan committed in #201. Co-authored-by: Isaac --- .scripts/agent-integration-tests/helpers.py | 56 +++++++++++++-------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/.scripts/agent-integration-tests/helpers.py b/.scripts/agent-integration-tests/helpers.py index 4da56e4a..1c2ed20b 100644 --- a/.scripts/agent-integration-tests/helpers.py +++ b/.scripts/agent-integration-tests/helpers.py @@ -17,28 +17,44 @@ from template_config import FileEdit # --------------------------------------------------------------------------- -# Environment scrub: UV_EXCLUDE_NEWER +# Environment scrub: prevent uv from contaminating the template's uv.lock # --------------------------------------------------------------------------- -# The runner workflow sets UV_EXCLUDE_NEWER workflow-wide to pin the -# runner's *own* deps to a fixed release date (the workflow's "Sync -# test dependencies" step is where that's actually meant to apply). -# But the variable leaks into every subprocess pytest spawns — and any -# `uv run` / `uv sync` invocation against a template subdir then -# re-resolves under the cutoff and rewrites the template's `uv.lock` -# with `excluded-newer` baked into its metadata. Bundle deploy uploads -# the contaminated lock; the Databricks Apps runtime runs -# `uv sync --locked` *without* the env var, sees the cutoff was -# removed, ignores the lock, and fails: -# Ignoring existing lockfile due to removal of global exclude newer -# The lockfile at `uv.lock` needs to be updated, but `--locked` -# was provided. -# End users running `bundle deploy` from their own shell don't have the -# env var either, so scrubbing it from this process's environment makes -# every uv subprocess (whether started via _run_cmd, subprocess.Popen -# in start_server, or anywhere else) behave like the end-user flow. -# Safe to do at import time: pytest imports helpers AFTER the workflow's -# "Sync test dependencies" step has already run. +# Two pieces of runner config leak into template uv invocations and +# rewrite the template's `uv.lock` with resolution-context metadata +# the Databricks Apps build environment doesn't share: +# +# 1. UV_EXCLUDE_NEWER (set workflow-wide) — pins the runner's *own* +# deps to a fixed release date. When applied to a template subdir's +# `uv run` / `uv sync`, uv re-resolves under the cutoff and bakes +# `excluded-newer` into the lock metadata. Apps runtime then runs +# `uv sync --locked` without the env var, sees: +# Ignoring existing lockfile due to removal of global exclude newer +# The lockfile at `uv.lock` needs to be updated, but `--locked` +# was provided. +# +# 2. The runner repo's `uv.toml` (referenced by UV_CONFIG_FILE, but +# also auto-discovered by uv walking up from the template subdir +# to the workspace root). It defines per-package exclude-newer +# overrides for databricks-* packages. Same failure mode, different +# message: +# Ignoring existing lockfile due to removal of exclude newer +# for package `databricks-openai`. +# +# End users running `bundle deploy` from their own shell don't have any +# of this configured, so we strip both at import time: +# * pop UV_EXCLUDE_NEWER and UV_CONFIG_FILE. +# * set UV_NO_CONFIG=1 to disable uv's walk-up discovery of the +# runner repo's `uv.toml` (popping UV_CONFIG_FILE alone isn't +# enough — uv finds the file via cwd ancestry). +# Every subprocess pytest spawns (whether via _run_cmd, subprocess.Popen +# in start_server, or anywhere else) inherits these scrubs. +# +# Safe at import time: pytest imports helpers AFTER the workflow's +# "Sync test dependencies" step has already run with full config +# applied. os.environ.pop("UV_EXCLUDE_NEWER", None) +os.environ.pop("UV_CONFIG_FILE", None) +os.environ["UV_NO_CONFIG"] = "1" # --------------------------------------------------------------------------- # Constants From a1323e863b1db68b68bcbcc3c7af6d7f1f7fb8fc Mon Sep 17 00:00:00 2001 From: Dhruv Gupta Date: Thu, 30 Apr 2026 17:35:11 +0000 Subject: [PATCH 5/5] Shorten in-code env-scrub comment; full rationale in PR --- .scripts/agent-integration-tests/helpers.py | 40 +++------------------ 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/.scripts/agent-integration-tests/helpers.py b/.scripts/agent-integration-tests/helpers.py index 1c2ed20b..af428b88 100644 --- a/.scripts/agent-integration-tests/helpers.py +++ b/.scripts/agent-integration-tests/helpers.py @@ -16,42 +16,10 @@ from databricks_ai_bridge.lakebase import LakebaseClient from template_config import FileEdit -# --------------------------------------------------------------------------- -# Environment scrub: prevent uv from contaminating the template's uv.lock -# --------------------------------------------------------------------------- -# Two pieces of runner config leak into template uv invocations and -# rewrite the template's `uv.lock` with resolution-context metadata -# the Databricks Apps build environment doesn't share: -# -# 1. UV_EXCLUDE_NEWER (set workflow-wide) — pins the runner's *own* -# deps to a fixed release date. When applied to a template subdir's -# `uv run` / `uv sync`, uv re-resolves under the cutoff and bakes -# `excluded-newer` into the lock metadata. Apps runtime then runs -# `uv sync --locked` without the env var, sees: -# Ignoring existing lockfile due to removal of global exclude newer -# The lockfile at `uv.lock` needs to be updated, but `--locked` -# was provided. -# -# 2. The runner repo's `uv.toml` (referenced by UV_CONFIG_FILE, but -# also auto-discovered by uv walking up from the template subdir -# to the workspace root). It defines per-package exclude-newer -# overrides for databricks-* packages. Same failure mode, different -# message: -# Ignoring existing lockfile due to removal of exclude newer -# for package `databricks-openai`. -# -# End users running `bundle deploy` from their own shell don't have any -# of this configured, so we strip both at import time: -# * pop UV_EXCLUDE_NEWER and UV_CONFIG_FILE. -# * set UV_NO_CONFIG=1 to disable uv's walk-up discovery of the -# runner repo's `uv.toml` (popping UV_CONFIG_FILE alone isn't -# enough — uv finds the file via cwd ancestry). -# Every subprocess pytest spawns (whether via _run_cmd, subprocess.Popen -# in start_server, or anywhere else) inherits these scrubs. -# -# Safe at import time: pytest imports helpers AFTER the workflow's -# "Sync test dependencies" step has already run with full config -# applied. +# Strip runner-specific uv config so template-side `uv run` / `uv sync` +# don't bake `excluded-newer` (global + per-package) into the template's +# `uv.lock`, which would then be rejected by Apps runtime's +# `uv sync --locked`. See databricks/app-templates#206 for full rationale. os.environ.pop("UV_EXCLUDE_NEWER", None) os.environ.pop("UV_CONFIG_FILE", None) os.environ["UV_NO_CONFIG"] = "1"