diff --git a/.ai/active/SPRINT_PACKET.md b/.ai/active/SPRINT_PACKET.md index 87b7769..e32ab02 100644 --- a/.ai/active/SPRINT_PACKET.md +++ b/.ai/active/SPRINT_PACKET.md @@ -2,7 +2,7 @@ ## Sprint Title -Sprint 5I: Compile-Path Semantic Artifact Retrieval Adoption +Sprint 5J: Deterministic Hybrid Artifact Merge in Context Compilation ## Sprint Type @@ -10,15 +10,15 @@ feature ## Sprint Reason -Milestone 5 now has deterministic lexical artifact retrieval in the compile path and a separate direct semantic artifact retrieval primitive over durable chunk embeddings. The next safe step is to adopt semantic artifact retrieval into the compiler as a separate context section, while still deferring hybrid lexical-plus-semantic artifact merging, reranking, connectors, and UI. +Milestone 5 now has compile-path lexical artifact retrieval and compile-path semantic artifact retrieval as separate sections. The next safe step is to merge those two existing artifact candidate paths into one governed compile-time artifact section with explicit deterministic deduplication and provenance rules, while still deferring reranking, connectors, and UI. ## Sprint Intent -Extend the existing context-compile path so it can optionally retrieve semantic artifact chunks using the shipped `task_artifact_chunk_embeddings` retrieval primitive, while keeping semantic artifact results separate from lexical artifact retrieval and deferring hybrid artifact fusion. +Complete the first hybrid artifact retrieval slice by merging the already-implemented lexical and semantic artifact chunk candidate sets inside `POST /v0/context/compile`, using explicit deterministic deduplication and selection rules without adding reranking or model-driven embedding generation. ## Git Instructions -- Branch Name: `codex/sprint-5i-compile-semantic-artifact-retrieval` +- Branch Name: `codex/sprint-5j-hybrid-artifact-merge` - Base Branch: `main` - PR Strategy: one sprint branch, one PR, no stacked PRs unless Control Tower explicitly opens a follow-up sprint - Merge Policy: squash merge only after reviewer `PASS` and explicit Control Tower merge approval @@ -28,45 +28,50 @@ Extend the existing context-compile path so it can optionally retrieve semantic - Sprint 5A shipped deterministic rooted task-workspace provisioning. - Sprint 5C shipped explicit task-artifact registration. - Sprint 5D shipped deterministic local artifact ingestion into durable chunk rows. -- Sprint 5E shipped deterministic lexical artifact-chunk retrieval. +- Sprint 5E shipped lexical artifact-chunk retrieval. - Sprint 5F shipped compile-path lexical artifact chunk inclusion. - Sprint 5G shipped durable artifact-chunk embedding persistence. -- Sprint 5H shipped direct semantic artifact retrieval over those durable chunk embeddings. -- The next narrow Milestone 5 seam is compile-path adoption of semantic artifact retrieval only, so later hybrid artifact retrieval can build on an explicit compile-time semantic section instead of collapsing lexical and semantic behavior in one sprint. +- Sprint 5H shipped direct semantic artifact retrieval. +- Sprint 5I shipped compile-path semantic artifact retrieval as a separate context section. +- The next narrow Milestone 5 seam is deterministic hybrid artifact merge, so compile can return one governed artifact section before any reranking or richer document behavior is introduced. ## In Scope - Define typed contracts for: - - optional semantic artifact retrieval input on compile requests - - semantic artifact chunk result items inside the compiled context pack - - semantic artifact retrieval summary metadata inside compile responses - - semantic artifact retrieval trace payloads + - hybrid artifact chunk items in the compiled context pack + - source provenance metadata for each included artifact chunk + - hybrid artifact retrieval summary metadata + - hybrid artifact retrieval trace payloads - Extend the compile path so it can: - - accept an explicit semantic artifact retrieval request scoped to one visible task or one visible artifact - - accept an explicit `embedding_config_id` and caller-supplied query vector - - reuse the existing semantic artifact retrieval primitive during compile - - include semantic artifact chunks in a separate context-pack section - - record semantic artifact include/exclude decisions in `trace_events` - - preserve deterministic output for the same stored data and inputs + - gather the existing lexical artifact chunk candidates + - optionally gather the existing semantic artifact chunk candidates + - merge both candidate sets into one artifact section + - deduplicate by durable artifact chunk identity + - preserve source provenance when a chunk is selected by both paths + - apply explicit deterministic selection rules and limits + - record hybrid include/exclude and deduplication decisions in `trace_events` +- Define explicit merge behavior, for example: + - lexical-first precedence when both sources compete for the same limit budget + - stable tie-breaking after source precedence + - predictable handling when a chunk appears in both candidate sets - Ensure compile behavior: - - leaves current continuity, memory, entity, lexical artifact, and other context sections intact - - does not merge lexical and semantic artifact sections - excludes non-ingested artifacts - scopes strictly by user ownership - - uses deterministic ordering and explicit per-section limits + - remains deterministic for the same stored data and inputs + - leaves memory, entity, and non-artifact sections unchanged - Add unit and integration tests for: - - compile request validation for semantic artifact retrieval input - - deterministic semantic artifact section ordering + - deterministic merge ordering + - deduplication behavior + - dual-source provenance behavior + - limit enforcement across merged artifact candidates - exclusion of non-ingested artifacts - - trace logging for included and excluded semantic artifact chunks - per-user isolation through the compile path - - response-shape stability for the new semantic artifact section + - response-shape stability for the merged artifact section ## Out of Scope -- No hybrid lexical plus semantic artifact retrieval. -- No reranking across semantic artifact chunks. -- No lexical-plus-semantic deduplication or fusion. +- No reranking across merged artifact candidates. +- No weighted or learned fusion logic. - No model or external API calls to generate query embeddings. - No richer document parsing beyond the already-shipped local text ingestion seam. - No Gmail or Calendar connector scope. @@ -75,61 +80,61 @@ Extend the existing context-compile path so it can optionally retrieve semantic ## Required Deliverables -- Stable compile-request and compile-response contract updates for semantic artifact retrieval input and output. -- Compile-path integration with the existing semantic artifact retrieval primitive. -- Trace coverage for semantic artifact retrieval decisions inside compile runs. -- Unit and integration coverage for compile-path semantic artifact behavior, ordering, exclusion rules, validation, and isolation. +- Stable compile-response contract updates for merged hybrid artifact output. +- Deterministic hybrid merge logic over the existing lexical and semantic artifact retrieval paths. +- Trace coverage for merge, deduplication, and exclusion decisions. +- Unit and integration coverage for hybrid artifact behavior, ordering, validation, and isolation. - Updated `BUILD_REPORT.md` with exact verification results and explicit deferred scope. ## Acceptance Criteria -- `POST /v0/context/compile` can optionally accept semantic artifact retrieval input and return a separate semantic artifact chunk section in the context pack. -- Compile-path semantic artifact retrieval uses only durable `task_artifact_chunk_embeddings`, `task_artifact_chunks`, and artifact records already persisted in the repo. -- Compile-path semantic artifact retrieval rejects missing configs, dimension mismatches, and cross-user access deterministically. -- Non-ingested artifacts are excluded from semantic artifact compile results. -- Semantic artifact include/exclude decisions are persisted in `trace_events`. -- Result ordering is deterministic within the semantic artifact section. +- `POST /v0/context/compile` can return one merged artifact section derived from the existing lexical and semantic artifact retrieval paths. +- The merged section deduplicates artifact chunks by durable identity and preserves source provenance. +- Merge behavior uses explicit deterministic rules and limits. +- Non-ingested artifacts are excluded from the merged section. +- Hybrid artifact merge decisions are persisted in `trace_events`. +- Result ordering is deterministic for the same stored data and inputs. - `./.venv/bin/python -m pytest tests/unit` passes. - `./.venv/bin/python -m pytest tests/integration` passes. -- No hybrid retrieval, reranking, connector, runner, UI, or broader side-effect scope enters the sprint. +- No reranking, connector, runner, UI, or broader side-effect scope enters the sprint. ## Implementation Constraints -- Keep compile-path adoption narrow and boring. -- Reuse the existing semantic artifact retrieval primitive; do not read raw files during compile. -- Keep semantic artifact chunks in a separate response section from lexical artifact chunks and from memory/entity context. -- Require explicit semantic artifact input; do not auto-enable semantic retrieval. -- Do not merge semantic and lexical artifact retrieval in the same sprint. +- Keep hybrid artifact behavior narrow and boring. +- Reuse the existing lexical and semantic artifact retrieval seams rather than introducing a third retrieval stack. +- Make source precedence explicit in contracts, code, and tests. +- Do not introduce weighted scoring, reranking, or learned fusion in this sprint. +- Keep memory, entity, and non-artifact sections unchanged. ## Suggested Work Breakdown -1. Define compile contract updates for optional semantic artifact retrieval input and output. -2. Integrate the existing semantic artifact retrieval primitive into the compile path. -3. Add semantic artifact result summaries and trace-event payloads. -4. Preserve current context sections while adding a separate semantic artifact section. -5. Add unit and integration tests. -6. Update `BUILD_REPORT.md` with executed verification. +1. Define hybrid artifact output and trace contracts. +2. Implement deterministic merge and deduplication over existing lexical and semantic artifact candidates. +3. Add source provenance and hybrid summary metadata. +4. Record hybrid merge decisions in `trace_events`. +5. Preserve existing retrieval seams while returning one merged artifact section. +6. Add unit and integration tests. +7. Update `BUILD_REPORT.md` with executed verification. ## Build Report Requirements `BUILD_REPORT.md` must include: -- the exact compile contract changes introduced -- the semantic artifact similarity metric and ordering rule used +- the exact hybrid artifact merge contract changes introduced +- the merge precedence and deduplication rule used - exact commands run - unit and integration test results -- one example compile request and response showing the semantic artifact section -- one example of semantic artifact retrieval trace events inside one compile run +- one example compile request and response showing merged artifact output +- one example of hybrid artifact retrieval trace events inside one compile run - what remains intentionally deferred to later milestones ## Review Focus `REVIEW_REPORT.md` should verify: -- the sprint stayed limited to compile-path semantic artifact retrieval adoption -- semantic artifact retrieval is explicit-input, durable-source-only, and validation-backed -- lexical and semantic artifact results remain separate rather than merged -- ordering, exclusion rules, trace visibility, and isolation are test-backed -- no hidden hybrid retrieval, reranking, connector, runner, UI, or broader side-effect scope entered the sprint +- the sprint stayed limited to deterministic hybrid artifact merge in the compile path +- hybrid artifact behavior reuses the existing lexical and semantic retrieval seams +- merge ordering, deduplication, provenance, exclusion rules, trace visibility, and isolation are deterministic and test-backed +- no hidden reranking, connector, runner, UI, or broader side-effect scope entered the sprint ## Exit Condition -This sprint is complete when the repo can optionally include semantic artifact chunks inside `POST /v0/context/compile`, trace those semantic inclusion decisions, and verify the full path with Postgres-backed tests, while still deferring hybrid artifact retrieval, reranking, connectors, and UI. +This sprint is complete when the repo can return one deterministic merged artifact section in `POST /v0/context/compile`, derived from the existing lexical and semantic artifact retrieval paths with trace-visible merge decisions and passing Postgres-backed tests, while still deferring reranking, connectors, and UI. diff --git a/BUILD_REPORT.md b/BUILD_REPORT.md index d3bde52..b8ce9ba 100644 --- a/BUILD_REPORT.md +++ b/BUILD_REPORT.md @@ -2,93 +2,92 @@ ## sprint objective -Implement Sprint 5I: adopt semantic artifact retrieval into `POST /v0/context/compile` as an explicit, separate compile-time section backed only by durable `task_artifact_chunk_embeddings`, `task_artifact_chunks`, and `task_artifacts`, while keeping lexical and semantic artifact retrieval separate. +Implement Sprint 5J: merge the existing lexical and semantic artifact chunk retrieval paths inside `POST /v0/context/compile` into one deterministic hybrid artifact section with explicit provenance, deduplication, limits, and trace visibility. ## completed work -- Added compile-request contracts for `semantic_artifact_retrieval` with explicit task-scoped and artifact-scoped variants: - - `kind` - - `task_id` or `task_artifact_id` - - `embedding_config_id` - - `query_vector` - - `limit` -- Added compile-response contracts for: - - `context_pack.semantic_artifact_chunks` - - `context_pack.semantic_artifact_chunk_summary` -- Added semantic artifact trace contracts for per-item include/exclude decisions with: - - scope - - artifact identity - - ingestion status - - embedding config id - - query vector dimensions - - limit - - similarity metric - - score and chunk coordinates when applicable -- Integrated semantic artifact retrieval into the compiler as an explicit optional path. -- Reused the shipped semantic artifact retrieval primitive for compile-path section assembly, then evaluated the full deterministic candidate set for compile-only include/exclude tracing and counts. -- Preserved existing compile sections and behavior for: - - continuity scope - - hybrid memory - - lexical artifact retrieval - - entities - - entity edges -- Kept lexical artifact chunks and semantic artifact chunks in separate response sections. -- Added trace coverage for: - - `within_semantic_artifact_chunk_limit` - - `semantic_artifact_chunk_limit_exceeded` - - `semantic_artifact_not_ingested` -- Added summary trace fields for semantic artifact retrieval request state, scope, candidate count, included count, limit exclusions, and non-ingested exclusions. -- Updated prompt-assembly context serialization so compiled context packs include the new semantic artifact section shape. +- Replaced the compile-time split artifact output with one merged `context_pack.artifact_chunks` section. +- Added hybrid artifact contracts for: + - per-chunk source provenance + - merged artifact summary metadata + - hybrid artifact decision trace payloads +- Implemented deterministic hybrid artifact merge logic that: + - reuses the existing lexical and semantic retrieval seams + - deduplicates by durable chunk id + - preserves dual-source provenance + - applies lexical-first precedence under the shared compile output limit + - excludes non-ingested artifacts + - emits explicit dedup/include/exclude trace events +- Updated compile summary trace payloads to report hybrid artifact request state, candidate counts, dedup counts, dual-source counts, and limit exclusions. +- Updated prompt assembly to serialize only the merged compile-time artifact section. - Added unit and integration coverage for: - - request-shape validation - - config existence validation - - query-vector dimension validation - - deterministic ordering - - exclusion of non-ingested artifacts - - trace logging for included and excluded semantic artifact results + - lexical-only artifact compile behavior + - semantic-only artifact compile behavior + - hybrid dual-source provenance + - deterministic merge ordering + - limit enforcement across merged candidates + - non-ingested exclusion - per-user isolation - response-shape stability -## exact compile contract changes introduced - -- Request: - - `CompileContextRequest.semantic_artifact_retrieval` - - `CompileContextTaskScopedSemanticArtifactRetrievalRequest` - - `CompileContextArtifactScopedSemanticArtifactRetrievalRequest` - - `CompileContextTaskScopedSemanticArtifactRetrievalInput` - - `CompileContextArtifactScopedSemanticArtifactRetrievalInput` -- Response: - - `CompiledContextPack.semantic_artifact_chunks` - - `CompiledContextPack.semantic_artifact_chunk_summary` - - `ContextPackSemanticArtifactChunk` - - `ContextPackSemanticArtifactChunkSummary` -- Trace payloads: - - `SemanticArtifactRetrievalDecisionTracePayload` -- Summary event additions: - - `semantic_artifact_retrieval_requested` - - `semantic_artifact_retrieval_scope_kind` - - `semantic_artifact_chunk_candidate_count` - - `included_semantic_artifact_chunk_count` - - `excluded_semantic_artifact_chunk_limit_count` - - `excluded_semantic_uningested_artifact_count` - -## similarity metric and ordering rule used +## exact hybrid artifact merge contract changes introduced -- Similarity metric: `cosine_similarity` -- Ordering rule: `score_desc`, `relative_path_asc`, `sequence_no_asc`, `id_asc` -- Compile candidate evaluation stays deterministic by using the durable semantic retrieval ordering and then applying explicit compile-time slicing by `limit`. +- Added `ArtifactSelectionSource = Literal["lexical", "semantic"]`. +- Updated `ContextPackArtifactChunk` to carry: + - `source_provenance.sources` + - `source_provenance.lexical_match` + - `source_provenance.semantic_score` +- Expanded `ContextPackArtifactChunkSummary` to include: + - `lexical_requested` + - `semantic_requested` + - `embedding_config_id` + - `query_vector_dimensions` + - `lexical_limit` + - `semantic_limit` + - `lexical_candidate_count` + - `semantic_candidate_count` + - `merged_candidate_count` + - `deduplicated_count` + - `included_lexical_only_count` + - `included_semantic_only_count` + - `included_dual_source_count` + - `source_precedence` + - `lexical_order` + - `semantic_order` + - `merged_order` +- Added `HybridArtifactRetrievalDecisionTracePayload`. +- Removed compile-time `semantic_artifact_chunks` and `semantic_artifact_chunk_summary` from `CompiledContextPack`. + +## merge precedence and deduplication rule used + +- Dedup key: durable artifact chunk id. +- Source precedence: `lexical` before `semantic`. +- Merged ordering: + - source precedence + - lexical rank + - semantic rank + - `relative_path` + - `sequence_no` + - `id` +- Final compile limit: + - use `artifact_retrieval.limit` when lexical retrieval is present + - otherwise use `semantic_artifact_retrieval.limit` +- When the same chunk appears in both candidate sets: + - keep one item + - set `source_provenance.sources` to `["lexical", "semantic"]` + - preserve the lexical match payload + - preserve the semantic score + - emit `hybrid_artifact_chunk_deduplicated` ## incomplete work -- None within Sprint 5I scope. +- None within Sprint 5J scope. ## files changed - `apps/api/src/alicebot_api/compiler.py` - `apps/api/src/alicebot_api/contracts.py` -- `apps/api/src/alicebot_api/main.py` - `apps/api/src/alicebot_api/response_generation.py` -- `apps/api/src/alicebot_api/semantic_retrieval.py` - `tests/integration/test_context_compile.py` - `tests/unit/test_compiler.py` - `tests/unit/test_main.py` @@ -98,139 +97,133 @@ Implement Sprint 5I: adopt semantic artifact retrieval into `POST /v0/context/co ## tests run - `./.venv/bin/python -m pytest tests/unit/test_compiler.py tests/unit/test_main.py tests/unit/test_response_generation.py` - - result: `50 passed in 0.48s` -- `./.venv/bin/python -m pytest tests/integration/test_context_compile.py` - - sandboxed attempt failed because localhost Postgres access was blocked - - rerun with local DB access allowed: `11 passed in 3.85s` + - `50 passed in 0.50s` - `./.venv/bin/python -m pytest tests/unit` - - result: `380 passed in 0.61s` + - `380 passed in 0.56s` +- `./.venv/bin/python -m pytest tests/integration/test_context_compile.py` + - initial sandboxed run failed because local Postgres on `localhost:5432` was blocked - `./.venv/bin/python -m pytest tests/integration` - - result: `117 passed in 36.46s` + - `118 passed in 35.59s` ## unit and integration test results -- Unit suite status: pass -- Integration suite status: pass -- Acceptance-criteria verification: - - compile request validation: covered and passing - - deterministic semantic artifact ordering: covered and passing - - exclusion of non-ingested artifacts: covered and passing - - include/exclude trace logging: covered and passing - - per-user isolation: covered and passing - - response-shape stability: covered and passing - -## example compile request - -```json -{ - "user_id": "11111111-1111-1111-8111-111111111111", - "thread_id": "22222222-2222-2222-8222-222222222222", - "semantic_artifact_retrieval": { - "kind": "task", - "task_id": "33333333-3333-3333-8333-333333333333", - "embedding_config_id": "44444444-4444-4444-8444-444444444444", - "query_vector": [1.0, 0.0, 0.0], - "limit": 2 - } -} -``` +- Unit suite: pass +- Integration suite: pass -## example compile response showing semantic artifact section +## example compile request and merged response ```json { - "context_pack": { - "semantic_artifact_chunks": [ - { - "id": "55555555-5555-5555-8555-555555555555", - "task_id": "33333333-3333-3333-8333-333333333333", - "task_artifact_id": "66666666-6666-6666-8666-666666666666", - "relative_path": "docs/a.txt", - "media_type": "text/plain", - "sequence_no": 1, - "char_start": 0, - "char_end_exclusive": 14, - "text": "beta alpha doc", - "score": 1.0 - }, - { - "id": "77777777-7777-7777-8777-777777777777", - "task_id": "33333333-3333-3333-8333-333333333333", - "task_artifact_id": "88888888-8888-8888-8888-888888888888", - "relative_path": "notes/b.md", - "media_type": "text/markdown", - "sequence_no": 1, - "char_start": 0, - "char_end_exclusive": 15, - "text": "alpha beta note", - "score": 1.0 - } - ], - "semantic_artifact_chunk_summary": { - "requested": true, - "scope": { - "kind": "task", - "task_id": "33333333-3333-3333-8333-333333333333" - }, + "request": { + "user_id": "11111111-1111-1111-8111-111111111111", + "thread_id": "22222222-2222-2222-8222-222222222222", + "artifact_retrieval": { + "kind": "task", + "task_id": "33333333-3333-3333-8333-333333333333", + "query": "Alpha beta", + "limit": 2 + }, + "semantic_artifact_retrieval": { + "kind": "task", + "task_id": "33333333-3333-3333-8333-333333333333", "embedding_config_id": "44444444-4444-4444-8444-444444444444", - "query_vector_dimensions": 3, - "limit": 2, - "searched_artifact_count": 3, - "candidate_count": 3, - "included_count": 2, - "excluded_uningested_artifact_count": 1, - "excluded_limit_count": 1, - "similarity_metric": "cosine_similarity", - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"] + "query_vector": [1.0, 0.0, 0.0], + "limit": 2 + } + }, + "response": { + "context_pack": { + "artifact_chunks": [ + { + "id": "55555555-5555-5555-8555-555555555555", + "task_id": "33333333-3333-3333-8333-333333333333", + "task_artifact_id": "66666666-6666-6666-8666-666666666666", + "relative_path": "docs/a.txt", + "media_type": "text/plain", + "sequence_no": 1, + "char_start": 0, + "char_end_exclusive": 14, + "text": "beta alpha doc", + "source_provenance": { + "sources": ["lexical", "semantic"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0 + }, + "semantic_score": 1.0 + } + } + ], + "artifact_chunk_summary": { + "requested": true, + "lexical_requested": true, + "semantic_requested": true, + "scope": { + "kind": "task", + "task_id": "33333333-3333-3333-8333-333333333333" + }, + "query": "Alpha beta", + "query_terms": ["alpha", "beta"], + "embedding_config_id": "44444444-4444-4444-8444-444444444444", + "query_vector_dimensions": 3, + "limit": 2, + "lexical_limit": 2, + "semantic_limit": 2, + "searched_artifact_count": 3, + "lexical_candidate_count": 3, + "semantic_candidate_count": 3, + "merged_candidate_count": 3, + "deduplicated_count": 3, + "included_count": 2, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 2, + "excluded_uningested_artifact_count": 1, + "excluded_limit_count": 1, + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "similarity_metric": "cosine_similarity" + } } } } ``` -## example semantic artifact retrieval trace events inside one compile run +## example hybrid artifact trace events inside one compile run ```json [ { "kind": "context.included", "payload": { - "entity_type": "semantic_artifact_chunk", + "entity_type": "artifact_chunk", "entity_id": "55555555-5555-5555-8555-555555555555", - "reason": "within_semantic_artifact_chunk_limit", + "reason": "hybrid_artifact_chunk_deduplicated", "position": 1, "scope_kind": "task", "task_id": "33333333-3333-3333-8333-333333333333", "task_artifact_id": "66666666-6666-6666-8666-666666666666", "relative_path": "docs/a.txt", - "media_type": "text/plain", "ingestion_status": "ingested", - "embedding_config_id": "44444444-4444-4444-8444-444444444444", - "query_vector_dimensions": 3, - "limit": 2, - "similarity_metric": "cosine_similarity", - "score": 1.0, - "sequence_no": 1, - "char_start": 0, - "char_end_exclusive": 14 + "selected_sources": ["lexical", "semantic"], + "matched_query_terms": ["alpha", "beta"], + "score": 1.0 } }, { "kind": "context.excluded", "payload": { - "entity_type": "task_artifact", - "entity_id": "99999999-9999-9999-8999-999999999999", - "reason": "semantic_artifact_not_ingested", + "entity_type": "artifact_chunk", + "entity_id": "77777777-7777-7777-8777-777777777777", + "reason": "hybrid_artifact_chunk_limit_exceeded", "position": 3, "scope_kind": "task", "task_id": "33333333-3333-3333-8333-333333333333", - "task_artifact_id": "99999999-9999-9999-8999-999999999999", - "relative_path": "notes/hidden.txt", - "media_type": "text/plain", - "ingestion_status": "pending", - "embedding_config_id": "44444444-4444-4444-8444-444444444444", - "query_vector_dimensions": 3, - "limit": 2, - "similarity_metric": "cosine_similarity" + "task_artifact_id": "88888888-8888-8888-8888-888888888888", + "relative_path": "notes/c.txt", + "ingestion_status": "ingested", + "selected_sources": ["lexical", "semantic"], + "score": 0.0 } } ] @@ -238,19 +231,19 @@ Implement Sprint 5I: adopt semantic artifact retrieval into `POST /v0/context/co ## blockers/issues -- No implementation blockers remained. -- Integration verification required local Postgres access outside the default sandbox because sandboxed TCP access to `localhost:5432` is not permitted. +- No product blocker. +- Integration tests required elevated localhost access because the sandbox blocked Postgres connections to `localhost:5432`. ## what remains intentionally deferred to later milestones -- hybrid lexical-plus-semantic artifact retrieval -- lexical/semantic deduplication or fusion -- reranking across semantic artifact chunks -- model-generated query embeddings -- connectors -- runner orchestration -- UI work +- Reranking across lexical and semantic artifact candidates +- Weighted or learned fusion +- Model-generated query embeddings +- Connector-backed artifact retrieval +- Richer document parsing +- Runner orchestration +- UI changes ## recommended next step -Implement the follow-up sprint for hybrid compile-path artifact fusion only after agreeing on explicit merge, deduplication, and reranking rules between lexical and semantic artifact sections. +Run review against the new merged compile artifact contract, with attention on downstream consumers that may still expect the removed compile-time `semantic_artifact_chunks` section. diff --git a/REVIEW_REPORT.md b/REVIEW_REPORT.md index ca4f29b..d487d25 100644 --- a/REVIEW_REPORT.md +++ b/REVIEW_REPORT.md @@ -1,48 +1,36 @@ -# REVIEW_REPORT - -## verdict - -PASS - -## criteria met - -- `POST /v0/context/compile` now accepts optional `semantic_artifact_retrieval` input and returns a separate `context_pack.semantic_artifact_chunks` section plus `semantic_artifact_chunk_summary`. -- Compile-path semantic artifact retrieval is backed by durable artifact chunk embedding tables and does not read raw files during compile. -- Validation is enforced for missing embedding configs, query-vector dimension mismatches, and cross-user access. -- Semantic artifact chunks remain separate from lexical artifact chunks and existing memory/entity sections. -- Non-ingested artifacts are excluded from semantic artifact compile results and exclusion decisions are traced. -- Include/exclude decisions for semantic artifact retrieval are persisted in `trace_events`. -- Ordering is deterministic and matches the documented semantic retrieval order: `score_desc`, `relative_path_asc`, `sequence_no_asc`, `id_asc`. -- Scope stayed within the sprint packet: no hybrid lexical/semantic fusion, reranking, connector work, runner orchestration, or UI work was introduced. -- `BUILD_REPORT.md` was updated with contract changes, verification commands, examples, and deferred scope. -- Review verification passed: - - `./.venv/bin/python -m pytest tests/unit` -> `380 passed` - - `./.venv/bin/python -m pytest tests/integration` -> `117 passed` - -## criteria missed - +verdict: PASS + +criteria met +- `POST /v0/context/compile` now returns one merged `context_pack.artifact_chunks` section and no longer emits a separate compile-time semantic artifact section. +- The merged artifact section deduplicates by durable chunk id and preserves dual-source provenance through `source_provenance.sources`, `lexical_match`, and `semantic_score`. +- Merge behavior is explicit and deterministic: lexical-first precedence, stable tie-breakers, shared final limit handling, and deterministic trace ordering are implemented in [compiler.py](/Users/samirusani/Desktop/Codex/AliceBot/apps/api/src/alicebot_api/compiler.py). +- Non-ingested artifacts remain excluded from compile output and now emit hybrid exclusion trace events. +- Hybrid merge, deduplication, include, and exclusion decisions are persisted in `trace_events`, including compile summary counts. +- Memory, entity, and non-artifact compile sections remain unchanged apart from expected trace/contract adjacency. +- Unit coverage for compiler/main/response generation passed: `./.venv/bin/python -m pytest tests/unit/test_compiler.py tests/unit/test_main.py tests/unit/test_response_generation.py` -> `50 passed`. +- Relevant Postgres-backed integration coverage passed: `./.venv/bin/python -m pytest tests/integration/test_context_compile.py` -> `12 passed`. +- The changed file set stayed within sprint scope: compiler/contracts/prompt serialization/tests/build report only; no reranking, connector, runner, or UI work was introduced. +- `BUILD_REPORT.md` was updated with contract changes, merge rules, commands run, example request/response, trace examples, and deferred scope. + +criteria missed - None. -## quality issues - -- No blocking quality issues found in the changed scope. +quality issues +- No blocking implementation issues found in the changed code. -## regression risks - -- Low. The change is narrowly scoped to compile-path semantic artifact retrieval and is covered by unit and Postgres-backed integration tests for ordering, exclusion rules, validation, tracing, and isolation. - -## docs issues - -- No documentation issues found within sprint scope. - -## should anything be added to RULES.md? - -- No. +regression risks +- The compile response contract removes `context_pack.semantic_artifact_chunks` and `context_pack.semantic_artifact_chunk_summary`. In-repo consumers were updated, but any external consumer not covered by this repository will need the new merged contract. +- There is no explicit negative test for the new mixed-input validation path where `artifact_retrieval` and `semantic_artifact_retrieval` target different scopes. The code does reject it with `400`, but that path is not directly exercised. -## should anything update ARCHITECTURE.md? +docs issues +- None blocking. `BUILD_REPORT.md` satisfies the sprint packet requirements. +should anything be added to RULES.md? - No. -## recommended next action +should anything update ARCHITECTURE.md? +- No immediate update required. The contract change is localized and already captured in the sprint/build artifacts. -- Mark Sprint 5I as review-approved and proceed only with the explicitly deferred follow-up work, starting with a separate sprint for hybrid lexical/semantic artifact behavior if desired. +recommended next action +- Accept Sprint 5J as complete and merge after normal approval flow. +- Optional follow-up: add one endpoint/integration test covering mismatched lexical/semantic artifact scopes returning `400`. diff --git a/apps/api/src/alicebot_api/compiler.py b/apps/api/src/alicebot_api/compiler.py index 46a89b1..e8eac76 100644 --- a/apps/api/src/alicebot_api/compiler.py +++ b/apps/api/src/alicebot_api/compiler.py @@ -5,7 +5,7 @@ from alicebot_api.contracts import ( COMPILER_VERSION_V0, - ArtifactRetrievalDecisionTracePayload, + ArtifactSelectionSource, CompileContextArtifactScopedSemanticArtifactRetrievalInput, CompilerDecision, CompileContextArtifactRetrievalInput, @@ -22,14 +22,12 @@ ContextPackHybridMemorySummary, ContextPackMemory, ContextPackMemorySummary, - ContextPackSemanticArtifactChunk, - ContextPackSemanticArtifactChunkSummary, HybridMemoryDecisionTracePayload, + HybridArtifactRetrievalDecisionTracePayload, MemorySelectionSource, SEMANTIC_MEMORY_RETRIEVAL_ORDER, TASK_ARTIFACT_CHUNK_RETRIEVAL_ORDER, TASK_ARTIFACT_CHUNK_SEMANTIC_RETRIEVAL_ORDER, - SemanticArtifactRetrievalDecisionTracePayload, SemanticMemoryRetrievalRequestInput, TRACE_KIND_CONTEXT_COMPILE, TraceEventRecord, @@ -44,8 +42,6 @@ retrieve_matching_task_artifact_chunks, ) from alicebot_api.semantic_retrieval import ( - retrieve_artifact_scoped_semantic_artifact_chunk_records, - retrieve_task_scoped_semantic_artifact_chunk_records, serialize_semantic_artifact_chunk_result_item, validate_semantic_artifact_chunk_retrieval_request, validate_semantic_memory_retrieval_request, @@ -68,6 +64,15 @@ _UNBOUNDED_SEMANTIC_ARTIFACT_RETRIEVAL_LIMIT = 2_147_483_647 HYBRID_MEMORY_SOURCE_PRECEDENCE: list[MemorySelectionSource] = ["symbolic", "semantic"] HYBRID_SYMBOLIC_ORDER = ["updated_at_asc", "created_at_asc", "id_asc"] +HYBRID_ARTIFACT_SOURCE_PRECEDENCE: list[ArtifactSelectionSource] = ["lexical", "semantic"] +HYBRID_ARTIFACT_MERGED_ORDER = [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", +] @dataclass(frozen=True, slots=True) @@ -91,13 +96,6 @@ class CompiledArtifactChunkSection: decisions: list[CompilerDecision] -@dataclass(frozen=True, slots=True) -class CompiledSemanticArtifactChunkSection: - items: list[ContextPackSemanticArtifactChunk] - summary: ContextPackSemanticArtifactChunkSummary - decisions: list[CompilerDecision] - - @dataclass(slots=True) class HybridMemoryCandidate: memory: MemoryRow @@ -105,6 +103,14 @@ class HybridMemoryCandidate: semantic_score: float | None = None +@dataclass(slots=True) +class HybridArtifactChunkCandidate: + item: ContextPackArtifactChunk + sources: list[ArtifactSelectionSource] + lexical_rank: int | None = None + semantic_rank: int | None = None + + def _session_sort_key( session: SessionRow, latest_session_sequence: dict[UUID, int], @@ -244,38 +250,37 @@ def _empty_hybrid_memory_summary() -> ContextPackHybridMemorySummary: def _empty_artifact_chunk_summary() -> ContextPackArtifactChunkSummary: return { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, "query": None, "query_terms": [], - "matching_rule": TASK_ARTIFACT_CHUNK_RETRIEVAL_MATCHING_RULE, - "limit": 0, - "searched_artifact_count": 0, - "candidate_count": 0, - "included_count": 0, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "order": list(TASK_ARTIFACT_CHUNK_RETRIEVAL_ORDER), - } - - -def _empty_semantic_artifact_chunk_summary() -> ContextPackSemanticArtifactChunkSummary: - return { - "requested": False, - "scope": None, "embedding_config_id": None, "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, + "matching_rule": None, "similarity_metric": None, - "order": list(TASK_ARTIFACT_CHUNK_SEMANTIC_RETRIEVAL_ORDER), + "source_precedence": list(HYBRID_ARTIFACT_SOURCE_PRECEDENCE), + "lexical_order": list(TASK_ARTIFACT_CHUNK_RETRIEVAL_ORDER), + "semantic_order": list(TASK_ARTIFACT_CHUNK_SEMANTIC_RETRIEVAL_ORDER), + "merged_order": list(HYBRID_ARTIFACT_MERGED_ORDER), } -def _artifact_retrieval_decision_metadata( +def _hybrid_artifact_retrieval_decision_metadata( *, scope_kind: str, task_id: UUID, @@ -284,63 +289,34 @@ def _artifact_retrieval_decision_metadata( media_type: str | None, ingestion_status: str, limit: int, + selected_sources: list[ArtifactSelectionSource], + embedding_config_id: UUID | None = None, + query_vector_dimensions: int = 0, match: dict[str, object] | None = None, + score: float | None = None, sequence_no: int | None = None, char_start: int | None = None, char_end_exclusive: int | None = None, -) -> ArtifactRetrievalDecisionTracePayload: - payload: ArtifactRetrievalDecisionTracePayload = { +) -> HybridArtifactRetrievalDecisionTracePayload: + payload: HybridArtifactRetrievalDecisionTracePayload = { "scope_kind": scope_kind, # type: ignore[typeddict-item] "task_id": str(task_id), "task_artifact_id": str(task_artifact_id), "relative_path": relative_path, "media_type": media_type, "ingestion_status": ingestion_status, # type: ignore[typeddict-item] + "selected_sources": list(selected_sources), + "embedding_config_id": None if embedding_config_id is None else str(embedding_config_id), + "query_vector_dimensions": query_vector_dimensions, "limit": limit, } if match is not None: payload["matched_query_terms"] = list(match["matched_query_terms"]) # type: ignore[index] payload["matched_query_term_count"] = int(match["matched_query_term_count"]) # type: ignore[index] payload["first_match_char_start"] = int(match["first_match_char_start"]) # type: ignore[index] - if sequence_no is not None: - payload["sequence_no"] = sequence_no - if char_start is not None: - payload["char_start"] = char_start - if char_end_exclusive is not None: - payload["char_end_exclusive"] = char_end_exclusive - return payload - - -def _semantic_artifact_retrieval_decision_metadata( - *, - scope_kind: str, - task_id: UUID, - task_artifact_id: UUID, - relative_path: str, - media_type: str | None, - ingestion_status: str, - embedding_config_id: UUID, - query_vector_dimensions: int, - limit: int, - score: float | None = None, - sequence_no: int | None = None, - char_start: int | None = None, - char_end_exclusive: int | None = None, -) -> SemanticArtifactRetrievalDecisionTracePayload: - payload: SemanticArtifactRetrievalDecisionTracePayload = { - "scope_kind": scope_kind, # type: ignore[typeddict-item] - "task_id": str(task_id), - "task_artifact_id": str(task_artifact_id), - "relative_path": relative_path, - "media_type": media_type, - "ingestion_status": ingestion_status, # type: ignore[typeddict-item] - "embedding_config_id": str(embedding_config_id), - "query_vector_dimensions": query_vector_dimensions, - "limit": limit, - "similarity_metric": "cosine_similarity", - } if score is not None: payload["score"] = score + payload["similarity_metric"] = "cosine_similarity" if sequence_no is not None: payload["sequence_no"] = sequence_no if char_start is not None: @@ -386,6 +362,110 @@ def _serialize_hybrid_memory(candidate: HybridMemoryCandidate) -> ContextPackMem } +def _serialize_hybrid_artifact_chunk(candidate: HybridArtifactChunkCandidate) -> ContextPackArtifactChunk: + item = candidate.item + return { + "id": item["id"], + "task_id": item["task_id"], + "task_artifact_id": item["task_artifact_id"], + "relative_path": item["relative_path"], + "media_type": item["media_type"], + "sequence_no": item["sequence_no"], + "char_start": item["char_start"], + "char_end_exclusive": item["char_end_exclusive"], + "text": item["text"], + "source_provenance": { + "sources": list(candidate.sources), + "lexical_match": item["source_provenance"]["lexical_match"], + "semantic_score": item["source_provenance"]["semantic_score"], + }, + } + + +def _resolve_artifact_scope( + store: ContinuityStore, + *, + artifact_retrieval: CompileContextArtifactRetrievalInput | None, + semantic_artifact_retrieval: CompileContextSemanticArtifactRetrievalInput | None, +) -> tuple[list[dict[str, object]], dict[str, str] | None, str | None]: + lexical_scope: tuple[list[dict[str, object]], dict[str, str], str] | None = None + semantic_scope: tuple[list[dict[str, object]], dict[str, str], str] | None = None + + if isinstance(artifact_retrieval, CompileContextTaskScopedArtifactRetrievalInput): + task = store.get_task_optional(artifact_retrieval.task_id) + if task is None: + raise TaskNotFoundError(f"task {artifact_retrieval.task_id} was not found") + lexical_scope = ( + store.list_task_artifacts_for_task(artifact_retrieval.task_id), + build_task_artifact_chunk_retrieval_scope( + kind="task", + task_id=artifact_retrieval.task_id, + ), + "task", + ) + elif isinstance(artifact_retrieval, CompileContextArtifactScopedArtifactRetrievalInput): + artifact_row = store.get_task_artifact_optional(artifact_retrieval.task_artifact_id) + if artifact_row is None: + raise TaskArtifactNotFoundError( + f"task artifact {artifact_retrieval.task_artifact_id} was not found" + ) + lexical_scope = ( + [artifact_row], + build_task_artifact_chunk_retrieval_scope( + kind="artifact", + task_id=artifact_row["task_id"], + task_artifact_id=artifact_row["id"], + ), + "artifact", + ) + + if isinstance( + semantic_artifact_retrieval, + CompileContextTaskScopedSemanticArtifactRetrievalInput, + ): + task = store.get_task_optional(semantic_artifact_retrieval.task_id) + if task is None: + raise TaskNotFoundError(f"task {semantic_artifact_retrieval.task_id} was not found") + semantic_scope = ( + store.list_task_artifacts_for_task(semantic_artifact_retrieval.task_id), + build_task_artifact_chunk_retrieval_scope( + kind="task", + task_id=semantic_artifact_retrieval.task_id, + ), + "task", + ) + elif isinstance( + semantic_artifact_retrieval, + CompileContextArtifactScopedSemanticArtifactRetrievalInput, + ): + artifact_row = store.get_task_artifact_optional( + semantic_artifact_retrieval.task_artifact_id + ) + if artifact_row is None: + raise TaskArtifactNotFoundError( + f"task artifact {semantic_artifact_retrieval.task_artifact_id} was not found" + ) + semantic_scope = ( + [artifact_row], + build_task_artifact_chunk_retrieval_scope( + kind="artifact", + task_id=artifact_row["task_id"], + task_artifact_id=artifact_row["id"], + ), + "artifact", + ) + + if lexical_scope is not None and semantic_scope is not None and lexical_scope[1] != semantic_scope[1]: + raise TaskArtifactChunkRetrievalValidationError( + "artifact_retrieval and semantic_artifact_retrieval must target the same scope" + ) + + resolved_scope = lexical_scope or semantic_scope + if resolved_scope is None: + return [], None, None + return resolved_scope + + def _build_symbolic_memory_section( *, memories: list[MemoryRow], @@ -649,194 +729,145 @@ def _compile_artifact_chunk_section( store: ContinuityStore, *, artifact_retrieval: CompileContextArtifactRetrievalInput | None, + semantic_artifact_retrieval: CompileContextSemanticArtifactRetrievalInput | None, ) -> CompiledArtifactChunkSection: - if artifact_retrieval is None: + if artifact_retrieval is None and semantic_artifact_retrieval is None: return CompiledArtifactChunkSection( items=[], summary=_empty_artifact_chunk_summary(), decisions=[], ) - if isinstance(artifact_retrieval, CompileContextTaskScopedArtifactRetrievalInput): - task = store.get_task_optional(artifact_retrieval.task_id) - if task is None: - raise TaskNotFoundError(f"task {artifact_retrieval.task_id} was not found") - artifact_rows = store.list_task_artifacts_for_task(artifact_retrieval.task_id) - scope = build_task_artifact_chunk_retrieval_scope( - kind="task", - task_id=artifact_retrieval.task_id, - ) - scope_kind = "task" - else: - artifact_row = store.get_task_artifact_optional(artifact_retrieval.task_artifact_id) - if artifact_row is None: - raise TaskArtifactNotFoundError( - f"task artifact {artifact_retrieval.task_artifact_id} was not found" - ) - artifact_rows = [artifact_row] - scope = build_task_artifact_chunk_retrieval_scope( - kind="artifact", - task_id=artifact_row["task_id"], - task_artifact_id=artifact_row["id"], - ) - scope_kind = "artifact" - - query_terms = resolve_artifact_chunk_retrieval_query_terms(artifact_retrieval.query) - matched_items, searched_artifact_count = retrieve_matching_task_artifact_chunks( + artifact_rows, scope, scope_kind = _resolve_artifact_scope( store, - artifact_rows=artifact_rows, - query_terms=query_terms, + artifact_retrieval=artifact_retrieval, + semantic_artifact_retrieval=semantic_artifact_retrieval, ) - included_items = matched_items[: artifact_retrieval.limit] - excluded_uningested_artifact_count = 0 - decisions: list[CompilerDecision] = [] - - for position, artifact_row in enumerate(artifact_rows, start=1): - if artifact_row["ingestion_status"] == "ingested": - continue - excluded_uningested_artifact_count += 1 - decisions.append( - CompilerDecision( - "excluded", - "task_artifact", - artifact_row["id"], - "artifact_not_ingested", - position, - metadata=_artifact_retrieval_decision_metadata( - scope_kind=scope_kind, - task_id=artifact_row["task_id"], - task_artifact_id=artifact_row["id"], - relative_path=artifact_row["relative_path"], - media_type=infer_task_artifact_media_type(artifact_row), - ingestion_status=artifact_row["ingestion_status"], - limit=artifact_retrieval.limit, - ), - ) - ) - - for position, item in enumerate(matched_items, start=1): - decision_kind = "included" if position <= artifact_retrieval.limit else "excluded" - decision_reason = ( - "within_artifact_chunk_limit" - if position <= artifact_retrieval.limit - else "artifact_chunk_limit_exceeded" - ) - decisions.append( - CompilerDecision( - decision_kind, - "artifact_chunk", - UUID(item["id"]), - decision_reason, - position, - metadata=_artifact_retrieval_decision_metadata( - scope_kind=scope_kind, - task_id=UUID(item["task_id"]), - task_artifact_id=UUID(item["task_artifact_id"]), - relative_path=item["relative_path"], - media_type=item["media_type"], - ingestion_status="ingested", - limit=artifact_retrieval.limit, - match=item["match"], - sequence_no=item["sequence_no"], - char_start=item["char_start"], - char_end_exclusive=item["char_end_exclusive"], - ), - ) - ) - - return CompiledArtifactChunkSection( - items=list(included_items), - summary={ - "requested": True, - "scope": scope, - "query": artifact_retrieval.query, - "query_terms": list(query_terms), - "matching_rule": TASK_ARTIFACT_CHUNK_RETRIEVAL_MATCHING_RULE, - "limit": artifact_retrieval.limit, - "searched_artifact_count": searched_artifact_count, - "candidate_count": len(matched_items), - "included_count": len(included_items), - "excluded_uningested_artifact_count": excluded_uningested_artifact_count, - "excluded_limit_count": max(len(matched_items) - len(included_items), 0), - "order": list(TASK_ARTIFACT_CHUNK_RETRIEVAL_ORDER), - }, - decisions=decisions, + assert scope is not None + assert scope_kind is not None + + query = None if artifact_retrieval is None else artifact_retrieval.query + query_terms: list[str] = [] + lexical_items: list[ContextPackArtifactChunk] = [] + searched_artifact_count = sum( + 1 for artifact_row in artifact_rows if artifact_row["ingestion_status"] == "ingested" ) - - -def _compile_semantic_artifact_chunk_section( - store: ContinuityStore, - *, - semantic_artifact_retrieval: CompileContextSemanticArtifactRetrievalInput | None, -) -> CompiledSemanticArtifactChunkSection: - if semantic_artifact_retrieval is None: - return CompiledSemanticArtifactChunkSection( - items=[], - summary=_empty_semantic_artifact_chunk_summary(), - decisions=[], + if artifact_retrieval is not None: + query_terms = resolve_artifact_chunk_retrieval_query_terms(artifact_retrieval.query) + lexical_matches, searched_artifact_count = retrieve_matching_task_artifact_chunks( + store, + artifact_rows=artifact_rows, + query_terms=query_terms, ) + lexical_items = [ + { + "id": item["id"], + "task_id": item["task_id"], + "task_artifact_id": item["task_artifact_id"], + "relative_path": item["relative_path"], + "media_type": item["media_type"], + "sequence_no": item["sequence_no"], + "char_start": item["char_start"], + "char_end_exclusive": item["char_end_exclusive"], + "text": item["text"], + "source_provenance": { + "sources": ["lexical"], + "lexical_match": item["match"], + "semantic_score": None, + }, + } + for item in lexical_matches + ] + semantic_items: list[ContextPackArtifactChunk] = [] + query_vector_dimensions = 0 if isinstance( semantic_artifact_retrieval, CompileContextTaskScopedSemanticArtifactRetrievalInput, ): - task = store.get_task_optional(semantic_artifact_retrieval.task_id) - if task is None: - raise TaskNotFoundError(f"task {semantic_artifact_retrieval.task_id} was not found") - artifact_rows = store.list_task_artifacts_for_task(semantic_artifact_retrieval.task_id) - scope_kind = "task" - section_payload = retrieve_task_scoped_semantic_artifact_chunk_records( - store, - user_id=task["id"], - request=semantic_artifact_retrieval, - ) _config, query_vector = validate_semantic_artifact_chunk_retrieval_request( store, embedding_config_id=semantic_artifact_retrieval.embedding_config_id, query_vector=semantic_artifact_retrieval.query_vector, ) - matched_items = [ - serialize_semantic_artifact_chunk_result_item(row) - for row in store.retrieve_task_scoped_semantic_artifact_chunk_matches( - task_id=semantic_artifact_retrieval.task_id, - embedding_config_id=semantic_artifact_retrieval.embedding_config_id, - query_vector=query_vector, - limit=_UNBOUNDED_SEMANTIC_ARTIFACT_RETRIEVAL_LIMIT, - ) + query_vector_dimensions = len(query_vector) + semantic_items = [ + { + "id": item["id"], + "task_id": item["task_id"], + "task_artifact_id": item["task_artifact_id"], + "relative_path": item["relative_path"], + "media_type": item["media_type"], + "sequence_no": item["sequence_no"], + "char_start": item["char_start"], + "char_end_exclusive": item["char_end_exclusive"], + "text": item["text"], + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": item["score"], + }, + } + for item in [ + serialize_semantic_artifact_chunk_result_item(row) + for row in store.retrieve_task_scoped_semantic_artifact_chunk_matches( + task_id=semantic_artifact_retrieval.task_id, + embedding_config_id=semantic_artifact_retrieval.embedding_config_id, + query_vector=query_vector, + limit=_UNBOUNDED_SEMANTIC_ARTIFACT_RETRIEVAL_LIMIT, + ) + ] ] - else: - artifact_row = store.get_task_artifact_optional( - semantic_artifact_retrieval.task_artifact_id - ) - if artifact_row is None: - raise TaskArtifactNotFoundError( - f"task artifact {semantic_artifact_retrieval.task_artifact_id} was not found" - ) - artifact_rows = [artifact_row] - scope_kind = "artifact" - section_payload = retrieve_artifact_scoped_semantic_artifact_chunk_records( - store, - user_id=artifact_row["task_id"], - request=semantic_artifact_retrieval, - ) + elif isinstance( + semantic_artifact_retrieval, + CompileContextArtifactScopedSemanticArtifactRetrievalInput, + ): _config, query_vector = validate_semantic_artifact_chunk_retrieval_request( store, embedding_config_id=semantic_artifact_retrieval.embedding_config_id, query_vector=semantic_artifact_retrieval.query_vector, ) - matched_items = [ - serialize_semantic_artifact_chunk_result_item(row) - for row in store.retrieve_artifact_scoped_semantic_artifact_chunk_matches( - task_artifact_id=semantic_artifact_retrieval.task_artifact_id, - embedding_config_id=semantic_artifact_retrieval.embedding_config_id, - query_vector=query_vector, - limit=_UNBOUNDED_SEMANTIC_ARTIFACT_RETRIEVAL_LIMIT, - ) + query_vector_dimensions = len(query_vector) + semantic_items = [ + { + "id": item["id"], + "task_id": item["task_id"], + "task_artifact_id": item["task_artifact_id"], + "relative_path": item["relative_path"], + "media_type": item["media_type"], + "sequence_no": item["sequence_no"], + "char_start": item["char_start"], + "char_end_exclusive": item["char_end_exclusive"], + "text": item["text"], + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": item["score"], + }, + } + for item in [ + serialize_semantic_artifact_chunk_result_item(row) + for row in store.retrieve_artifact_scoped_semantic_artifact_chunk_matches( + task_artifact_id=semantic_artifact_retrieval.task_artifact_id, + embedding_config_id=semantic_artifact_retrieval.embedding_config_id, + query_vector=query_vector, + limit=_UNBOUNDED_SEMANTIC_ARTIFACT_RETRIEVAL_LIMIT, + ) + ] ] - included_items = list(section_payload["items"]) + merged_candidates: list[HybridArtifactChunkCandidate] = [] + merged_candidates_by_id: dict[str, HybridArtifactChunkCandidate] = {} + deduplicated_count = 0 excluded_uningested_artifact_count = 0 decisions: list[CompilerDecision] = [] + final_limit = ( + artifact_retrieval.limit + if artifact_retrieval is not None + else semantic_artifact_retrieval.limit + if semantic_artifact_retrieval is not None + else 0 + ) for position, artifact_row in enumerate(artifact_rows, start=1): if artifact_row["ingestion_status"] == "ingested": @@ -847,70 +878,216 @@ def _compile_semantic_artifact_chunk_section( "excluded", "task_artifact", artifact_row["id"], - "semantic_artifact_not_ingested", + "hybrid_artifact_not_ingested", position, - metadata=_semantic_artifact_retrieval_decision_metadata( + metadata=_hybrid_artifact_retrieval_decision_metadata( scope_kind=scope_kind, task_id=artifact_row["task_id"], task_artifact_id=artifact_row["id"], relative_path=artifact_row["relative_path"], media_type=infer_task_artifact_media_type(artifact_row), ingestion_status=artifact_row["ingestion_status"], + limit=final_limit, + selected_sources=[], + embedding_config_id=( + None + if semantic_artifact_retrieval is None + else semantic_artifact_retrieval.embedding_config_id + ), + query_vector_dimensions=query_vector_dimensions, + ), + ) + ) + + for lexical_rank, item in enumerate(lexical_items, start=1): + candidate = HybridArtifactChunkCandidate( + item=item, + sources=["lexical"], + lexical_rank=lexical_rank, + ) + merged_candidates.append(candidate) + merged_candidates_by_id[item["id"]] = candidate + + for semantic_rank, item in enumerate(semantic_items, start=1): + existing_candidate = merged_candidates_by_id.get(item["id"]) + if existing_candidate is None: + candidate = HybridArtifactChunkCandidate( + item=item, + sources=["semantic"], + semantic_rank=semantic_rank, + ) + merged_candidates.append(candidate) + merged_candidates_by_id[item["id"]] = candidate + continue + + deduplicated_count += 1 + if "semantic" not in existing_candidate.sources: + existing_candidate.sources.append("semantic") + existing_candidate.semantic_rank = semantic_rank + existing_candidate.item["source_provenance"]["semantic_score"] = item["source_provenance"][ + "semantic_score" + ] + decisions.append( + CompilerDecision( + "included", + "artifact_chunk", + UUID(existing_candidate.item["id"]), + "hybrid_artifact_chunk_deduplicated", + semantic_rank, + metadata=_hybrid_artifact_retrieval_decision_metadata( + scope_kind=scope_kind, + task_id=UUID(existing_candidate.item["task_id"]), + task_artifact_id=UUID(existing_candidate.item["task_artifact_id"]), + relative_path=existing_candidate.item["relative_path"], + media_type=existing_candidate.item["media_type"], + ingestion_status="ingested", + limit=final_limit, + selected_sources=existing_candidate.sources, embedding_config_id=semantic_artifact_retrieval.embedding_config_id, - query_vector_dimensions=len(query_vector), - limit=semantic_artifact_retrieval.limit, + query_vector_dimensions=query_vector_dimensions, + match=existing_candidate.item["source_provenance"]["lexical_match"], + score=existing_candidate.item["source_provenance"]["semantic_score"], + sequence_no=existing_candidate.item["sequence_no"], + char_start=existing_candidate.item["char_start"], + char_end_exclusive=existing_candidate.item["char_end_exclusive"], ), ) ) - for position, item in enumerate(matched_items, start=1): - decision_kind = "included" if position <= semantic_artifact_retrieval.limit else "excluded" - decision_reason = ( - "within_semantic_artifact_chunk_limit" - if position <= semantic_artifact_retrieval.limit - else "semantic_artifact_chunk_limit_exceeded" + merged_candidates.sort( + key=lambda candidate: ( + min( + HYBRID_ARTIFACT_SOURCE_PRECEDENCE.index(source) + for source in candidate.sources + ), + candidate.lexical_rank if candidate.lexical_rank is not None else 2_147_483_647, + candidate.semantic_rank if candidate.semantic_rank is not None else 2_147_483_647, + candidate.item["relative_path"], + candidate.item["sequence_no"], + candidate.item["id"], ) + ) + + included_candidates = merged_candidates[:final_limit] if final_limit > 0 else [] + excluded_candidates = merged_candidates[final_limit:] if final_limit > 0 else merged_candidates + included_lexical_only_count = 0 + included_semantic_only_count = 0 + included_dual_source_count = 0 + + for position, candidate in enumerate(merged_candidates, start=1): + if position <= final_limit and final_limit > 0: + if candidate.sources == ["lexical"]: + included_lexical_only_count += 1 + elif candidate.sources == ["semantic"]: + included_semantic_only_count += 1 + else: + included_dual_source_count += 1 + decisions.append( + CompilerDecision( + "included", + "artifact_chunk", + UUID(candidate.item["id"]), + "within_hybrid_artifact_chunk_limit", + position, + metadata=_hybrid_artifact_retrieval_decision_metadata( + scope_kind=scope_kind, + task_id=UUID(candidate.item["task_id"]), + task_artifact_id=UUID(candidate.item["task_artifact_id"]), + relative_path=candidate.item["relative_path"], + media_type=candidate.item["media_type"], + ingestion_status="ingested", + limit=final_limit, + selected_sources=candidate.sources, + embedding_config_id=( + None + if semantic_artifact_retrieval is None + else semantic_artifact_retrieval.embedding_config_id + ), + query_vector_dimensions=query_vector_dimensions, + match=candidate.item["source_provenance"]["lexical_match"], + score=candidate.item["source_provenance"]["semantic_score"], + sequence_no=candidate.item["sequence_no"], + char_start=candidate.item["char_start"], + char_end_exclusive=candidate.item["char_end_exclusive"], + ), + ) + ) + continue + decisions.append( CompilerDecision( - decision_kind, - "semantic_artifact_chunk", - UUID(item["id"]), - decision_reason, + "excluded", + "artifact_chunk", + UUID(candidate.item["id"]), + "hybrid_artifact_chunk_limit_exceeded", position, - metadata=_semantic_artifact_retrieval_decision_metadata( + metadata=_hybrid_artifact_retrieval_decision_metadata( scope_kind=scope_kind, - task_id=UUID(item["task_id"]), - task_artifact_id=UUID(item["task_artifact_id"]), - relative_path=item["relative_path"], - media_type=item["media_type"], + task_id=UUID(candidate.item["task_id"]), + task_artifact_id=UUID(candidate.item["task_artifact_id"]), + relative_path=candidate.item["relative_path"], + media_type=candidate.item["media_type"], ingestion_status="ingested", - embedding_config_id=semantic_artifact_retrieval.embedding_config_id, - query_vector_dimensions=len(query_vector), - limit=semantic_artifact_retrieval.limit, - score=item["score"], - sequence_no=item["sequence_no"], - char_start=item["char_start"], - char_end_exclusive=item["char_end_exclusive"], + limit=final_limit, + selected_sources=candidate.sources, + embedding_config_id=( + None + if semantic_artifact_retrieval is None + else semantic_artifact_retrieval.embedding_config_id + ), + query_vector_dimensions=query_vector_dimensions, + match=candidate.item["source_provenance"]["lexical_match"], + score=candidate.item["source_provenance"]["semantic_score"], + sequence_no=candidate.item["sequence_no"], + char_start=candidate.item["char_start"], + char_end_exclusive=candidate.item["char_end_exclusive"], ), ) ) - section_summary = section_payload["summary"] - return CompiledSemanticArtifactChunkSection( - items=included_items, + return CompiledArtifactChunkSection( + items=[_serialize_hybrid_artifact_chunk(candidate) for candidate in included_candidates], summary={ "requested": True, - "scope": section_summary["scope"], - "embedding_config_id": section_summary["embedding_config_id"], - "query_vector_dimensions": section_summary["query_vector_dimensions"], - "limit": section_summary["limit"], - "searched_artifact_count": section_summary["searched_artifact_count"], - "candidate_count": len(matched_items), - "included_count": len(included_items), + "lexical_requested": artifact_retrieval is not None, + "semantic_requested": semantic_artifact_retrieval is not None, + "scope": scope, + "query": query, + "query_terms": list(query_terms), + "embedding_config_id": ( + None + if semantic_artifact_retrieval is None + else str(semantic_artifact_retrieval.embedding_config_id) + ), + "query_vector_dimensions": query_vector_dimensions, + "limit": final_limit, + "lexical_limit": 0 if artifact_retrieval is None else artifact_retrieval.limit, + "semantic_limit": ( + 0 if semantic_artifact_retrieval is None else semantic_artifact_retrieval.limit + ), + "searched_artifact_count": searched_artifact_count, + "lexical_candidate_count": len(lexical_items), + "semantic_candidate_count": len(semantic_items), + "merged_candidate_count": len(merged_candidates), + "deduplicated_count": deduplicated_count, + "included_count": len(included_candidates), + "included_lexical_only_count": included_lexical_only_count, + "included_semantic_only_count": included_semantic_only_count, + "included_dual_source_count": included_dual_source_count, "excluded_uningested_artifact_count": excluded_uningested_artifact_count, - "excluded_limit_count": max(len(matched_items) - len(included_items), 0), - "similarity_metric": section_summary["similarity_metric"], - "order": list(section_summary["order"]), + "excluded_limit_count": len(excluded_candidates), + "matching_rule": ( + None + if artifact_retrieval is None + else TASK_ARTIFACT_CHUNK_RETRIEVAL_MATCHING_RULE + ), + "similarity_metric": ( + None if semantic_artifact_retrieval is None else "cosine_similarity" + ), + "source_precedence": list(HYBRID_ARTIFACT_SOURCE_PRECEDENCE), + "lexical_order": list(TASK_ARTIFACT_CHUNK_RETRIEVAL_ORDER), + "semantic_order": list(TASK_ARTIFACT_CHUNK_SEMANTIC_RETRIEVAL_ORDER), + "merged_order": list(HYBRID_ARTIFACT_MERGED_ORDER), }, decisions=decisions, ) @@ -928,7 +1105,6 @@ def compile_continuity_context( limits: ContextCompilerLimits, memory_section: CompiledMemorySection | None = None, artifact_chunk_section: CompiledArtifactChunkSection | None = None, - semantic_artifact_chunk_section: CompiledSemanticArtifactChunkSection | None = None, ) -> CompilerRunResult: latest_session_sequence: dict[UUID, int] = {} for event in events: @@ -1027,15 +1203,6 @@ def compile_continuity_context( decisions=[], ) decisions.extend(resolved_artifact_chunk_section.decisions) - resolved_semantic_artifact_chunk_section = ( - semantic_artifact_chunk_section - or CompiledSemanticArtifactChunkSection( - items=[], - summary=_empty_semantic_artifact_chunk_summary(), - decisions=[], - ) - ) - decisions.extend(resolved_semantic_artifact_chunk_section.decisions) ordered_entities = sorted(entities, key=_entity_sort_key) included_entities = ordered_entities[-limits.max_entities :] if limits.max_entities > 0 else [] included_entity_ids = {entity["id"] for entity in included_entities} @@ -1172,36 +1339,34 @@ def compile_continuity_context( if resolved_artifact_chunk_section.summary["scope"] is None else resolved_artifact_chunk_section.summary["scope"]["kind"] ), - "artifact_chunk_candidate_count": resolved_artifact_chunk_section.summary[ - "candidate_count" + "artifact_lexical_retrieval_requested": resolved_artifact_chunk_section.summary[ + "lexical_requested" ], - "included_artifact_chunk_count": resolved_artifact_chunk_section.summary[ - "included_count" + "artifact_semantic_retrieval_requested": resolved_artifact_chunk_section.summary[ + "semantic_requested" ], - "excluded_artifact_chunk_limit_count": resolved_artifact_chunk_section.summary[ - "excluded_limit_count" + "artifact_lexical_candidate_count": resolved_artifact_chunk_section.summary[ + "lexical_candidate_count" ], - "excluded_uningested_artifact_count": resolved_artifact_chunk_section.summary[ - "excluded_uningested_artifact_count" + "artifact_semantic_candidate_count": resolved_artifact_chunk_section.summary[ + "semantic_candidate_count" ], - "semantic_artifact_retrieval_requested": resolved_semantic_artifact_chunk_section.summary[ - "requested" + "artifact_merged_candidate_count": resolved_artifact_chunk_section.summary[ + "merged_candidate_count" ], - "semantic_artifact_retrieval_scope_kind": ( - None - if resolved_semantic_artifact_chunk_section.summary["scope"] is None - else resolved_semantic_artifact_chunk_section.summary["scope"]["kind"] - ), - "semantic_artifact_chunk_candidate_count": resolved_semantic_artifact_chunk_section.summary[ - "candidate_count" + "artifact_deduplicated_count": resolved_artifact_chunk_section.summary[ + "deduplicated_count" ], - "included_semantic_artifact_chunk_count": resolved_semantic_artifact_chunk_section.summary[ + "included_artifact_chunk_count": resolved_artifact_chunk_section.summary[ "included_count" ], - "excluded_semantic_artifact_chunk_limit_count": resolved_semantic_artifact_chunk_section.summary[ + "included_dual_source_artifact_chunk_count": resolved_artifact_chunk_section.summary[ + "included_dual_source_count" + ], + "excluded_artifact_chunk_limit_count": resolved_artifact_chunk_section.summary[ "excluded_limit_count" ], - "excluded_semantic_uningested_artifact_count": resolved_semantic_artifact_chunk_section.summary[ + "excluded_uningested_artifact_count": resolved_artifact_chunk_section.summary[ "excluded_uningested_artifact_count" ], "included_entity_count": len(included_entities), @@ -1237,8 +1402,6 @@ def compile_continuity_context( "memory_summary": resolved_memory_section.summary, "artifact_chunks": list(resolved_artifact_chunk_section.items), "artifact_chunk_summary": resolved_artifact_chunk_section.summary, - "semantic_artifact_chunks": list(resolved_semantic_artifact_chunk_section.items), - "semantic_artifact_chunk_summary": resolved_semantic_artifact_chunk_section.summary, "entities": [_serialize_entity(entity) for entity in included_entities], "entity_summary": { "candidate_count": len(ordered_entities), @@ -1281,9 +1444,6 @@ def compile_and_persist_trace( artifact_chunk_section = _compile_artifact_chunk_section( store, artifact_retrieval=artifact_retrieval, - ) - semantic_artifact_chunk_section = _compile_semantic_artifact_chunk_section( - store, semantic_artifact_retrieval=semantic_artifact_retrieval, ) entities = store.list_entities() @@ -1301,7 +1461,6 @@ def compile_and_persist_trace( limits=limits, memory_section=memory_section, artifact_chunk_section=artifact_chunk_section, - semantic_artifact_chunk_section=semantic_artifact_chunk_section, ) trace = store.create_trace( user_id=user_id, diff --git a/apps/api/src/alicebot_api/contracts.py b/apps/api/src/alicebot_api/contracts.py index 8d4882b..7362392 100644 --- a/apps/api/src/alicebot_api/contracts.py +++ b/apps/api/src/alicebot_api/contracts.py @@ -80,6 +80,7 @@ "remember_that_i_prefer", ] MemorySelectionSource = Literal["symbolic", "semantic"] +ArtifactSelectionSource = Literal["lexical", "semantic"] DEFAULT_MAX_SESSIONS = 3 DEFAULT_MAX_EVENTS = 8 @@ -416,50 +417,44 @@ class ContextPackArtifactChunk(TypedDict): char_start: int char_end_exclusive: int text: str - match: "TaskArtifactChunkRetrievalMatch" + source_provenance: "ContextPackArtifactChunkSourceProvenance" + + +class ContextPackArtifactChunkSourceProvenance(TypedDict): + sources: list[ArtifactSelectionSource] + lexical_match: "TaskArtifactChunkRetrievalMatch | None" + semantic_score: float | None class ContextPackArtifactChunkSummary(TypedDict): requested: bool + lexical_requested: bool + semantic_requested: bool scope: TaskArtifactChunkRetrievalScope | None query: str | None query_terms: list[str] - matching_rule: str - limit: int - searched_artifact_count: int - candidate_count: int - included_count: int - excluded_uningested_artifact_count: int - excluded_limit_count: int - order: list[str] - - -class ContextPackSemanticArtifactChunk(TypedDict): - id: str - task_id: str - task_artifact_id: str - relative_path: str - media_type: str - sequence_no: int - char_start: int - char_end_exclusive: int - text: str - score: float - - -class ContextPackSemanticArtifactChunkSummary(TypedDict): - requested: bool - scope: TaskArtifactChunkRetrievalScope | None embedding_config_id: str | None query_vector_dimensions: int limit: int + lexical_limit: int + semantic_limit: int searched_artifact_count: int - candidate_count: int + lexical_candidate_count: int + semantic_candidate_count: int + merged_candidate_count: int + deduplicated_count: int included_count: int + included_lexical_only_count: int + included_semantic_only_count: int + included_dual_source_count: int excluded_uningested_artifact_count: int excluded_limit_count: int + matching_rule: str | None similarity_metric: Literal["cosine_similarity"] | None - order: list[str] + source_precedence: list[ArtifactSelectionSource] + lexical_order: list[str] + semantic_order: list[str] + merged_order: list[str] class ArtifactRetrievalDecisionTracePayload(TypedDict): @@ -478,18 +473,22 @@ class ArtifactRetrievalDecisionTracePayload(TypedDict): char_end_exclusive: NotRequired[int] -class SemanticArtifactRetrievalDecisionTracePayload(TypedDict): +class HybridArtifactRetrievalDecisionTracePayload(TypedDict): scope_kind: TaskArtifactChunkRetrievalScopeKind task_id: str task_artifact_id: str relative_path: str media_type: str | None ingestion_status: TaskArtifactIngestionStatus - embedding_config_id: str - query_vector_dimensions: int limit: int - similarity_metric: Literal["cosine_similarity"] + selected_sources: list[ArtifactSelectionSource] + embedding_config_id: str | None + query_vector_dimensions: int + matched_query_terms: NotRequired[list[str]] + matched_query_term_count: NotRequired[int] + first_match_char_start: NotRequired[int] score: NotRequired[float] + similarity_metric: NotRequired[Literal["cosine_similarity"]] sequence_no: NotRequired[int] char_start: NotRequired[int] char_end_exclusive: NotRequired[int] @@ -580,8 +579,6 @@ class CompiledContextPack(TypedDict): memory_summary: ContextPackMemorySummary artifact_chunks: list[ContextPackArtifactChunk] artifact_chunk_summary: ContextPackArtifactChunkSummary - semantic_artifact_chunks: list[ContextPackSemanticArtifactChunk] - semantic_artifact_chunk_summary: ContextPackSemanticArtifactChunkSummary entities: list[ContextPackEntity] entity_summary: ContextPackEntitySummary entity_edges: list[ContextPackEntityEdge] diff --git a/apps/api/src/alicebot_api/response_generation.py b/apps/api/src/alicebot_api/response_generation.py index 78f2ee6..7f6c055 100644 --- a/apps/api/src/alicebot_api/response_generation.py +++ b/apps/api/src/alicebot_api/response_generation.py @@ -92,8 +92,6 @@ def _context_section_payload(context_pack: CompiledContextPack) -> JsonObject: "memory_summary": context_pack["memory_summary"], "artifact_chunks": context_pack["artifact_chunks"], "artifact_chunk_summary": context_pack["artifact_chunk_summary"], - "semantic_artifact_chunks": context_pack["semantic_artifact_chunks"], - "semantic_artifact_chunk_summary": context_pack["semantic_artifact_chunk_summary"], "entities": context_pack["entities"], "entity_summary": context_pack["entity_summary"], "entity_edges": context_pack["entity_edges"], diff --git a/tests/integration/test_context_compile.py b/tests/integration/test_context_compile.py index cc43cd5..77979c5 100644 --- a/tests/integration/test_context_compile.py +++ b/tests/integration/test_context_compile.py @@ -485,20 +485,49 @@ def test_compile_context_endpoint_persists_trace_and_trace_events(migrated_datab "semantic_order": ["score_desc", "created_at_asc", "id_asc"], }, } - assert payload["context_pack"]["semantic_artifact_chunks"] == [] - assert payload["context_pack"]["semantic_artifact_chunk_summary"] == { + assert payload["context_pack"]["artifact_chunks"] == [] + assert payload["context_pack"]["artifact_chunk_summary"] == { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, + "query": None, + "query_terms": [], "embedding_config_id": None, "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, + "matching_rule": None, "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } assert payload["context_pack"]["entities"] == [ { @@ -608,11 +637,13 @@ def test_compile_context_endpoint_persists_trace_and_trace_events(migrated_datab assert trace_events[-1]["payload"]["hybrid_memory_candidate_count"] == 2 assert trace_events[-1]["payload"]["hybrid_memory_merged_candidate_count"] == 1 assert trace_events[-1]["payload"]["hybrid_memory_deduplicated_count"] == 0 - assert trace_events[-1]["payload"]["semantic_artifact_retrieval_requested"] is False - assert trace_events[-1]["payload"]["semantic_artifact_chunk_candidate_count"] == 0 - assert trace_events[-1]["payload"]["included_semantic_artifact_chunk_count"] == 0 - assert trace_events[-1]["payload"]["excluded_semantic_artifact_chunk_limit_count"] == 0 - assert trace_events[-1]["payload"]["excluded_semantic_uningested_artifact_count"] == 0 + assert trace_events[-1]["payload"]["artifact_lexical_retrieval_requested"] is False + assert trace_events[-1]["payload"]["artifact_semantic_retrieval_requested"] is False + assert trace_events[-1]["payload"]["artifact_lexical_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_semantic_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_merged_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_deduplicated_count"] == 0 + assert trace_events[-1]["payload"]["included_dual_source_artifact_chunk_count"] == 0 assert trace_events[-1]["payload"]["included_entity_count"] == 1 assert trace_events[-1]["payload"]["excluded_entity_limit_count"] == 2 assert trace_events[-1]["payload"]["included_entity_edge_count"] == 1 @@ -691,20 +722,49 @@ def test_compile_context_prefers_updated_active_memory_within_same_transaction( "semantic_order": ["score_desc", "created_at_asc", "id_asc"], }, } - assert payload["context_pack"]["semantic_artifact_chunks"] == [] - assert payload["context_pack"]["semantic_artifact_chunk_summary"] == { + assert payload["context_pack"]["artifact_chunks"] == [] + assert payload["context_pack"]["artifact_chunk_summary"] == { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, + "query": None, + "query_terms": [], "embedding_config_id": None, "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, + "matching_rule": None, "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } assert payload["context_pack"]["entity_summary"] == { "candidate_count": 2, @@ -1005,10 +1065,14 @@ def test_compile_context_artifact_retrieval_integrates_chunks_traces_and_exclusi "char_start": 0, "char_end_exclusive": 14, "text": "beta alpha doc", - "match": { - "matched_query_terms": ["alpha", "beta"], - "matched_query_term_count": 2, - "first_match_char_start": 0, + "source_provenance": { + "sources": ["lexical"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": None, }, }, { @@ -1021,32 +1085,59 @@ def test_compile_context_artifact_retrieval_integrates_chunks_traces_and_exclusi "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "match": { - "matched_query_terms": ["alpha", "beta"], - "matched_query_term_count": 2, - "first_match_char_start": 0, + "source_provenance": { + "sources": ["lexical"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": None, }, }, ] assert payload["context_pack"]["artifact_chunk_summary"] == { "requested": True, + "lexical_requested": True, + "semantic_requested": False, "scope": {"kind": "task", "task_id": str(artifact_scope["task_id"])}, "query": "Alpha beta", "query_terms": ["alpha", "beta"], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 2, + "lexical_limit": 2, + "semantic_limit": 0, "searched_artifact_count": 3, - "candidate_count": 3, + "lexical_candidate_count": 3, + "semantic_candidate_count": 0, + "merged_candidate_count": 3, + "deduplicated_count": 0, "included_count": 2, + "included_lexical_only_count": 2, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 1, "excluded_limit_count": 1, - "order": [ + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } assert payload["context_pack"]["memories"] assert payload["context_pack"]["entities"] @@ -1056,7 +1147,7 @@ def test_compile_context_artifact_retrieval_integrates_chunks_traces_and_exclusi trace_events = ContinuityStore(conn).list_trace_events(trace_id) assert any( - event["payload"]["reason"] == "within_artifact_chunk_limit" + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["docs"]) and event["payload"]["relative_path"] == "docs/a.txt" and event["payload"]["matched_query_terms"] == ["alpha", "beta"] @@ -1064,21 +1155,21 @@ def test_compile_context_artifact_retrieval_integrates_chunks_traces_and_exclusi if event["kind"] == "context.included" ) assert any( - event["payload"]["reason"] == "within_artifact_chunk_limit" + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["notes"]) and event["payload"]["relative_path"] == "notes/b.md" for event in trace_events if event["kind"] == "context.included" ) assert any( - event["payload"]["reason"] == "artifact_chunk_limit_exceeded" + event["payload"]["reason"] == "hybrid_artifact_chunk_limit_exceeded" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["weak"]) and event["payload"]["relative_path"] == "notes/c.txt" for event in trace_events if event["kind"] == "context.excluded" ) assert any( - event["payload"]["reason"] == "artifact_not_ingested" + event["payload"]["reason"] == "hybrid_artifact_not_ingested" and event["payload"]["entity_id"] == str(artifact_scope["artifact_ids"]["pending"]) and event["payload"]["relative_path"] == "notes/hidden.txt" and event["payload"]["ingestion_status"] == "pending" @@ -1087,8 +1178,14 @@ def test_compile_context_artifact_retrieval_integrates_chunks_traces_and_exclusi ) assert trace_events[-1]["payload"]["artifact_retrieval_requested"] is True assert trace_events[-1]["payload"]["artifact_retrieval_scope_kind"] == "task" - assert trace_events[-1]["payload"]["artifact_chunk_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_lexical_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_semantic_retrieval_requested"] is False + assert trace_events[-1]["payload"]["artifact_lexical_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_semantic_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_merged_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_deduplicated_count"] == 0 assert trace_events[-1]["payload"]["included_artifact_chunk_count"] == 2 + assert trace_events[-1]["payload"]["included_dual_source_artifact_chunk_count"] == 0 assert trace_events[-1]["payload"]["excluded_artifact_chunk_limit_count"] == 1 assert trace_events[-1]["payload"]["excluded_uningested_artifact_count"] == 1 @@ -1134,15 +1231,21 @@ def test_compile_context_artifact_scoped_retrieval_returns_only_visible_artifact "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "match": { - "matched_query_terms": ["alpha", "beta"], - "matched_query_term_count": 2, - "first_match_char_start": 0, + "source_provenance": { + "sources": ["lexical"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": None, }, } ] assert payload["context_pack"]["artifact_chunk_summary"] == { "requested": True, + "lexical_requested": True, + "semantic_requested": False, "scope": { "kind": "artifact", "task_id": str(artifact_scope["task_id"]), @@ -1150,20 +1253,41 @@ def test_compile_context_artifact_scoped_retrieval_returns_only_visible_artifact }, "query": "Alpha beta", "query_terms": ["alpha", "beta"], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 2, + "lexical_limit": 2, + "semantic_limit": 0, "searched_artifact_count": 1, - "candidate_count": 1, + "lexical_candidate_count": 1, + "semantic_candidate_count": 0, + "merged_candidate_count": 1, + "deduplicated_count": 0, "included_count": 1, + "included_lexical_only_count": 1, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } trace_id = UUID(payload["trace_id"]) @@ -1171,7 +1295,7 @@ def test_compile_context_artifact_scoped_retrieval_returns_only_visible_artifact trace_events = ContinuityStore(conn).list_trace_events(trace_id) assert any( - event["payload"]["reason"] == "within_artifact_chunk_limit" + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["notes"]) and event["payload"]["scope_kind"] == "artifact" and event["payload"]["task_artifact_id"] == str(artifact_scope["artifact_ids"]["notes"]) @@ -1180,12 +1304,207 @@ def test_compile_context_artifact_scoped_retrieval_returns_only_visible_artifact ) assert trace_events[-1]["payload"]["artifact_retrieval_requested"] is True assert trace_events[-1]["payload"]["artifact_retrieval_scope_kind"] == "artifact" - assert trace_events[-1]["payload"]["artifact_chunk_candidate_count"] == 1 + assert trace_events[-1]["payload"]["artifact_lexical_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_semantic_retrieval_requested"] is False + assert trace_events[-1]["payload"]["artifact_lexical_candidate_count"] == 1 + assert trace_events[-1]["payload"]["artifact_semantic_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_merged_candidate_count"] == 1 + assert trace_events[-1]["payload"]["artifact_deduplicated_count"] == 0 assert trace_events[-1]["payload"]["included_artifact_chunk_count"] == 1 + assert trace_events[-1]["payload"]["included_dual_source_artifact_chunk_count"] == 0 assert trace_events[-1]["payload"]["excluded_artifact_chunk_limit_count"] == 0 assert trace_events[-1]["payload"]["excluded_uningested_artifact_count"] == 0 +def test_compile_context_hybrid_artifact_merge_preserves_dual_source_provenance_and_limits( + migrated_database_urls, + monkeypatch, +) -> None: + seeded = seed_traceable_thread(migrated_database_urls["app"]) + artifact_scope = seed_compile_artifact_scope( + migrated_database_urls["app"], + user_id=seeded["user_id"], + thread_id=seeded["thread_id"], + ) + config_id = seed_embedding_config_for_user( + migrated_database_urls["app"], + user_id=seeded["user_id"], + ) + seed_task_artifact_chunk_embedding_for_user( + migrated_database_urls["app"], + user_id=seeded["user_id"], + task_artifact_chunk_id=artifact_scope["chunk_ids"]["docs"], + embedding_config_id=config_id, + vector=[1.0, 0.0, 0.0], + ) + seed_task_artifact_chunk_embedding_for_user( + migrated_database_urls["app"], + user_id=seeded["user_id"], + task_artifact_chunk_id=artifact_scope["chunk_ids"]["notes"], + embedding_config_id=config_id, + vector=[1.0, 0.0, 0.0], + ) + seed_task_artifact_chunk_embedding_for_user( + migrated_database_urls["app"], + user_id=seeded["user_id"], + task_artifact_chunk_id=artifact_scope["chunk_ids"]["weak"], + embedding_config_id=config_id, + vector=[0.0, 1.0, 0.0], + ) + monkeypatch.setattr( + main_module, + "get_settings", + lambda: Settings(database_url=migrated_database_urls["app"]), + ) + + status_code, payload = invoke_compile_context( + { + "user_id": str(seeded["user_id"]), + "thread_id": str(seeded["thread_id"]), + "artifact_retrieval": { + "kind": "task", + "task_id": str(artifact_scope["task_id"]), + "query": "Alpha beta", + "limit": 2, + }, + "semantic_artifact_retrieval": { + "kind": "task", + "task_id": str(artifact_scope["task_id"]), + "embedding_config_id": str(config_id), + "query_vector": [1.0, 0.0, 0.0], + "limit": 2, + }, + } + ) + + assert status_code == 200 + assert payload["context_pack"]["artifact_chunks"] == [ + { + "id": str(artifact_scope["chunk_ids"]["docs"]), + "task_id": str(artifact_scope["task_id"]), + "task_artifact_id": str(artifact_scope["artifact_ids"]["docs"]), + "relative_path": "docs/a.txt", + "media_type": "text/plain", + "sequence_no": 1, + "char_start": 0, + "char_end_exclusive": 14, + "text": "beta alpha doc", + "source_provenance": { + "sources": ["lexical", "semantic"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": 1.0, + }, + }, + { + "id": str(artifact_scope["chunk_ids"]["notes"]), + "task_id": str(artifact_scope["task_id"]), + "task_artifact_id": str(artifact_scope["artifact_ids"]["notes"]), + "relative_path": "notes/b.md", + "media_type": "text/markdown", + "sequence_no": 1, + "char_start": 0, + "char_end_exclusive": 15, + "text": "alpha beta note", + "source_provenance": { + "sources": ["lexical", "semantic"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": 1.0, + }, + }, + ] + assert payload["context_pack"]["artifact_chunk_summary"] == { + "requested": True, + "lexical_requested": True, + "semantic_requested": True, + "scope": {"kind": "task", "task_id": str(artifact_scope["task_id"])}, + "query": "Alpha beta", + "query_terms": ["alpha", "beta"], + "embedding_config_id": str(config_id), + "query_vector_dimensions": 3, + "limit": 2, + "lexical_limit": 2, + "semantic_limit": 2, + "searched_artifact_count": 3, + "lexical_candidate_count": 3, + "semantic_candidate_count": 3, + "merged_candidate_count": 3, + "deduplicated_count": 3, + "included_count": 2, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 2, + "excluded_uningested_artifact_count": 1, + "excluded_limit_count": 1, + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "similarity_metric": "cosine_similarity", + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + } + + trace_id = UUID(payload["trace_id"]) + with user_connection(migrated_database_urls["app"], seeded["user_id"]) as conn: + trace_events = ContinuityStore(conn).list_trace_events(trace_id) + + assert any( + event["payload"]["reason"] == "hybrid_artifact_chunk_deduplicated" + and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["docs"]) + and event["payload"]["selected_sources"] == ["lexical", "semantic"] + and event["payload"]["score"] == 1.0 + for event in trace_events + if event["kind"] == "context.included" + ) + assert any( + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" + and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["notes"]) + and event["payload"]["selected_sources"] == ["lexical", "semantic"] + for event in trace_events + if event["kind"] == "context.included" + ) + assert any( + event["payload"]["reason"] == "hybrid_artifact_chunk_limit_exceeded" + and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["weak"]) + and event["payload"]["selected_sources"] == ["lexical", "semantic"] + and event["payload"]["score"] == 0.0 + for event in trace_events + if event["kind"] == "context.excluded" + ) + assert trace_events[-1]["payload"]["artifact_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_retrieval_scope_kind"] == "task" + assert trace_events[-1]["payload"]["artifact_lexical_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_semantic_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_lexical_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_semantic_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_merged_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_deduplicated_count"] == 3 + assert trace_events[-1]["payload"]["included_artifact_chunk_count"] == 2 + assert trace_events[-1]["payload"]["included_dual_source_artifact_chunk_count"] == 2 + assert trace_events[-1]["payload"]["excluded_artifact_chunk_limit_count"] == 1 + assert trace_events[-1]["payload"]["excluded_uningested_artifact_count"] == 1 + + def test_compile_context_semantic_artifact_retrieval_integrates_chunks_traces_and_exclusion_rules( migrated_database_urls, monkeypatch, @@ -1242,7 +1561,7 @@ def test_compile_context_semantic_artifact_retrieval_integrates_chunks_traces_an ) assert status_code == 200 - assert payload["context_pack"]["semantic_artifact_chunks"] == [ + assert payload["context_pack"]["artifact_chunks"] == [ { "id": str(artifact_scope["chunk_ids"]["docs"]), "task_id": str(artifact_scope["task_id"]), @@ -1253,7 +1572,11 @@ def test_compile_context_semantic_artifact_retrieval_integrates_chunks_traces_an "char_start": 0, "char_end_exclusive": 14, "text": "beta alpha doc", - "score": 1.0, + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": 1.0, + }, }, { "id": str(artifact_scope["chunk_ids"]["notes"]), @@ -1265,31 +1588,63 @@ def test_compile_context_semantic_artifact_retrieval_integrates_chunks_traces_an "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "score": 1.0, + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": 1.0, + }, }, ] - assert payload["context_pack"]["semantic_artifact_chunk_summary"] == { + assert payload["context_pack"]["artifact_chunk_summary"] == { "requested": True, + "lexical_requested": False, + "semantic_requested": True, "scope": {"kind": "task", "task_id": str(artifact_scope["task_id"])}, + "query": None, + "query_terms": [], "embedding_config_id": str(config_id), "query_vector_dimensions": 3, "limit": 2, + "lexical_limit": 0, + "semantic_limit": 2, "searched_artifact_count": 3, - "candidate_count": 3, + "lexical_candidate_count": 0, + "semantic_candidate_count": 3, + "merged_candidate_count": 3, + "deduplicated_count": 0, "included_count": 2, + "included_lexical_only_count": 0, + "included_semantic_only_count": 2, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 1, "excluded_limit_count": 1, + "matching_rule": None, "similarity_metric": "cosine_similarity", - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } - assert payload["context_pack"]["artifact_chunks"] == [] trace_id = UUID(payload["trace_id"]) with user_connection(migrated_database_urls["app"], seeded["user_id"]) as conn: trace_events = ContinuityStore(conn).list_trace_events(trace_id) assert any( - event["payload"]["reason"] == "within_semantic_artifact_chunk_limit" + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["docs"]) and event["payload"]["relative_path"] == "docs/a.txt" and event["payload"]["score"] == 1.0 @@ -1297,14 +1652,14 @@ def test_compile_context_semantic_artifact_retrieval_integrates_chunks_traces_an if event["kind"] == "context.included" ) assert any( - event["payload"]["reason"] == "within_semantic_artifact_chunk_limit" + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["notes"]) and event["payload"]["relative_path"] == "notes/b.md" for event in trace_events if event["kind"] == "context.included" ) assert any( - event["payload"]["reason"] == "semantic_artifact_chunk_limit_exceeded" + event["payload"]["reason"] == "hybrid_artifact_chunk_limit_exceeded" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["weak"]) and event["payload"]["relative_path"] == "notes/c.txt" and event["payload"]["score"] == 0.0 @@ -1312,19 +1667,25 @@ def test_compile_context_semantic_artifact_retrieval_integrates_chunks_traces_an if event["kind"] == "context.excluded" ) assert any( - event["payload"]["reason"] == "semantic_artifact_not_ingested" + event["payload"]["reason"] == "hybrid_artifact_not_ingested" and event["payload"]["entity_id"] == str(artifact_scope["artifact_ids"]["pending"]) and event["payload"]["relative_path"] == "notes/hidden.txt" and event["payload"]["ingestion_status"] == "pending" for event in trace_events if event["kind"] == "context.excluded" ) - assert trace_events[-1]["payload"]["semantic_artifact_retrieval_requested"] is True - assert trace_events[-1]["payload"]["semantic_artifact_retrieval_scope_kind"] == "task" - assert trace_events[-1]["payload"]["semantic_artifact_chunk_candidate_count"] == 3 - assert trace_events[-1]["payload"]["included_semantic_artifact_chunk_count"] == 2 - assert trace_events[-1]["payload"]["excluded_semantic_artifact_chunk_limit_count"] == 1 - assert trace_events[-1]["payload"]["excluded_semantic_uningested_artifact_count"] == 1 + assert trace_events[-1]["payload"]["artifact_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_retrieval_scope_kind"] == "task" + assert trace_events[-1]["payload"]["artifact_lexical_retrieval_requested"] is False + assert trace_events[-1]["payload"]["artifact_semantic_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_lexical_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_semantic_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_merged_candidate_count"] == 3 + assert trace_events[-1]["payload"]["artifact_deduplicated_count"] == 0 + assert trace_events[-1]["payload"]["included_artifact_chunk_count"] == 2 + assert trace_events[-1]["payload"]["included_dual_source_artifact_chunk_count"] == 0 + assert trace_events[-1]["payload"]["excluded_artifact_chunk_limit_count"] == 1 + assert trace_events[-1]["payload"]["excluded_uningested_artifact_count"] == 1 def test_compile_context_semantic_artifact_scoped_retrieval_returns_only_visible_artifact_chunks( @@ -1369,7 +1730,7 @@ def test_compile_context_semantic_artifact_scoped_retrieval_returns_only_visible ) assert status_code == 200 - assert payload["context_pack"]["semantic_artifact_chunks"] == [ + assert payload["context_pack"]["artifact_chunks"] == [ { "id": str(artifact_scope["chunk_ids"]["notes"]), "task_id": str(artifact_scope["task_id"]), @@ -1380,26 +1741,59 @@ def test_compile_context_semantic_artifact_scoped_retrieval_returns_only_visible "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "score": 1.0, + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": 1.0, + }, } ] - assert payload["context_pack"]["semantic_artifact_chunk_summary"] == { + assert payload["context_pack"]["artifact_chunk_summary"] == { "requested": True, + "lexical_requested": False, + "semantic_requested": True, "scope": { "kind": "artifact", "task_id": str(artifact_scope["task_id"]), "task_artifact_id": str(artifact_scope["artifact_ids"]["notes"]), }, + "query": None, + "query_terms": [], "embedding_config_id": str(config_id), "query_vector_dimensions": 3, "limit": 2, + "lexical_limit": 0, + "semantic_limit": 2, "searched_artifact_count": 1, - "candidate_count": 1, + "lexical_candidate_count": 0, + "semantic_candidate_count": 1, + "merged_candidate_count": 1, + "deduplicated_count": 0, "included_count": 1, + "included_lexical_only_count": 0, + "included_semantic_only_count": 1, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, + "matching_rule": None, "similarity_metric": "cosine_similarity", - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } trace_id = UUID(payload["trace_id"]) @@ -1407,19 +1801,25 @@ def test_compile_context_semantic_artifact_scoped_retrieval_returns_only_visible trace_events = ContinuityStore(conn).list_trace_events(trace_id) assert any( - event["payload"]["reason"] == "within_semantic_artifact_chunk_limit" + event["payload"]["reason"] == "within_hybrid_artifact_chunk_limit" and event["payload"]["entity_id"] == str(artifact_scope["chunk_ids"]["notes"]) and event["payload"]["scope_kind"] == "artifact" and event["payload"]["task_artifact_id"] == str(artifact_scope["artifact_ids"]["notes"]) for event in trace_events if event["kind"] == "context.included" ) - assert trace_events[-1]["payload"]["semantic_artifact_retrieval_requested"] is True - assert trace_events[-1]["payload"]["semantic_artifact_retrieval_scope_kind"] == "artifact" - assert trace_events[-1]["payload"]["semantic_artifact_chunk_candidate_count"] == 1 - assert trace_events[-1]["payload"]["included_semantic_artifact_chunk_count"] == 1 - assert trace_events[-1]["payload"]["excluded_semantic_artifact_chunk_limit_count"] == 0 - assert trace_events[-1]["payload"]["excluded_semantic_uningested_artifact_count"] == 0 + assert trace_events[-1]["payload"]["artifact_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_retrieval_scope_kind"] == "artifact" + assert trace_events[-1]["payload"]["artifact_lexical_retrieval_requested"] is False + assert trace_events[-1]["payload"]["artifact_semantic_retrieval_requested"] is True + assert trace_events[-1]["payload"]["artifact_lexical_candidate_count"] == 0 + assert trace_events[-1]["payload"]["artifact_semantic_candidate_count"] == 1 + assert trace_events[-1]["payload"]["artifact_merged_candidate_count"] == 1 + assert trace_events[-1]["payload"]["artifact_deduplicated_count"] == 0 + assert trace_events[-1]["payload"]["included_artifact_chunk_count"] == 1 + assert trace_events[-1]["payload"]["included_dual_source_artifact_chunk_count"] == 0 + assert trace_events[-1]["payload"]["excluded_artifact_chunk_limit_count"] == 0 + assert trace_events[-1]["payload"]["excluded_uningested_artifact_count"] == 0 def test_compile_context_semantic_artifact_retrieval_validation_and_isolation( diff --git a/tests/unit/test_compiler.py b/tests/unit/test_compiler.py index 7ff19c9..8267391 100644 --- a/tests/unit/test_compiler.py +++ b/tests/unit/test_compiler.py @@ -7,10 +7,10 @@ SUMMARY_TRACE_EVENT_KIND, _compile_artifact_chunk_section, _compile_memory_section, - _compile_semantic_artifact_chunk_section, compile_continuity_context, ) from alicebot_api.contracts import ( + CompileContextArtifactScopedArtifactRetrievalInput, CompileContextArtifactScopedSemanticArtifactRetrievalInput, CompileContextSemanticRetrievalInput, CompileContextTaskScopedSemanticArtifactRetrievalInput, @@ -298,38 +298,46 @@ def test_compile_continuity_context_is_deterministic_and_stably_ordered() -> Non assert first_run.context_pack["artifact_chunks"] == [] assert first_run.context_pack["artifact_chunk_summary"] == { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, "query": None, "query_terms": [], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": None, + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], - } - assert first_run.context_pack["semantic_artifact_chunks"] == [] - assert first_run.context_pack["semantic_artifact_chunk_summary"] == { - "requested": False, - "scope": None, - "embedding_config_id": None, - "query_vector_dimensions": 0, - "limit": 0, - "searched_artifact_count": 0, - "candidate_count": 0, - "included_count": 0, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } assert first_run.context_pack["entity_summary"] == { "candidate_count": 3, @@ -643,38 +651,46 @@ def test_compile_continuity_context_records_included_and_excluded_reasons() -> N assert compiler_run.context_pack["artifact_chunks"] == [] assert compiler_run.context_pack["artifact_chunk_summary"] == { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, "query": None, "query_terms": [], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": None, + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], - } - assert compiler_run.context_pack["semantic_artifact_chunks"] == [] - assert compiler_run.context_pack["semantic_artifact_chunk_summary"] == { - "requested": False, - "scope": None, - "embedding_config_id": None, - "query_vector_dimensions": 0, - "limit": 0, - "searched_artifact_count": 0, - "candidate_count": 0, - "included_count": 0, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } assert compiler_run.context_pack["entities"] == [ { @@ -710,18 +726,16 @@ def test_compile_continuity_context_records_included_and_excluded_reasons() -> N assert compiler_run.trace_events[-1].payload["hybrid_memory_merged_candidate_count"] == 1 assert compiler_run.trace_events[-1].payload["hybrid_memory_deduplicated_count"] == 0 assert compiler_run.trace_events[-1].payload["artifact_retrieval_requested"] is False - assert compiler_run.trace_events[-1].payload["artifact_chunk_candidate_count"] == 0 + assert compiler_run.trace_events[-1].payload["artifact_lexical_retrieval_requested"] is False + assert compiler_run.trace_events[-1].payload["artifact_semantic_retrieval_requested"] is False + assert compiler_run.trace_events[-1].payload["artifact_lexical_candidate_count"] == 0 + assert compiler_run.trace_events[-1].payload["artifact_semantic_candidate_count"] == 0 + assert compiler_run.trace_events[-1].payload["artifact_merged_candidate_count"] == 0 + assert compiler_run.trace_events[-1].payload["artifact_deduplicated_count"] == 0 assert compiler_run.trace_events[-1].payload["included_artifact_chunk_count"] == 0 + assert compiler_run.trace_events[-1].payload["included_dual_source_artifact_chunk_count"] == 0 assert compiler_run.trace_events[-1].payload["excluded_artifact_chunk_limit_count"] == 0 assert compiler_run.trace_events[-1].payload["excluded_uningested_artifact_count"] == 0 - assert compiler_run.trace_events[-1].payload["semantic_artifact_retrieval_requested"] is False - assert compiler_run.trace_events[-1].payload["semantic_artifact_chunk_candidate_count"] == 0 - assert compiler_run.trace_events[-1].payload["included_semantic_artifact_chunk_count"] == 0 - assert ( - compiler_run.trace_events[-1].payload["excluded_semantic_artifact_chunk_limit_count"] - == 0 - ) - assert compiler_run.trace_events[-1].payload["excluded_semantic_uningested_artifact_count"] == 0 class SemanticCompileStoreStub: @@ -994,6 +1008,7 @@ def test_compile_artifact_chunk_section_orders_limits_and_excludes_non_ingested( query="Alpha beta", limit=2, ), + semantic_artifact_retrieval=None, ) assert artifact_section.items == [ @@ -1007,10 +1022,14 @@ def test_compile_artifact_chunk_section_orders_limits_and_excludes_non_ingested( "char_start": 0, "char_end_exclusive": 14, "text": "beta alpha doc", - "match": { - "matched_query_terms": ["alpha", "beta"], - "matched_query_term_count": 2, - "first_match_char_start": 0, + "source_provenance": { + "sources": ["lexical"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": None, }, }, { @@ -1023,48 +1042,76 @@ def test_compile_artifact_chunk_section_orders_limits_and_excludes_non_ingested( "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "match": { - "matched_query_terms": ["alpha", "beta"], - "matched_query_term_count": 2, - "first_match_char_start": 0, + "source_provenance": { + "sources": ["lexical"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": None, }, }, ] assert artifact_section.summary == { "requested": True, + "lexical_requested": True, + "semantic_requested": False, "scope": {"kind": "task", "task_id": str(store.task_id)}, "query": "Alpha beta", "query_terms": ["alpha", "beta"], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 2, + "lexical_limit": 2, + "semantic_limit": 0, "searched_artifact_count": 3, - "candidate_count": 3, + "lexical_candidate_count": 3, + "semantic_candidate_count": 0, + "merged_candidate_count": 3, + "deduplicated_count": 0, "included_count": 2, + "included_lexical_only_count": 2, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 1, "excluded_limit_count": 1, - "order": [ + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } assert [decision.reason for decision in artifact_section.decisions] == [ - "artifact_not_ingested", - "within_artifact_chunk_limit", - "within_artifact_chunk_limit", - "artifact_chunk_limit_exceeded", + "hybrid_artifact_not_ingested", + "within_hybrid_artifact_chunk_limit", + "within_hybrid_artifact_chunk_limit", + "hybrid_artifact_chunk_limit_exceeded", ] assert artifact_section.decisions[0].metadata["relative_path"] == "notes/hidden.txt" assert artifact_section.decisions[-1].metadata["relative_path"] == "notes/c.txt" -def test_compile_semantic_artifact_chunk_section_orders_limits_and_excludes_non_ingested() -> None: +def test_compile_artifact_chunk_section_supports_semantic_only_scope() -> None: store = ArtifactCompileStoreStub() - semantic_artifact_section = _compile_semantic_artifact_chunk_section( + artifact_section = _compile_artifact_chunk_section( store, # type: ignore[arg-type] + artifact_retrieval=None, semantic_artifact_retrieval=CompileContextTaskScopedSemanticArtifactRetrievalInput( task_id=store.task_id, embedding_config_id=store.config_id, @@ -1073,7 +1120,7 @@ def test_compile_semantic_artifact_chunk_section_orders_limits_and_excludes_non_ ), ) - assert semantic_artifact_section.items == [ + assert artifact_section.items == [ { "id": str(store.chunk_ids[0]), "task_id": str(store.task_id), @@ -1084,7 +1131,11 @@ def test_compile_semantic_artifact_chunk_section_orders_limits_and_excludes_non_ "char_start": 0, "char_end_exclusive": 14, "text": "beta alpha doc", - "score": 1.0, + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": 1.0, + }, }, { "id": str(store.chunk_ids[1]), @@ -1096,38 +1147,76 @@ def test_compile_semantic_artifact_chunk_section_orders_limits_and_excludes_non_ "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "score": 1.0, + "source_provenance": { + "sources": ["semantic"], + "lexical_match": None, + "semantic_score": 1.0, + }, }, ] - assert semantic_artifact_section.summary == { + assert artifact_section.summary == { "requested": True, + "lexical_requested": False, + "semantic_requested": True, "scope": {"kind": "task", "task_id": str(store.task_id)}, + "query": None, + "query_terms": [], "embedding_config_id": str(store.config_id), "query_vector_dimensions": 3, "limit": 2, + "lexical_limit": 0, + "semantic_limit": 2, "searched_artifact_count": 3, - "candidate_count": 3, + "lexical_candidate_count": 0, + "semantic_candidate_count": 3, + "merged_candidate_count": 3, + "deduplicated_count": 0, "included_count": 2, + "included_lexical_only_count": 0, + "included_semantic_only_count": 2, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 1, "excluded_limit_count": 1, + "matching_rule": None, "similarity_metric": "cosine_similarity", - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } - assert [decision.reason for decision in semantic_artifact_section.decisions] == [ - "semantic_artifact_not_ingested", - "within_semantic_artifact_chunk_limit", - "within_semantic_artifact_chunk_limit", - "semantic_artifact_chunk_limit_exceeded", + assert [decision.reason for decision in artifact_section.decisions] == [ + "hybrid_artifact_not_ingested", + "within_hybrid_artifact_chunk_limit", + "within_hybrid_artifact_chunk_limit", + "hybrid_artifact_chunk_limit_exceeded", ] - assert semantic_artifact_section.decisions[0].metadata["relative_path"] == "notes/hidden.txt" - assert semantic_artifact_section.decisions[-1].metadata["relative_path"] == "notes/c.txt" + assert artifact_section.decisions[0].metadata["relative_path"] == "notes/hidden.txt" + assert artifact_section.decisions[-1].metadata["relative_path"] == "notes/c.txt" -def test_compile_semantic_artifact_chunk_section_supports_artifact_scope() -> None: +def test_compile_artifact_chunk_section_merges_dual_source_provenance_for_artifact_scope() -> None: store = ArtifactCompileStoreStub() - semantic_artifact_section = _compile_semantic_artifact_chunk_section( + artifact_section = _compile_artifact_chunk_section( store, # type: ignore[arg-type] + artifact_retrieval=CompileContextArtifactScopedArtifactRetrievalInput( + task_artifact_id=store.artifact_ids[1], + query="Alpha beta", + limit=2, + ), semantic_artifact_retrieval=CompileContextArtifactScopedSemanticArtifactRetrievalInput( task_artifact_id=store.artifact_ids[1], embedding_config_id=store.config_id, @@ -1136,7 +1225,7 @@ def test_compile_semantic_artifact_chunk_section_supports_artifact_scope() -> No ), ) - assert semantic_artifact_section.items == [ + assert artifact_section.items == [ { "id": str(store.chunk_ids[1]), "task_id": str(store.task_id), @@ -1147,28 +1236,69 @@ def test_compile_semantic_artifact_chunk_section_supports_artifact_scope() -> No "char_start": 0, "char_end_exclusive": 15, "text": "alpha beta note", - "score": 1.0, + "source_provenance": { + "sources": ["lexical", "semantic"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": 1.0, + }, } ] - assert semantic_artifact_section.summary == { + assert artifact_section.summary == { "requested": True, + "lexical_requested": True, + "semantic_requested": True, "scope": { "kind": "artifact", "task_id": str(store.task_id), "task_artifact_id": str(store.artifact_ids[1]), }, + "query": "Alpha beta", + "query_terms": ["alpha", "beta"], "embedding_config_id": str(store.config_id), "query_vector_dimensions": 3, "limit": 2, + "lexical_limit": 2, + "semantic_limit": 2, "searched_artifact_count": 1, - "candidate_count": 1, + "lexical_candidate_count": 1, + "semantic_candidate_count": 1, + "merged_candidate_count": 1, + "deduplicated_count": 1, "included_count": 1, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 1, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", "similarity_metric": "cosine_similarity", - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ + "matched_query_term_count_desc", + "first_match_char_start_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], } - assert semantic_artifact_section.decisions[0].metadata["scope_kind"] == "artifact" + assert [decision.reason for decision in artifact_section.decisions] == [ + "hybrid_artifact_chunk_deduplicated", + "within_hybrid_artifact_chunk_limit", + ] + assert artifact_section.decisions[1].metadata["selected_sources"] == ["lexical", "semantic"] def test_compile_memory_section_orders_limits_and_excludes_deleted() -> None: diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index c8e63b8..67ad9a8 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -275,38 +275,46 @@ def fake_compile_and_persist_trace( "artifact_chunks": [], "artifact_chunk_summary": { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, "query": None, "query_terms": [], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": None, + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], - }, - "semantic_artifact_chunks": [], - "semantic_artifact_chunk_summary": { - "requested": False, - "scope": None, - "embedding_config_id": None, - "query_vector_dimensions": 0, - "limit": 0, - "searched_artifact_count": 0, - "candidate_count": 0, - "included_count": 0, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], }, "entities": [ { @@ -425,38 +433,46 @@ def fake_compile_and_persist_trace( "artifact_chunks": [], "artifact_chunk_summary": { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, "query": None, "query_terms": [], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": None, + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], - }, - "semantic_artifact_chunks": [], - "semantic_artifact_chunk_summary": { - "requested": False, - "scope": None, - "embedding_config_id": None, - "query_vector_dimensions": 0, - "limit": 0, - "searched_artifact_count": 0, - "candidate_count": 0, - "included_count": 0, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], }, "entities": [ { @@ -639,60 +655,59 @@ def fake_compile_and_persist_trace( "char_start": 0, "char_end_exclusive": 16, "text": "alpha beta spec", - "match": { - "matched_query_terms": ["alpha", "beta"], - "matched_query_term_count": 2, - "first_match_char_start": 0, + "source_provenance": { + "sources": ["lexical", "semantic"], + "lexical_match": { + "matched_query_terms": ["alpha", "beta"], + "matched_query_term_count": 2, + "first_match_char_start": 0, + }, + "semantic_score": 0.99, }, } ], "artifact_chunk_summary": { "requested": True, + "lexical_requested": True, + "semantic_requested": True, "scope": {"kind": "task", "task_id": "task-123"}, "query": "alpha beta", "query_terms": ["alpha", "beta"], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": str(config_id), + "query_vector_dimensions": 3, "limit": 2, + "lexical_limit": 2, + "semantic_limit": 2, "searched_artifact_count": 1, - "candidate_count": 1, + "lexical_candidate_count": 1, + "semantic_candidate_count": 1, + "merged_candidate_count": 1, + "deduplicated_count": 1, "included_count": 1, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 1, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "similarity_metric": "cosine_similarity", + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], - }, - "semantic_artifact_chunks": [ - { - "id": "semantic-chunk-123", - "task_id": "task-123", - "task_artifact_id": "artifact-123", - "relative_path": "docs/spec.txt", - "media_type": "text/plain", - "sequence_no": 1, - "char_start": 0, - "char_end_exclusive": 16, - "text": "alpha beta spec", - "score": 0.99, - } - ], - "semantic_artifact_chunk_summary": { - "requested": True, - "scope": {"kind": "task", "task_id": "task-123"}, - "embedding_config_id": str(config_id), - "query_vector_dimensions": 3, - "limit": 2, - "searched_artifact_count": 1, - "candidate_count": 1, - "included_count": 1, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "similarity_metric": "cosine_similarity", - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], }, "entities": [], "entity_summary": { diff --git a/tests/unit/test_response_generation.py b/tests/unit/test_response_generation.py index 59cbd40..9c6c22c 100644 --- a/tests/unit/test_response_generation.py +++ b/tests/unit/test_response_generation.py @@ -91,38 +91,46 @@ def make_context_pack() -> dict[str, object]: "artifact_chunks": [], "artifact_chunk_summary": { "requested": False, + "lexical_requested": False, + "semantic_requested": False, "scope": None, "query": None, "query_terms": [], - "matching_rule": "casefolded_unicode_word_overlap_unique_query_terms_v1", + "embedding_config_id": None, + "query_vector_dimensions": 0, "limit": 0, + "lexical_limit": 0, + "semantic_limit": 0, "searched_artifact_count": 0, - "candidate_count": 0, + "lexical_candidate_count": 0, + "semantic_candidate_count": 0, + "merged_candidate_count": 0, + "deduplicated_count": 0, "included_count": 0, + "included_lexical_only_count": 0, + "included_semantic_only_count": 0, + "included_dual_source_count": 0, "excluded_uningested_artifact_count": 0, "excluded_limit_count": 0, - "order": [ + "matching_rule": None, + "similarity_metric": None, + "source_precedence": ["lexical", "semantic"], + "lexical_order": [ "matched_query_term_count_desc", "first_match_char_start_asc", "relative_path_asc", "sequence_no_asc", "id_asc", ], - }, - "semantic_artifact_chunks": [], - "semantic_artifact_chunk_summary": { - "requested": False, - "scope": None, - "embedding_config_id": None, - "query_vector_dimensions": 0, - "limit": 0, - "searched_artifact_count": 0, - "candidate_count": 0, - "included_count": 0, - "excluded_uningested_artifact_count": 0, - "excluded_limit_count": 0, - "similarity_metric": None, - "order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "semantic_order": ["score_desc", "relative_path_asc", "sequence_no_asc", "id_asc"], + "merged_order": [ + "source_precedence_asc", + "lexical_rank_asc", + "semantic_rank_asc", + "relative_path_asc", + "sequence_no_asc", + "id_asc", + ], }, "entities": [], "entity_summary": {