From 5df3d59d1b59587daca96dbe5a23e86113c7da7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E7=BF=94=E5=AE=87?= Date: Mon, 2 Feb 2026 02:01:30 +0800 Subject: [PATCH 1/5] feat: add KQL search for completions (#54) Implement a KQL (Kibana Query Language) search engine for querying completion logs. Includes a full lexer/parser/compiler pipeline, aggregation support, time-series histograms, and a search UI integrated into the requests page. Backend: - KQL lexer, parser, and SQL compiler with parameterized queries - Search, histogram, validate, fields, and export API endpoints - Aggregation support (count, avg, sum, min, max, p50/p95/p99) - Time-bucketed histogram queries for visualizations - Field autocomplete with distinct value suggestions Frontend: - Search bar with KQL query input and time range picker - Aggregation results display - CSV/JSON export functionality - Integrated into existing requests page with URL search params Co-Authored-By: Claude Opus 4.5 --- backend/src/api/admin/index.ts | 2 + backend/src/api/admin/search.ts | 243 ++++++ backend/src/db/index.ts | 210 +++++ backend/src/search/__tests__/compiler.test.ts | 353 +++++++++ backend/src/search/__tests__/parser.test.ts | 433 +++++++++++ backend/src/search/compiler.ts | 719 ++++++++++++++++++ backend/src/search/index.ts | 17 + backend/src/search/lexer.ts | 187 +++++ backend/src/search/parser.ts | 337 ++++++++ backend/src/search/types.ts | 133 ++++ frontend/src/pages/requests/search-bar.tsx | 87 +++ .../src/pages/search/aggregation-results.tsx | 56 ++ frontend/src/pages/search/export-button.tsx | 69 ++ frontend/src/pages/search/query-input.tsx | 238 ++++++ .../src/pages/search/search-histogram.tsx | 54 ++ .../src/pages/search/time-range-picker.tsx | 76 ++ frontend/src/routes/requests/index.tsx | 131 +++- 17 files changed, 3339 insertions(+), 6 deletions(-) create mode 100644 backend/src/api/admin/search.ts create mode 100644 backend/src/search/__tests__/compiler.test.ts create mode 100644 backend/src/search/__tests__/parser.test.ts create mode 100644 backend/src/search/compiler.ts create mode 100644 backend/src/search/index.ts create mode 100644 backend/src/search/lexer.ts create mode 100644 backend/src/search/parser.ts create mode 100644 backend/src/search/types.ts create mode 100644 frontend/src/pages/requests/search-bar.tsx create mode 100644 frontend/src/pages/search/aggregation-results.tsx create mode 100644 frontend/src/pages/search/export-button.tsx create mode 100644 frontend/src/pages/search/query-input.tsx create mode 100644 frontend/src/pages/search/search-histogram.tsx create mode 100644 frontend/src/pages/search/time-range-picker.tsx diff --git a/backend/src/api/admin/index.ts b/backend/src/api/admin/index.ts index 26a7db5..1e6724c 100644 --- a/backend/src/api/admin/index.ts +++ b/backend/src/api/admin/index.ts @@ -10,6 +10,7 @@ import { adminEmbeddings } from "./embeddings"; import { adminModels } from "./models"; import { adminProviders } from "./providers"; import { adminRateLimits } from "./rateLimits"; +import { adminSearch } from "./search"; import { adminSettings } from "./settings"; import { adminStats } from "./stats"; import { adminUpstream } from "./upstream"; @@ -34,6 +35,7 @@ export const routes = new Elysia({ .use(adminModels) .use(adminEmbeddings) .use(adminStats) + .use(adminSearch) .use(adminSettings) .use(adminDashboards) .use(adminGrafana) diff --git a/backend/src/api/admin/search.ts b/backend/src/api/admin/search.ts new file mode 100644 index 0000000..24938cb --- /dev/null +++ b/backend/src/api/admin/search.ts @@ -0,0 +1,243 @@ +import { Elysia, t } from "elysia"; +import { + parseKql, + compileSearch, + getSearchableFields, +} from "@/search"; +import { + searchCompletions, + aggregateCompletions, + searchCompletionsTimeSeries, + getDistinctFieldValues, +} from "@/db"; + +function parseTimeRange( + from?: string, + to?: string, +): { from: Date; to: Date } | undefined { + if (!from && !to) { + return undefined; + } + return { + from: from ? new Date(from) : new Date(Date.now() - 3600_000), // default: 1h ago + to: to ? new Date(to) : new Date(), + }; +} + +export const adminSearch = new Elysia() + // Search completions + .post( + "/search", + async ({ body, status }) => { + const result = parseKql(body.query); + if (!result.success) { + return status(400, { + error: "Invalid query", + details: result.error, + }); + } + + const timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + const compiled = compileSearch(result.query, { timeRange }); + + // If the query has aggregation, return aggregation results + if (compiled.aggregation) { + try { + const results = await aggregateCompletions(compiled); + return { type: "aggregation" as const, results }; + } catch (err) { + return status(500, { + error: "Aggregation failed", + details: String(err), + }); + } + } + + // Otherwise, return paginated document results + try { + const data = await searchCompletions( + compiled, + body.offset ?? 0, + body.limit ?? 100, + ); + // Truncate model names that contain '@' + data.data.forEach((row) => { + if (row.model && row.model.includes("@")) { + row.model = row.model.split("@", 2)[0]!; + } + }); + return { type: "documents" as const, ...data }; + } catch (err) { + return status(500, { + error: "Search failed", + details: String(err), + }); + } + }, + { + body: t.Object({ + query: t.String({ maxLength: 2000 }), + timeRange: t.Optional( + t.Object({ + from: t.Optional(t.String()), + to: t.Optional(t.String()), + }), + ), + offset: t.Optional(t.Integer({ minimum: 0 })), + limit: t.Optional(t.Integer({ minimum: 1, maximum: 500 })), + }), + }, + ) + // Search histogram (time series) + .post( + "/search/histogram", + async ({ body, status }) => { + const result = parseKql(body.query); + if (!result.success) { + return status(400, { + error: "Invalid query", + details: result.error, + }); + } + + const timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + const compiled = compileSearch(result.query, { timeRange }); + + try { + const buckets = await searchCompletionsTimeSeries( + compiled, + body.bucketSeconds ?? 60, + ); + return { buckets }; + } catch (err) { + return status(500, { + error: "Histogram query failed", + details: String(err), + }); + } + }, + { + body: t.Object({ + query: t.String({ maxLength: 2000 }), + timeRange: t.Optional( + t.Object({ + from: t.Optional(t.String()), + to: t.Optional(t.String()), + }), + ), + bucketSeconds: t.Optional(t.Integer({ minimum: 1 })), + }), + }, + ) + // Validate KQL query + .post( + "/search/validate", + ({ body }) => { + const result = parseKql(body.query); + if (result.success) { + return { valid: true, hasAggregation: !!result.query.aggregation }; + } + return { valid: false, error: result.error }; + }, + { + body: t.Object({ + query: t.String({ maxLength: 2000 }), + }), + }, + ) + // Get searchable fields (for autocomplete) + .get("/search/fields", async () => { + const fields = getSearchableFields(); + + // Enrich with distinct values for key fields + const modelValues = await getDistinctFieldValues("model"); + + return { + fields: fields.map((f) => { + if (f.name === "model") { + return Object.assign({}, f, { values: modelValues }); + } + return f; + }), + }; + }) + // Export search results + .post( + "/search/export", + async ({ body, status, set }) => { + const result = parseKql(body.query); + if (!result.success) { + return status(400, { + error: "Invalid query", + details: result.error, + }); + } + + const timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + const compiled = compileSearch(result.query, { timeRange }); + + try { + // Fetch all results (up to 10000 for export) + const data = await searchCompletions(compiled, 0, 10000); + + if (body.format === "csv") { + set.headers["content-type"] = "text/csv"; + set.headers["content-disposition"] = + 'attachment; filename="search-results.csv"'; + + const headers = [ + "id", + "model", + "status", + "duration", + "ttft", + "prompt_tokens", + "completion_tokens", + "created_at", + "provider_name", + "api_format", + "rating", + ]; + const rows = data.data.map((row) => + [ + row.id, + `"${(row.model || "").replace(/"/g, '""')}"`, + row.status, + row.duration, + row.ttft, + row.prompt_tokens, + row.completion_tokens, + row.created_at, + `"${(row.provider_name || "").replace(/"/g, '""')}"`, + row.api_format || "", + row.rating ?? "", + ].join(","), + ); + return [headers.join(","), ...rows].join("\n"); + } + + // JSON format + set.headers["content-type"] = "application/json"; + set.headers["content-disposition"] = + 'attachment; filename="search-results.json"'; + return JSON.stringify(data.data, null, 2); + } catch (err) { + return status(500, { + error: "Export failed", + details: String(err), + }); + } + }, + { + body: t.Object({ + query: t.String({ maxLength: 2000 }), + timeRange: t.Optional( + t.Object({ + from: t.Optional(t.String()), + to: t.Optional(t.String()), + }), + ), + format: t.Union([t.Literal("csv"), t.Literal("json")]), + }), + }, + ); diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index 9cde582..836c5ff 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -1925,3 +1925,213 @@ export async function updateAlertChannelGrafanaSync( .set(fields) .where(eq(schema.AlertChannelsTable.id, id)); } + +// ============================================ +// KQL Search Operations +// ============================================ + +import type { CompiledQuery } from "@/search/types"; + +/** + * Convert a compiled KQL query (string with $N placeholders + params array) + * into a Drizzle SQL object with proper parameterization. + */ +function buildDrizzleSql(template: string, params: unknown[]) { + // Split on $N placeholders, interleave raw SQL with parameterized values + const parts = template.split(/\$(\d+)/); + const chunks: ReturnType[] = []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (i % 2 === 0) { + // Raw SQL structure (from trusted field whitelist) + if (part) { + chunks.push(sql.raw(part)); + } + } else if (part) { + // Parameter value (user-provided, properly parameterized) + const paramIdx = parseInt(part) - 1; + const value = params[paramIdx]; + chunks.push(sql`${value}`); + } + } + return sql.join(chunks, sql.raw("")); +} + +export type SearchCompletionRow = { + id: number; + api_key_id: number; + model: string; + status: string; + duration: number; + ttft: number; + prompt_tokens: number; + completion_tokens: number; + created_at: Date; + updated_at: Date; + rating: number | null; + req_id: string | null; + api_format: string | null; + prompt: unknown; + completion: unknown; + provider_name: string | null; +}; + +/** + * Execute a compiled KQL search query with pagination. + */ +export async function searchCompletions( + compiled: CompiledQuery, + offset: number, + limit: number, +): Promise<{ data: SearchCompletionRow[]; total: number; from: number }> { + logger.debug("searchCompletions", compiled.whereClause, offset, limit); + + const whereSql = buildDrizzleSql(compiled.whereClause, compiled.params); + + // Get paginated results with provider join + const result = await db.execute(sql` + SELECT + c.id, c.api_key_id, c.model, c.status, c.duration, c.ttft, + c.prompt_tokens, c.completion_tokens, c.created_at, c.updated_at, + c.rating, c.req_id, c.api_format, c.prompt, c.completion, + p.name AS provider_name + FROM completions c + LEFT JOIN models m ON c.model_id = m.id + LEFT JOIN providers p ON m.provider_id = p.id + WHERE ${whereSql} + ORDER BY c.id DESC + OFFSET ${offset} + LIMIT ${limit} + `); + + // Get total count + const countResult = await db.execute(sql` + SELECT COUNT(*) AS total + FROM completions c + LEFT JOIN models m ON c.model_id = m.id + LEFT JOIN providers p ON m.provider_id = p.id + WHERE ${whereSql} + `); + + const total = Number( + (countResult as unknown as { total: string }[])[0]?.total ?? 0, + ); + + return { + data: result as unknown as SearchCompletionRow[], + total, + from: offset, + }; +} + +/** + * Execute a compiled KQL aggregation query. + */ +export async function aggregateCompletions( + compiled: CompiledQuery, +): Promise[]> { + if (!compiled.aggregation) { + throw new Error("No aggregation defined in compiled query"); + } + + logger.debug("aggregateCompletions", compiled.whereClause); + + const whereSql = buildDrizzleSql(compiled.whereClause, compiled.params); + const { selectExpressions, groupByColumn, groupByField } = + compiled.aggregation; + + // Build SELECT clause + const selectParts: string[] = []; + if (groupByColumn && groupByField) { + selectParts.push(`${groupByColumn} AS "${groupByField}"`); + } + for (const expr of selectExpressions) { + selectParts.push(`${expr.sql} AS "${expr.alias}"`); + } + const selectClause = selectParts.join(", "); + const groupBySql = groupByColumn + ? sql.raw(`GROUP BY ${groupByColumn} ORDER BY COUNT(*) DESC NULLS LAST`) + : sql.raw(""); + + const result = await db.execute( + sql`SELECT ${sql.raw(selectClause)} + FROM completions c + LEFT JOIN models m ON c.model_id = m.id + LEFT JOIN providers p ON m.provider_id = p.id + WHERE ${whereSql} + ${groupBySql} + LIMIT 1000`, + ); + + return result as unknown as Record[]; +} + +/** + * Execute a compiled KQL search query and return time-bucketed histogram data. + */ +export async function searchCompletionsTimeSeries( + compiled: CompiledQuery, + bucketSeconds: number, +): Promise< + { + bucket: Date; + total: string; + completed: string; + failed: string; + }[] +> { + logger.debug("searchCompletionsTimeSeries", compiled.whereClause, bucketSeconds); + + const whereSql = buildDrizzleSql(compiled.whereClause, compiled.params); + + const result = await db.execute(sql` + SELECT + to_timestamp(floor(extract(epoch from c.created_at) / ${bucketSeconds}) * ${bucketSeconds}) AS bucket, + COUNT(*) AS total, + SUM(CASE WHEN c.status = 'completed' THEN 1 ELSE 0 END) AS completed, + SUM(CASE WHEN c.status = 'failed' THEN 1 ELSE 0 END) AS failed + FROM completions c + LEFT JOIN models m ON c.model_id = m.id + LEFT JOIN providers p ON m.provider_id = p.id + WHERE ${whereSql} + GROUP BY bucket + ORDER BY bucket ASC + `); + + return result as unknown as { + bucket: Date; + total: string; + completed: string; + failed: string; + }[]; +} + +/** + * Get distinct values for a field (for autocomplete suggestions). + */ +export async function getDistinctFieldValues( + column: string, + maxResults = 50, +): Promise { + // Only allow known safe column expressions + const SAFE_COLUMNS: Record = { + model: "model", + status: "status", + api_format: "api_format", + }; + + const safeColumn = SAFE_COLUMNS[column]; + if (!safeColumn) { + return []; + } + + const result = await db.execute(sql` + SELECT DISTINCT ${sql.raw(safeColumn)} AS value + FROM completions + WHERE deleted = false AND ${sql.raw(safeColumn)} IS NOT NULL + ORDER BY value ASC + LIMIT ${maxResults} + `); + + return (result as unknown as { value: string }[]).map((r) => r.value); +} diff --git a/backend/src/search/__tests__/compiler.test.ts b/backend/src/search/__tests__/compiler.test.ts new file mode 100644 index 0000000..693e47f --- /dev/null +++ b/backend/src/search/__tests__/compiler.test.ts @@ -0,0 +1,353 @@ +import { describe, expect, test } from "bun:test"; +import { parseKql } from "../parser"; +import { compileSearch } from "../compiler"; + +function compile(input: string, options?: { timeRange?: { from: Date; to: Date } }) { + const result = parseKql(input); + if (!result.success) { + throw new Error(`Parse error: ${result.error.message}`); + } + return compileSearch(result.query, options); +} + +describe("SQL Compiler", () => { + describe("basic filters", () => { + test("empty query produces only deleted=false filter", () => { + const compiled = compileSearch({}); + expect(compiled.whereClause).toBe("c.deleted = false"); + expect(compiled.params).toEqual([]); + }); + + test('model: "gpt-4" compiles to equality', () => { + const compiled = compile('model: "gpt-4"'); + expect(compiled.whereClause).toBe("c.deleted = false AND c.model = $1"); + expect(compiled.params).toEqual(["gpt-4"]); + }); + + test("duration >= 1000 compiles to range comparison", () => { + const compiled = compile("duration >= 1000"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.duration >= $1", + ); + expect(compiled.params).toEqual([1000]); + }); + + test("status: completed validates enum value", () => { + const compiled = compile("status: completed"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.status = $1", + ); + expect(compiled.params).toEqual(["completed"]); + }); + + test("status with invalid enum value throws", () => { + expect(() => compile("status: invalid")).toThrow("Invalid value"); + }); + + test("wildcard compiles to ILIKE", () => { + const compiled = compile("model: *gpt*"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.model ILIKE $1", + ); + expect(compiled.params).toEqual(["%gpt%"]); + }); + + test("enum wildcard casts to text", () => { + const compiled = compile("status: *fail*"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.status::text ILIKE $1", + ); + expect(compiled.params).toEqual(["%fail%"]); + }); + }); + + describe("boolean operators", () => { + test("AND combines with AND", () => { + const compiled = compile( + 'model: "gpt-4" AND status: completed', + ); + expect(compiled.whereClause).toBe( + "c.deleted = false AND (c.model = $1 AND c.status = $2)", + ); + expect(compiled.params).toEqual(["gpt-4", "completed"]); + }); + + test("OR combines with OR", () => { + const compiled = compile( + 'status: failed OR status: aborted', + ); + expect(compiled.whereClause).toBe( + "c.deleted = false AND (c.status = $1 OR c.status = $2)", + ); + expect(compiled.params).toEqual(["failed", "aborted"]); + }); + + test("NOT wraps with NOT", () => { + const compiled = compile("NOT status: pending"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND NOT (c.status = $1)", + ); + expect(compiled.params).toEqual(["pending"]); + }); + + test("grouped expression", () => { + const compiled = compile( + '(status: completed OR status: cache_hit) AND model: *gpt*', + ); + expect(compiled.whereClause).toContain("(c.status = $1 OR c.status = $2)"); + expect(compiled.whereClause).toContain("c.model ILIKE $3"); + expect(compiled.params).toEqual(["completed", "cache_hit", "%gpt%"]); + }); + }); + + describe("timestamp fields", () => { + test("createdAt >= compiles with ::timestamp cast", () => { + const compiled = compile('createdAt >= "2024-01-01"'); + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.created_at >= $1::timestamp", + ); + expect(compiled.params).toEqual(["2024-01-01"]); + }); + }); + + describe("JSONB fields", () => { + test('extraHeaders.x-experiment compiles to JSONB path', () => { + const compiled = compile( + 'extraHeaders.x-experiment: "group_a"', + ); + expect(compiled.whereClause).toBe( + "c.deleted = false AND (c.prompt #>> '{}')::jsonb->'extraHeaders'->>'x-experiment' = $1", + ); + expect(compiled.params).toEqual(["group_a"]); + }); + + test("JSONB wildcard compiles to ILIKE", () => { + const compiled = compile("extraHeaders.x-experiment: *group*"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND (c.prompt #>> '{}')::jsonb->'extraHeaders'->>'x-experiment' ILIKE $1", + ); + expect(compiled.params).toEqual(["%group%"]); + }); + + test("bare JSONB field without path throws", () => { + expect(() => compile('extraHeaders: "value"')).toThrow( + "requires a nested path", + ); + }); + + test("unknown field throws", () => { + expect(() => compile('unknownField: "value"')).toThrow("Unknown field"); + }); + }); + + describe("time range option", () => { + test("time range adds created_at bounds", () => { + const from = new Date("2024-01-01T00:00:00Z"); + const to = new Date("2024-01-31T23:59:59Z"); + const compiled = compile('model: "gpt-4"', { + timeRange: { from, to }, + }); + expect(compiled.whereClause).toContain("c.created_at >= $1"); + expect(compiled.whereClause).toContain("c.created_at <= $2"); + expect(compiled.whereClause).toContain("c.model = $3"); + expect(compiled.params).toEqual([from, to, "gpt-4"]); + }); + + test("time range is skipped when query has explicit createdAt filter", () => { + const from = new Date("2024-01-01T00:00:00Z"); + const to = new Date("2024-01-31T23:59:59Z"); + const compiled = compile('createdAt < "2026-01-30 22:12:18"', { + timeRange: { from, to }, + }); + // Should NOT contain automatic time range params + expect(compiled.whereClause).not.toContain("c.created_at >= "); + expect(compiled.whereClause).not.toContain("c.created_at <= "); + // Should contain only the user's explicit filter + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.created_at < $1::timestamp", + ); + expect(compiled.params).toEqual(["2026-01-30 22:12:18"]); + }); + + test("time range is skipped when createdAt is inside AND/OR", () => { + const from = new Date("2024-01-01T00:00:00Z"); + const to = new Date("2024-01-31T23:59:59Z"); + const compiled = compile( + 'createdAt >= "2025-01-01" AND createdAt < "2025-02-01"', + { timeRange: { from, to } }, + ); + expect(compiled.whereClause).not.toContain("c.created_at <= "); + expect(compiled.params).toEqual(["2025-01-01", "2025-02-01"]); + }); + }); + + describe("aggregation", () => { + test("count() compiles to COUNT(*)", () => { + const compiled = compile("| stats count()"); + expect(compiled.aggregation).toBeDefined(); + expect(compiled.aggregation!.selectExpressions).toEqual([ + { sql: "COUNT(*)", alias: "count" }, + ]); + }); + + test("avg(duration) compiles to AVG()", () => { + const compiled = compile("| stats avg(duration)"); + expect(compiled.aggregation!.selectExpressions).toEqual([ + { sql: "AVG(c.duration)", alias: "avg_duration" }, + ]); + }); + + test("p95(ttft) compiles to percentile_cont", () => { + const compiled = compile("| stats p95(ttft)"); + expect(compiled.aggregation!.selectExpressions).toEqual([ + { + sql: "percentile_cont(0.95) WITHIN GROUP (ORDER BY c.ttft)", + alias: "p95_ttft", + }, + ]); + }); + + test("multiple aggregations", () => { + const compiled = compile( + "| stats avg(duration), count(), p95(ttft)", + ); + expect(compiled.aggregation!.selectExpressions).toHaveLength(3); + }); + + test("GROUP BY compiles correctly", () => { + const compiled = compile("| stats count() by status"); + expect(compiled.aggregation!.groupByColumn).toBe("c.status"); + expect(compiled.aggregation!.groupByField).toBe("status"); + }); + + test("filter + aggregation + group by", () => { + const compiled = compile( + 'model: "gpt-4" | stats avg(duration), count() by status', + ); + expect(compiled.whereClause).toContain("c.model = $1"); + expect(compiled.params).toEqual(["gpt-4"]); + expect(compiled.aggregation!.selectExpressions).toHaveLength(2); + expect(compiled.aggregation!.groupByColumn).toBe("c.status"); + }); + + test("aggregation on unknown field throws", () => { + expect(() => compile("| stats avg(unknown)")).toThrow("Unknown field"); + }); + }); + + describe("parameter ordering", () => { + test("parameters are numbered sequentially", () => { + const compiled = compile( + 'model: "gpt-4" AND duration >= 1000 AND status: completed', + ); + // Should have $1, $2, $3 + expect(compiled.params).toEqual(["gpt-4", 1000, "completed"]); + expect(compiled.whereClause).toContain("$1"); + expect(compiled.whereClause).toContain("$2"); + expect(compiled.whereClause).toContain("$3"); + }); + }); + + describe("provider field", () => { + test("provider compiles to joined p.name", () => { + const compiled = compile('provider: "openai"'); + expect(compiled.whereClause).toBe( + "c.deleted = false AND p.name = $1", + ); + expect(compiled.params).toEqual(["openai"]); + }); + }); + + describe("EXISTS expressions", () => { + test("JSONB root field EXISTS compiles to IS NOT NULL", () => { + const compiled = compile("extraBody EXISTS"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND (c.prompt #>> '{}')::jsonb->'extraBody' IS NOT NULL", + ); + expect(compiled.params).toEqual([]); + }); + + test("toolCalls EXISTS compiles with array element search", () => { + const compiled = compile("toolCalls EXISTS"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND EXISTS (SELECT 1 FROM jsonb_array_elements((c.completion #>> '{}')::jsonb) _elem WHERE _elem->'tool_calls' IS NOT NULL)", + ); + expect(compiled.params).toEqual([]); + }); + + test("nested JSONB EXISTS compiles to path IS NOT NULL", () => { + const compiled = compile("extraHeaders.x-app EXISTS"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND (c.prompt #>> '{}')::jsonb->'extraHeaders'->>'x-app' IS NOT NULL", + ); + expect(compiled.params).toEqual([]); + }); + + test("non-JSONB field EXISTS compiles to IS NOT NULL", () => { + const compiled = compile("rating EXISTS"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND c.rating IS NOT NULL", + ); + expect(compiled.params).toEqual([]); + }); + + test("NOT EXISTS compiles correctly", () => { + const compiled = compile("NOT extraBody EXISTS"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND NOT ((c.prompt #>> '{}')::jsonb->'extraBody' IS NOT NULL)", + ); + expect(compiled.params).toEqual([]); + }); + + test("EXISTS combined with other filters", () => { + const compiled = compile( + 'toolCalls EXISTS AND model: "claude-haiku-*"', + ); + expect(compiled.whereClause).toContain( + "jsonb_array_elements", + ); + expect(compiled.whereClause).toContain("c.model"); + }); + }); + + describe("array-rooted JSONB comparisons", () => { + test("toolCalls.function.name equality", () => { + const compiled = compile('toolCalls.function.name: "calculate"'); + expect(compiled.whereClause).toBe( + "c.deleted = false AND EXISTS (SELECT 1 FROM jsonb_array_elements((c.completion #>> '{}')::jsonb) _msg, jsonb_array_elements(_msg->'tool_calls') _tc WHERE _tc->'function'->>'name' = $1)", + ); + expect(compiled.params).toEqual(["calculate"]); + }); + + test("toolCalls.function.name wildcard", () => { + const compiled = compile("toolCalls.function.name: *calc*"); + expect(compiled.whereClause).toBe( + "c.deleted = false AND EXISTS (SELECT 1 FROM jsonb_array_elements((c.completion #>> '{}')::jsonb) _msg, jsonb_array_elements(_msg->'tool_calls') _tc WHERE _tc->'function'->>'name' ILIKE $1)", + ); + expect(compiled.params).toEqual(["%calc%"]); + }); + + test("toolCalls.type single-level path", () => { + const compiled = compile('toolCalls.type: "function"'); + expect(compiled.whereClause).toBe( + "c.deleted = false AND EXISTS (SELECT 1 FROM jsonb_array_elements((c.completion #>> '{}')::jsonb) _msg, jsonb_array_elements(_msg->'tool_calls') _tc WHERE _tc->>'type' = $1)", + ); + expect(compiled.params).toEqual(["function"]); + }); + + test("bare toolCalls comparison throws", () => { + expect(() => compile('toolCalls: "value"')).toThrow( + "requires a nested path", + ); + }); + + test("combined with other filters", () => { + const compiled = compile( + 'toolCalls.function.name: "calculate" AND status: completed', + ); + expect(compiled.whereClause).toContain("jsonb_array_elements"); + expect(compiled.whereClause).toContain("c.status = $2"); + expect(compiled.params).toEqual(["calculate", "completed"]); + }); + }); +}); diff --git a/backend/src/search/__tests__/parser.test.ts b/backend/src/search/__tests__/parser.test.ts new file mode 100644 index 0000000..81d1184 --- /dev/null +++ b/backend/src/search/__tests__/parser.test.ts @@ -0,0 +1,433 @@ +import { describe, expect, test } from "bun:test"; +import { parseKql } from "../parser"; + +describe("KQL Parser", () => { + describe("empty/whitespace input", () => { + test("empty string returns empty query", () => { + const result = parseKql(""); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query).toEqual({}); + } + }); + + test("whitespace-only returns empty query", () => { + const result = parseKql(" "); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query).toEqual({}); + } + }); + }); + + describe("simple comparisons", () => { + test('model: "gpt-4"', () => { + const result = parseKql('model: "gpt-4"'); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "model", + operator: ":", + value: { type: "string", value: "gpt-4" }, + }); + } + }); + + test("status: completed (unquoted value)", () => { + const result = parseKql("status: completed"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "status", + operator: ":", + value: { type: "string", value: "completed" }, + }); + } + }); + + test("duration >= 1000", () => { + const result = parseKql("duration >= 1000"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "duration", + operator: ">=", + value: { type: "number", value: 1000 }, + }); + } + }); + + test("promptTokens < 5000", () => { + const result = parseKql("promptTokens < 5000"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "promptTokens", + operator: "<", + value: { type: "number", value: 5000 }, + }); + } + }); + + test("rating = 4.5", () => { + const result = parseKql("rating = 4.5"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "rating", + operator: "=", + value: { type: "number", value: 4.5 }, + }); + } + }); + + test("status != failed", () => { + const result = parseKql('status != "failed"'); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "status", + operator: "!=", + value: { type: "string", value: "failed" }, + }); + } + }); + }); + + describe("wildcard values", () => { + test("model: *gpt*", () => { + const result = parseKql("model: *gpt*"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "model", + operator: ":", + value: { type: "wildcard", pattern: "*gpt*" }, + }); + } + }); + + test("model: gpt-*", () => { + const result = parseKql("model: gpt-*"); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter; + expect(filter?.type).toBe("comparison"); + if (filter?.type === "comparison") { + expect(filter.value).toEqual({ type: "wildcard", pattern: "gpt-*" }); + } + } + }); + }); + + describe("boolean operators", () => { + test('model: "gpt-4" AND status: "completed"', () => { + const result = parseKql('model: "gpt-4" AND status: "completed"'); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter!; + expect(filter.type).toBe("and"); + if (filter.type === "and") { + expect(filter.left).toEqual({ + type: "comparison", + field: "model", + operator: ":", + value: { type: "string", value: "gpt-4" }, + }); + expect(filter.right).toEqual({ + type: "comparison", + field: "status", + operator: ":", + value: { type: "string", value: "completed" }, + }); + } + } + }); + + test('status: "failed" OR status: "aborted"', () => { + const result = parseKql('status: "failed" OR status: "aborted"'); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter!; + expect(filter.type).toBe("or"); + } + }); + + test("NOT status: pending", () => { + const result = parseKql("NOT status: pending"); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter!; + expect(filter.type).toBe("not"); + if (filter.type === "not") { + expect(filter.expression.type).toBe("comparison"); + } + } + }); + + test("AND has higher precedence than OR", () => { + // a OR b AND c should parse as a OR (b AND c) + const result = parseKql( + 'status: failed OR model: "gpt-4" AND duration >= 1000', + ); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter!; + expect(filter.type).toBe("or"); + if (filter.type === "or") { + expect(filter.left.type).toBe("comparison"); + expect(filter.right.type).toBe("and"); + } + } + }); + }); + + describe("grouping with parentheses", () => { + test('(status: "completed" OR status: "cache_hit") AND model: *gpt*', () => { + const result = parseKql( + '(status: "completed" OR status: "cache_hit") AND model: *gpt*', + ); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter!; + expect(filter.type).toBe("and"); + if (filter.type === "and") { + expect(filter.left.type).toBe("group"); + if (filter.left.type === "group") { + expect(filter.left.expression.type).toBe("or"); + } + expect(filter.right.type).toBe("comparison"); + } + } + }); + }); + + describe("nested JSONB fields", () => { + test('extraHeaders.x-experiment: "group_a"', () => { + const result = parseKql('extraHeaders.x-experiment: "group_a"'); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "comparison", + field: "extraHeaders.x-experiment", + operator: ":", + value: { type: "string", value: "group_a" }, + }); + } + }); + + test('extraBody.temperature: "0.7"', () => { + const result = parseKql('extraBody.temperature: "0.7"'); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter?.type).toBe("comparison"); + } + }); + }); + + describe("aggregation", () => { + test("| stats count()", () => { + const result = parseKql("| stats count()"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toBeUndefined(); + expect(result.query.aggregation).toEqual({ + functions: [{ fn: "count", field: undefined }], + groupBy: undefined, + }); + } + }); + + test('model: "gpt-4" | stats avg(duration), count(), p95(ttft) by status', () => { + const result = parseKql( + 'model: "gpt-4" | stats avg(duration), count(), p95(ttft) by status', + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter?.type).toBe("comparison"); + expect(result.query.aggregation).toEqual({ + functions: [ + { fn: "avg", field: "duration" }, + { fn: "count", field: undefined }, + { fn: "p95", field: "ttft" }, + ], + groupBy: ["status"], + }); + } + }); + + test("| stats sum(promptTokens), max(duration) by model", () => { + const result = parseKql( + "| stats sum(promptTokens), max(duration) by model", + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.aggregation?.functions).toHaveLength(2); + expect(result.query.aggregation?.groupBy).toEqual(["model"]); + } + }); + }); + + describe("complex queries", () => { + test('(model: "gpt-4" OR model: "claude-3") AND status: "failed"', () => { + const result = parseKql( + '(model: "gpt-4" OR model: "claude-3") AND status: "failed"', + ); + expect(result.success).toBe(true); + }); + + test("duration >= 1000 AND promptTokens < 5000 AND status: completed", () => { + const result = parseKql( + "duration >= 1000 AND promptTokens < 5000 AND status: completed", + ); + expect(result.success).toBe(true); + if (result.success) { + // Should produce nested AND nodes + const filter = result.query.filter!; + expect(filter.type).toBe("and"); + } + }); + + test("filter with aggregation", () => { + const result = parseKql( + 'status: completed AND duration >= 500 | stats avg(duration), p95(ttft), count() by model', + ); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toBeDefined(); + expect(result.query.aggregation).toBeDefined(); + expect(result.query.aggregation?.functions).toHaveLength(3); + expect(result.query.aggregation?.groupBy).toEqual(["model"]); + } + }); + }); + + describe("error cases", () => { + test("missing value after operator", () => { + const result = parseKql("model:"); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("Expected value"); + } + }); + + test("unclosed parenthesis", () => { + const result = parseKql('(status: "failed"'); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("RPAREN"); + } + }); + + test("unterminated string", () => { + const result = parseKql('model: "gpt-4'); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("Unterminated string"); + } + }); + + test("missing operator after field", () => { + const result = parseKql('model "gpt-4"'); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("operator"); + } + }); + + test("unknown aggregate function", () => { + const result = parseKql("| stats unknown(field)"); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("Unknown aggregate function"); + } + }); + + test("count() with field argument", () => { + const result = parseKql("| stats count(duration)"); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("does not take a field"); + } + }); + + test("avg() without field argument", () => { + const result = parseKql("| stats avg()"); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("requires a field"); + } + }); + + test("unexpected token after valid query", () => { + const result = parseKql('model: "gpt-4" extra'); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toContain("Unexpected"); + } + }); + }); + + describe("quoted strings with escapes", () => { + test('escaped quote in string', () => { + const result = parseKql('model: "gpt-\\"4\\""'); + expect(result.success).toBe(true); + if (result.success) { + const filter = result.query.filter; + if (filter?.type === "comparison") { + expect(filter.value).toEqual({ + type: "string", + value: 'gpt-"4"', + }); + } + } + }); + }); + + describe("EXISTS expressions", () => { + test("simple EXISTS parses correctly", () => { + const result = parseKql("extraBody EXISTS"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "exists", + field: "extraBody", + }); + } + }); + + test("nested field EXISTS parses correctly", () => { + const result = parseKql("extraHeaders.x-app EXISTS"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter).toEqual({ + type: "exists", + field: "extraHeaders.x-app", + }); + } + }); + + test("EXISTS combined with AND", () => { + const result = parseKql('extraBody EXISTS AND model: "gpt-4"'); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter?.type).toBe("and"); + } + }); + + test("NOT EXISTS", () => { + const result = parseKql("NOT toolCalls EXISTS"); + expect(result.success).toBe(true); + if (result.success) { + expect(result.query.filter?.type).toBe("not"); + } + }); + }); +}); diff --git a/backend/src/search/compiler.ts b/backend/src/search/compiler.ts new file mode 100644 index 0000000..18f015c --- /dev/null +++ b/backend/src/search/compiler.ts @@ -0,0 +1,719 @@ +import type { + KqlQuery, + KqlExpression, + KqlValue, + ComparisonOperator, + AggregateExpression, + CompiledQuery, + FieldInfo, + FieldType, +} from "./types"; + +// --- Field registry --- + +interface FieldMapping { + /** SQL column expression (using table alias "c" for completions, "p" for providers) */ + column: string; + type: FieldType; + description: string; + /** Enum values for validation */ + values?: string[]; + /** Whether this field supports nested path access (JSONB) */ + nested?: boolean; + /** The JSONB root column for nested fields */ + jsonbColumn?: string; + /** The root key inside the JSONB column */ + jsonbRootKey?: string; + /** Whether the JSONB root value is an array (key lives inside array elements) */ + jsonbRootIsArray?: boolean; +} + +const COMPLETIONS_STATUS_VALUES = [ + "pending", + "completed", + "failed", + "aborted", + "cache_hit", +]; + +const API_FORMAT_VALUES = ["openai-chat", "openai-responses", "anthropic"]; + +const FIELD_REGISTRY: Record = { + id: { + column: "c.id", + type: "number", + description: "Completion record ID", + }, + model: { + column: "c.model", + type: "text", + description: "Model name", + }, + status: { + column: "c.status", + type: "enum", + description: "Request status", + values: COMPLETIONS_STATUS_VALUES, + }, + duration: { + column: "c.duration", + type: "number", + description: "Total request duration (ms)", + }, + ttft: { + column: "c.ttft", + type: "number", + description: "Time to first token (ms)", + }, + promptTokens: { + column: "c.prompt_tokens", + type: "number", + description: "Input token count", + }, + completionTokens: { + column: "c.completion_tokens", + type: "number", + description: "Output token count", + }, + apiKeyId: { + column: "c.api_key_id", + type: "number", + description: "API key ID", + }, + apiFormat: { + column: "c.api_format", + type: "enum", + description: "API format used", + values: API_FORMAT_VALUES, + }, + rating: { + column: "c.rating", + type: "number", + description: "User rating", + }, + reqId: { + column: "c.req_id", + type: "text", + description: "Request deduplication ID", + }, + createdAt: { + column: "c.created_at", + type: "timestamp", + description: "Request creation time", + }, + provider: { + column: "p.name", + type: "text", + description: "Provider name (via model join)", + }, + // JSONB nested fields + extraHeaders: { + column: "c.prompt", + type: "jsonb", + description: "Extra headers passed with the request", + nested: true, + jsonbColumn: "c.prompt", + jsonbRootKey: "extraHeaders", + }, + extraBody: { + column: "c.prompt", + type: "jsonb", + description: "Extra body parameters", + nested: true, + jsonbColumn: "c.prompt", + jsonbRootKey: "extraBody", + }, + toolCalls: { + column: "c.completion", + type: "jsonb", + description: "Tool calls in the response", + nested: true, + jsonbColumn: "c.completion", + jsonbRootKey: "tool_calls", + jsonbRootIsArray: true, + }, +}; + +// Validate JSONB path segments to prevent SQL injection +const SAFE_PATH_SEGMENT = /^[a-zA-Z0-9_-]+$/; + +/** + * Unwrap a double-encoded JSONB column. + * The prompt/completion columns store data as JSONB strings (double-encoded), + * so we need (#>> '{}')::jsonb to get the actual JSON object/array. + */ +function unwrapJsonb(column: string): string { + return `(${column} #>> '{}')::jsonb`; +} + +function validateJsonbPath(segments: string[], position?: number): void { + for (const segment of segments) { + if (!SAFE_PATH_SEGMENT.test(segment)) { + throw new CompilerError( + `Invalid JSONB path segment '${segment}'. Only alphanumeric characters, underscores, and hyphens are allowed.`, + position, + ); + } + } +} + +class CompilerError extends Error { + constructor( + message: string, + public position?: number, + ) { + super(message); + this.name = "CompilerError"; + } +} + +// --- SQL Compiler --- + +class SqlCompiler { + private params: unknown[] = []; + private paramIndex = 0; + + constructor(startParamIndex = 0) { + this.paramIndex = startParamIndex; + } + + addParam(value: unknown): string { + this.paramIndex++; + this.params.push(value); + return `$${this.paramIndex}`; + } + + /** + * Resolve a dotted field name to its SQL representation and field mapping. + * Handles both direct fields (e.g., "model") and nested JSONB paths (e.g., "extraHeaders.x-experiment"). + */ + private resolveField(fieldName: string): { + sql: string; + mapping: FieldMapping; + isJsonbPath: boolean; + } { + const parts = fieldName.split("."); + const [rootField] = parts; + if (!rootField) { + throw new CompilerError(`Empty field name`); + } + const mapping = FIELD_REGISTRY[rootField]; + + if (!mapping) { + throw new CompilerError( + `Unknown field '${fieldName}'. Available fields: ${Object.keys(FIELD_REGISTRY).join(", ")}`, + ); + } + + // Simple field (no dots) + if (parts.length === 1) { + return { sql: mapping.column, mapping, isJsonbPath: false }; + } + + // Nested JSONB path + if (!mapping.nested || !mapping.jsonbColumn || !mapping.jsonbRootKey) { + throw new CompilerError( + `Field '${rootField}' does not support nested path access`, + ); + } + + const pathSegments = parts.slice(1); + validateJsonbPath(pathSegments); + + // Build JSONB path expression: + // For extraHeaders.x-experiment: (c.prompt #>> '{}')::jsonb->'extraHeaders'->>'x-experiment' + // For deeper paths: (c.prompt #>> '{}')::jsonb->'extraHeaders'->'nested'->>'key' + const unwrapped = unwrapJsonb(mapping.jsonbColumn); + let jsonbSql = `${unwrapped}->'${mapping.jsonbRootKey}'`; + for (let i = 0; i < pathSegments.length - 1; i++) { + jsonbSql += `->'${pathSegments[i]}'`; + } + jsonbSql += `->>'${pathSegments[pathSegments.length - 1]}'`; + + return { sql: jsonbSql, mapping, isJsonbPath: true }; + } + + compileExpression(expr: KqlExpression): string { + switch (expr.type) { + case "comparison": + return this.compileComparison(expr); + case "exists": + return this.compileExists(expr.field); + case "and": + return `(${this.compileExpression(expr.left)} AND ${this.compileExpression(expr.right)})`; + case "or": + return `(${this.compileExpression(expr.left)} OR ${this.compileExpression(expr.right)})`; + case "not": + return `NOT (${this.compileExpression(expr.expression)})`; + case "group": + return `(${this.compileExpression(expr.expression)})`; + } + } + + private compileExists(fieldName: string): string { + const parts = fieldName.split("."); + const [rootField] = parts; + if (!rootField) { + throw new CompilerError(`Empty field name`); + } + const mapping = FIELD_REGISTRY[rootField]; + if (!mapping) { + throw new CompilerError( + `Unknown field '${fieldName}'. Available fields: ${Object.keys(FIELD_REGISTRY).join(", ")}`, + ); + } + + // For JSONB root fields (e.g., "extraBody EXISTS") + if (mapping.type === "jsonb" && mapping.jsonbColumn && mapping.jsonbRootKey) { + const unwrapped = unwrapJsonb(mapping.jsonbColumn); + + // Array-rooted JSONB (e.g., toolCalls — completion is an array of messages) + if (mapping.jsonbRootIsArray) { + if (parts.length === 1) { + return `EXISTS (SELECT 1 FROM jsonb_array_elements(${unwrapped}) _elem WHERE _elem->'${mapping.jsonbRootKey}' IS NOT NULL)`; + } + const pathSegments = parts.slice(1); + validateJsonbPath(pathSegments); + let pathSql = `_elem->'${mapping.jsonbRootKey}'`; + for (let i = 0; i < pathSegments.length - 1; i++) { + pathSql += `->'${pathSegments[i]}'`; + } + pathSql += `->>'${pathSegments[pathSegments.length - 1]}'`; + return `EXISTS (SELECT 1 FROM jsonb_array_elements(${unwrapped}) _elem WHERE ${pathSql} IS NOT NULL)`; + } + + if (parts.length === 1) { + return `${unwrapped}->'${mapping.jsonbRootKey}' IS NOT NULL`; + } + // Nested existence: extraHeaders.x-app EXISTS + const pathSegments = parts.slice(1); + validateJsonbPath(pathSegments); + let jsonbSql = `${unwrapped}->'${mapping.jsonbRootKey}'`; + for (let i = 0; i < pathSegments.length - 1; i++) { + jsonbSql += `->'${pathSegments[i]}'`; + } + jsonbSql += `->>'${pathSegments[pathSegments.length - 1]}'`; + return `${jsonbSql} IS NOT NULL`; + } + + // For non-JSONB fields, EXISTS means IS NOT NULL + return `${mapping.column} IS NOT NULL`; + } + + /** + * Compile a comparison on an array-rooted JSONB field. + * Generates EXISTS (SELECT 1 FROM jsonb_array_elements(...) _msg, jsonb_array_elements(_msg->'key') _tc WHERE _tc->>'path' op $N) + */ + private compileArrayJsonbComparison( + expr: { field: string; operator: ComparisonOperator; value: KqlValue }, + mapping: FieldMapping, + pathAfterRoot: string[], + ): string { + if (pathAfterRoot.length === 0) { + throw new CompilerError( + `Field '${expr.field}' requires a nested path (e.g., '${expr.field}.function.name')`, + ); + } + + validateJsonbPath(pathAfterRoot); + + const unwrapped = unwrapJsonb(mapping.jsonbColumn!); + + // Build path from _tc element to the target field + let pathSql = "_tc"; + for (let i = 0; i < pathAfterRoot.length - 1; i++) { + pathSql += `->'${pathAfterRoot[i]}'`; + } + pathSql += `->>'${pathAfterRoot[pathAfterRoot.length - 1]}'`; + + // Build the comparison expression + let comparison: string; + if (expr.value.type === "wildcard") { + const pattern = expr.value.pattern.replace(/\*/g, "%"); + const param = this.addParam(pattern); + comparison = `${pathSql} ILIKE ${param}`; + } else { + const strValue = + expr.value.type === "string" ? expr.value.value : String(expr.value.value); + const param = this.addParam(strValue); + if (expr.operator === ":" || expr.operator === "=") { + comparison = `${pathSql} = ${param}`; + } else if (expr.operator === "!=") { + comparison = `${pathSql} != ${param}`; + } else { + comparison = `${pathSql} ${expr.operator} ${param}`; + } + } + + return `EXISTS (SELECT 1 FROM jsonb_array_elements(${unwrapped}) _msg, jsonb_array_elements(_msg->'${mapping.jsonbRootKey}') _tc WHERE ${comparison})`; + } + + private compileComparison(expr: { + field: string; + operator: ComparisonOperator; + value: KqlValue; + }): string { + // Check for array-rooted JSONB fields (e.g., toolCalls.function.name) + const parts = expr.field.split("."); + const rootField = parts[0]; + if (rootField) { + const rootMapping = FIELD_REGISTRY[rootField]; + if (rootMapping?.jsonbRootIsArray) { + return this.compileArrayJsonbComparison(expr, rootMapping, parts.slice(1)); + } + } + + const { sql: fieldSql, mapping, isJsonbPath } = this.resolveField( + expr.field, + ); + + // For JSONB paths, values are always text (extracted with ->>) + if (isJsonbPath) { + return this.compileJsonbComparison(fieldSql, expr.operator, expr.value); + } + + switch (mapping.type) { + case "text": + return this.compileTextComparison(fieldSql, expr.operator, expr.value); + case "number": + return this.compileNumberComparison( + fieldSql, + expr.operator, + expr.value, + ); + case "enum": + return this.compileEnumComparison( + fieldSql, + expr.operator, + expr.value, + mapping.values || [], + expr.field, + ); + case "timestamp": + return this.compileTimestampComparison( + fieldSql, + expr.operator, + expr.value, + ); + case "jsonb": + // Direct JSONB field reference without path — not directly queryable + throw new CompilerError( + `Field '${expr.field}' requires a nested path (e.g., '${expr.field}.key')`, + ); + } + } + + private compileTextComparison( + fieldSql: string, + operator: ComparisonOperator, + value: KqlValue, + ): string { + if (value.type === "wildcard") { + const pattern = value.pattern.replace(/\*/g, "%"); + const param = this.addParam(pattern); + return `${fieldSql} ILIKE ${param}`; + } + + const strValue = value.type === "string" ? value.value : String(value.value); + const param = this.addParam(strValue); + + if (operator === ":" || operator === "=") { + return `${fieldSql} = ${param}`; + } + if (operator === "!=") { + return `${fieldSql} != ${param}`; + } + + // Range operators on text fields — compare lexicographically + return `${fieldSql} ${operator} ${param}`; + } + + private compileNumberComparison( + fieldSql: string, + operator: ComparisonOperator, + value: KqlValue, + ): string { + let numValue: number; + if (value.type === "number") { + numValue = value.value; + } else if (value.type === "string") { + numValue = Number(value.value); + if (Number.isNaN(numValue)) { + throw new CompilerError( + `Expected numeric value for numeric field, got '${value.value}'`, + ); + } + } else { + throw new CompilerError( + `Wildcards are not supported for numeric fields`, + ); + } + + const param = this.addParam(numValue); + const sqlOp = operator === ":" ? "=" : operator; + return `${fieldSql} ${sqlOp} ${param}`; + } + + private compileEnumComparison( + fieldSql: string, + operator: ComparisonOperator, + value: KqlValue, + validValues: string[], + fieldName: string, + ): string { + if (value.type === "wildcard") { + const pattern = value.pattern.replace(/\*/g, "%"); + const param = this.addParam(pattern); + return `${fieldSql}::text ILIKE ${param}`; + } + + const strValue = value.type === "string" ? value.value : String(value.value); + + if (validValues.length > 0 && !validValues.includes(strValue)) { + throw new CompilerError( + `Invalid value '${strValue}' for field '${fieldName}'. Valid values: ${validValues.join(", ")}`, + ); + } + + const param = this.addParam(strValue); + if (operator === ":" || operator === "=") { + return `${fieldSql} = ${param}`; + } + if (operator === "!=") { + return `${fieldSql} != ${param}`; + } + + throw new CompilerError( + `Operator '${operator}' is not supported for enum field '${fieldName}'`, + ); + } + + private compileTimestampComparison( + fieldSql: string, + operator: ComparisonOperator, + value: KqlValue, + ): string { + if (value.type === "wildcard") { + throw new CompilerError(`Wildcards are not supported for timestamp fields`); + } + + const strValue = value.type === "string" ? value.value : String(value.value); + const param = this.addParam(strValue); + const sqlOp = operator === ":" ? "=" : operator; + return `${fieldSql} ${sqlOp} ${param}::timestamp`; + } + + private compileJsonbComparison( + fieldSql: string, + operator: ComparisonOperator, + value: KqlValue, + ): string { + // JSONB ->> returns text, so all comparisons are text-based + if (value.type === "wildcard") { + const pattern = value.pattern.replace(/\*/g, "%"); + const param = this.addParam(pattern); + return `${fieldSql} ILIKE ${param}`; + } + + const strValue = value.type === "string" ? value.value : String(value.value); + const param = this.addParam(strValue); + + if (operator === ":" || operator === "=") { + return `${fieldSql} = ${param}`; + } + if (operator === "!=") { + return `(${fieldSql} IS NULL OR ${fieldSql} != ${param})`; + } + + // For numeric comparisons on JSONB text, cast to numeric + if (value.type === "number") { + return `(${fieldSql})::numeric ${operator} ${param}::numeric`; + } + + return `${fieldSql} ${operator} ${param}`; + } + + compileAggregation(agg: { + functions: AggregateExpression[]; + groupBy?: string[]; + }): { + selectExpressions: { sql: string; alias: string }[]; + groupByColumn?: string; + groupByField?: string; + } { + const selectExpressions = agg.functions.map((fn) => + this.compileAggregateFunction(fn), + ); + + let groupByColumn: string | undefined; + let groupByField: string | undefined; + + if (agg.groupBy && agg.groupBy.length > 0) { + // Currently support single GROUP BY field + const [field] = agg.groupBy; + if (!field) { + throw new CompilerError("Empty GROUP BY field"); + } + const { sql: fieldSql } = this.resolveField(field); + groupByColumn = fieldSql; + groupByField = field; + } + + return { selectExpressions, groupByColumn, groupByField }; + } + + private compileAggregateFunction(fn: AggregateExpression): { + sql: string; + alias: string; + } { + if (fn.fn === "count") { + return { sql: "COUNT(*)", alias: "count" }; + } + + if (!fn.field) { + throw new CompilerError(`${fn.fn}() requires a field argument`); + } + + const { sql: fieldSql } = this.resolveField(fn.field); + + switch (fn.fn) { + case "avg": + return { + sql: `AVG(${fieldSql})`, + alias: `avg_${fn.field}`, + }; + case "sum": + return { + sql: `SUM(${fieldSql})`, + alias: `sum_${fn.field}`, + }; + case "min": + return { + sql: `MIN(${fieldSql})`, + alias: `min_${fn.field}`, + }; + case "max": + return { + sql: `MAX(${fieldSql})`, + alias: `max_${fn.field}`, + }; + case "p50": + return { + sql: `percentile_cont(0.50) WITHIN GROUP (ORDER BY ${fieldSql})`, + alias: `p50_${fn.field}`, + }; + case "p95": + return { + sql: `percentile_cont(0.95) WITHIN GROUP (ORDER BY ${fieldSql})`, + alias: `p95_${fn.field}`, + }; + case "p99": + return { + sql: `percentile_cont(0.99) WITHIN GROUP (ORDER BY ${fieldSql})`, + alias: `p99_${fn.field}`, + }; + } + } + + getParams(): unknown[] { + return this.params; + } + + getParamIndex(): number { + return this.paramIndex; + } +} + +// --- Helpers --- + +/** + * Check if a KQL expression references a specific field name. + */ +function expressionReferencesField( + expr: KqlExpression | undefined, + fieldName: string, +): boolean { + if (!expr) { + return false; + } + switch (expr.type) { + case "comparison": + case "exists": + return expr.field === fieldName || expr.field.startsWith(`${fieldName}.`); + case "and": + case "or": + return ( + expressionReferencesField(expr.left, fieldName) || + expressionReferencesField(expr.right, fieldName) + ); + case "not": + return expressionReferencesField(expr.expression, fieldName); + case "group": + return expressionReferencesField(expr.expression, fieldName); + } +} + +// --- Public API --- + +export interface CompileOptions { + /** Time range filter to add to the WHERE clause */ + timeRange?: { from: Date; to: Date }; + /** Starting parameter index (for combining with other parameterized queries) */ + startParamIndex?: number; +} + +export function compileSearch( + query: KqlQuery, + options?: CompileOptions, +): CompiledQuery { + const compiler = new SqlCompiler(options?.startParamIndex ?? 0); + const parts: string[] = []; + + // Always exclude deleted records + parts.push("c.deleted = false"); + + // Add time range filter only if the user's query doesn't already filter on createdAt + const hasExplicitTimeFilter = expressionReferencesField( + query.filter, + "createdAt", + ); + if (options?.timeRange && !hasExplicitTimeFilter) { + const fromParam = compiler.addParam(options.timeRange.from); + const toParam = compiler.addParam(options.timeRange.to); + parts.push(`c.created_at >= ${fromParam}`); + parts.push(`c.created_at <= ${toParam}`); + } + + // Add KQL filter expression + if (query.filter) { + parts.push(compiler.compileExpression(query.filter)); + } + + const whereClause = parts.join(" AND "); + + // Compile aggregation if present + let aggregation: CompiledQuery["aggregation"]; + if (query.aggregation) { + aggregation = compiler.compileAggregation(query.aggregation); + } + + return { + whereClause, + params: compiler.getParams(), + aggregation, + }; +} + +/** + * Get the list of searchable fields for autocomplete. + */ +export function getSearchableFields(): FieldInfo[] { + return Object.entries(FIELD_REGISTRY).map(([name, mapping]) => ({ + name, + type: mapping.type, + description: mapping.description, + values: mapping.values, + nested: mapping.nested, + })); +} + diff --git a/backend/src/search/index.ts b/backend/src/search/index.ts new file mode 100644 index 0000000..55b624e --- /dev/null +++ b/backend/src/search/index.ts @@ -0,0 +1,17 @@ +export { parseKql } from "./parser"; +export { compileSearch, getSearchableFields } from "./compiler"; +export type { CompileOptions } from "./compiler"; +export type { + KqlQuery, + KqlExpression, + KqlValue, + ComparisonOperator, + AggregateExpression, + AggregateFunction, + KqlAggregation, + ParseResult, + ParseError, + CompiledQuery, + FieldInfo, + FieldType, +} from "./types"; diff --git a/backend/src/search/lexer.ts b/backend/src/search/lexer.ts new file mode 100644 index 0000000..70e17f5 --- /dev/null +++ b/backend/src/search/lexer.ts @@ -0,0 +1,187 @@ +import type { Token, TokenType } from "./types"; + +const KEYWORDS: Record = { + AND: "AND", + OR: "OR", + NOT: "NOT", + EXISTS: "EXISTS", + stats: "STATS", + by: "BY", +}; + +const OPERATORS = new Set([":", "=", "!=", ">", ">=", "<", "<="]); + +export class LexerError extends Error { + constructor( + message: string, + public position: number, + public length: number, + ) { + super(message); + this.name = "LexerError"; + } +} + +export function tokenize(input: string): Token[] { + const tokens: Token[] = []; + let pos = 0; + + const charAt = (i: number): string => input.charAt(i); + + while (pos < input.length) { + // Skip whitespace + if (/\s/.test(charAt(pos))) { + pos++; + continue; + } + + const start = pos; + const ch = charAt(pos); + + // Single-character tokens + if (ch === "(") { + tokens.push({ type: "LPAREN", value: "(", position: start }); + pos++; + continue; + } + if (ch === ")") { + tokens.push({ type: "RPAREN", value: ")", position: start }); + pos++; + continue; + } + if (ch === "|") { + tokens.push({ type: "PIPE", value: "|", position: start }); + pos++; + continue; + } + if (ch === ",") { + tokens.push({ type: "COMMA", value: ",", position: start }); + pos++; + continue; + } + + // Operators: !=, >=, <=, >, <, =, : + if (ch === "!" && pos + 1 < input.length && charAt(pos + 1) === "=") { + tokens.push({ type: "OPERATOR", value: "!=", position: start }); + pos += 2; + continue; + } + if (ch === ">" && pos + 1 < input.length && charAt(pos + 1) === "=") { + tokens.push({ type: "OPERATOR", value: ">=", position: start }); + pos += 2; + continue; + } + if (ch === "<" && pos + 1 < input.length && charAt(pos + 1) === "=") { + tokens.push({ type: "OPERATOR", value: "<=", position: start }); + pos += 2; + continue; + } + if (ch === ">") { + tokens.push({ type: "OPERATOR", value: ">", position: start }); + pos++; + continue; + } + if (ch === "<") { + tokens.push({ type: "OPERATOR", value: "<", position: start }); + pos++; + continue; + } + if (ch === "=") { + tokens.push({ type: "OPERATOR", value: "=", position: start }); + pos++; + continue; + } + if (ch === ":") { + tokens.push({ type: "OPERATOR", value: ":", position: start }); + pos++; + continue; + } + + // Quoted string + if (ch === '"') { + pos++; // skip opening quote + let value = ""; + while (pos < input.length && charAt(pos) !== '"') { + if (charAt(pos) === "\\" && pos + 1 < input.length) { + pos++; // skip backslash + value += charAt(pos); + } else { + value += charAt(pos); + } + pos++; + } + if (pos >= input.length) { + throw new LexerError("Unterminated string", start, pos - start); + } + pos++; // skip closing quote + tokens.push({ type: "STRING", value, position: start }); + continue; + } + + // Number (integer or decimal, optionally negative) + if (/\d/.test(ch) || (ch === "-" && pos + 1 < input.length && /\d/.test(charAt(pos + 1)))) { + let num = ch; + pos++; + while (pos < input.length && /[\d.]/.test(charAt(pos))) { + num += charAt(pos); + pos++; + } + tokens.push({ type: "NUMBER", value: num, position: start }); + continue; + } + + // Wildcard or unquoted identifier/field + if (ch === "*" || /[a-zA-Z_]/.test(ch)) { + let word = ""; + let hasWildcard = false; + while ( + pos < input.length && + /[a-zA-Z0-9_.*-]/.test(charAt(pos)) && + !OPERATORS.has(charAt(pos)) + ) { + // The `:` is an operator, so it stops identifier scanning + if (charAt(pos) === ":") { + break; + } + if (charAt(pos) === "*") { + hasWildcard = true; + } + word += charAt(pos); + pos++; + } + + // Check for keywords (case-sensitive for AND/OR/NOT, case-insensitive for stats/by) + const keyword = KEYWORDS[word] || KEYWORDS[word.toLowerCase()]; + if (keyword && !hasWildcard) { + // Only treat as keyword if it's AND/OR/NOT (uppercase) or stats/by (lowercase) + if ( + word === "AND" || + word === "OR" || + word === "NOT" || + word === "EXISTS" || + word.toLowerCase() === "stats" || + word.toLowerCase() === "by" + ) { + tokens.push({ type: keyword, value: word, position: start }); + continue; + } + } + + if (hasWildcard) { + tokens.push({ type: "WILDCARD", value: word, position: start }); + } else { + tokens.push({ type: "FIELD", value: word, position: start }); + } + continue; + } + + throw new LexerError( + `Unexpected character '${ch}'`, + pos, + 1, + ); + } + + tokens.push({ type: "EOF", value: "", position: pos }); + return tokens; +} diff --git a/backend/src/search/parser.ts b/backend/src/search/parser.ts new file mode 100644 index 0000000..9293b69 --- /dev/null +++ b/backend/src/search/parser.ts @@ -0,0 +1,337 @@ +import { tokenize, LexerError } from "./lexer"; +import type { + Token, + TokenType, + KqlQuery, + KqlExpression, + KqlValue, + ComparisonOperator, + AggregateExpression, + AggregateFunction, + ParseResult, +} from "./types"; + +const AGGREGATE_FUNCTIONS = new Set([ + "count", + "avg", + "sum", + "min", + "max", + "p50", + "p95", + "p99", +]); + +class Parser { + private tokens: Token[]; + private pos: number; + + constructor(tokens: Token[]) { + this.tokens = tokens; + this.pos = 0; + } + + private current(): Token { + // pos is always valid (lexer ends with EOF token) + const token = this.tokens[this.pos]; + if (!token) { + throw new Error("Unexpected end of token stream"); + } + return token; + } + + private peek(type: TokenType): boolean { + return this.current().type === type; + } + + private advance(): Token { + const token = this.current(); + if (token.type !== "EOF") { + this.pos++; + } + return token; + } + + private expect(type: TokenType): Token { + const token = this.current(); + if (token.type !== type) { + throw new ParserError( + `Expected ${type} but found ${token.type}${token.value ? ` '${token.value}'` : ""}`, + token.position, + token.value.length || 1, + ); + } + return this.advance(); + } + + // query = filter_expr (PIPE "stats" agg_list ("by" field_list)?)? + parse(): KqlQuery { + const query: KqlQuery = {}; + + // Parse filter expression if present (not starting with pipe or at EOF) + if (!this.peek("EOF") && !this.peek("PIPE")) { + query.filter = this.parseOrExpr(); + } + + // Parse aggregation pipeline if present + if (this.peek("PIPE")) { + this.advance(); // consume PIPE + this.expect("STATS"); + + const functions = this.parseAggList(); + let groupBy: string[] | undefined; + + if (this.peek("BY")) { + this.advance(); // consume BY + groupBy = this.parseFieldList(); + } + + query.aggregation = { functions, groupBy }; + } + + if (!this.peek("EOF")) { + const token = this.current(); + throw new ParserError( + `Unexpected token '${token.value}'`, + token.position, + token.value.length || 1, + ); + } + + return query; + } + + // or_expr = and_expr ("OR" and_expr)* + private parseOrExpr(): KqlExpression { + let left = this.parseAndExpr(); + + while (this.peek("OR")) { + this.advance(); // consume OR + const right = this.parseAndExpr(); + left = { type: "or", left, right }; + } + + return left; + } + + // and_expr = not_expr ("AND" not_expr)* + private parseAndExpr(): KqlExpression { + let left = this.parseNotExpr(); + + while (this.peek("AND")) { + this.advance(); // consume AND + const right = this.parseNotExpr(); + left = { type: "and", left, right }; + } + + return left; + } + + // not_expr = "NOT" not_expr | primary + private parseNotExpr(): KqlExpression { + if (this.peek("NOT")) { + this.advance(); // consume NOT + const expression = this.parseNotExpr(); + return { type: "not", expression }; + } + + return this.parsePrimary(); + } + + // primary = LPAREN or_expr RPAREN | comparison + private parsePrimary(): KqlExpression { + if (this.peek("LPAREN")) { + this.advance(); // consume LPAREN + const expression = this.parseOrExpr(); + this.expect("RPAREN"); + return { type: "group", expression }; + } + + return this.parseComparison(); + } + + // comparison = FIELD (operator value | "EXISTS") + private parseComparison(): KqlExpression { + const fieldToken = this.current(); + + if (fieldToken.type !== "FIELD") { + throw new ParserError( + `Expected field name but found ${fieldToken.type}${fieldToken.value ? ` '${fieldToken.value}'` : ""}`, + fieldToken.position, + fieldToken.value.length || 1, + ); + } + this.advance(); // consume FIELD + + // Check for EXISTS keyword (e.g., "extraBody EXISTS") + if (this.peek("EXISTS")) { + this.advance(); // consume EXISTS + return { type: "exists", field: fieldToken.value }; + } + + const opToken = this.current(); + if (opToken.type !== "OPERATOR") { + throw new ParserError( + `Expected operator after field '${fieldToken.value}'`, + opToken.position, + opToken.value.length || 1, + ); + } + const operator = opToken.value as ComparisonOperator; + this.advance(); // consume OPERATOR + + const value = this.parseValue(); + + return { + type: "comparison", + field: fieldToken.value, + operator, + value, + }; + } + + // value = STRING | NUMBER | WILDCARD | FIELD (unquoted string treated as string value) + private parseValue(): KqlValue { + const token = this.current(); + + if (token.type === "STRING") { + this.advance(); + return { type: "string", value: token.value }; + } + + if (token.type === "NUMBER") { + this.advance(); + return { type: "number", value: Number(token.value) }; + } + + if (token.type === "WILDCARD") { + this.advance(); + return { type: "wildcard", pattern: token.value }; + } + + // Allow unquoted identifiers as string values (e.g., `status: completed`) + if (token.type === "FIELD") { + this.advance(); + return { type: "string", value: token.value }; + } + + throw new ParserError( + `Expected value after operator`, + token.position, + token.value.length || 1, + ); + } + + // agg_list = agg_fn ("," agg_fn)* + private parseAggList(): AggregateExpression[] { + const functions: AggregateExpression[] = []; + functions.push(this.parseAggFn()); + + while (this.peek("COMMA")) { + this.advance(); // consume COMMA + functions.push(this.parseAggFn()); + } + + return functions; + } + + // agg_fn = FIELD "(" FIELD? ")" + private parseAggFn(): AggregateExpression { + const fnToken = this.current(); + if (fnToken.type !== "FIELD") { + throw new ParserError( + `Expected aggregate function name`, + fnToken.position, + fnToken.value.length || 1, + ); + } + + const fnName = fnToken.value.toLowerCase(); + if (!AGGREGATE_FUNCTIONS.has(fnName as AggregateFunction)) { + throw new ParserError( + `Unknown aggregate function '${fnToken.value}'. Supported: ${[...AGGREGATE_FUNCTIONS].join(", ")}`, + fnToken.position, + fnToken.value.length, + ); + } + this.advance(); // consume function name + + this.expect("LPAREN"); + + let field: string | undefined; + if (!this.peek("RPAREN")) { + const fieldToken = this.expect("FIELD"); + field = fieldToken.value; + } + + this.expect("RPAREN"); + + // Validate count() has no field, others require a field + if (fnName === "count" && field) { + throw new ParserError( + `count() does not take a field argument`, + fnToken.position, + fnToken.value.length, + ); + } + if (fnName !== "count" && !field) { + throw new ParserError( + `${fnName}() requires a field argument`, + fnToken.position, + fnToken.value.length, + ); + } + + return { fn: fnName as AggregateFunction, field }; + } + + // field_list = FIELD ("," FIELD)* + private parseFieldList(): string[] { + const fields: string[] = []; + fields.push(this.expect("FIELD").value); + + while (this.peek("COMMA")) { + this.advance(); // consume COMMA + fields.push(this.expect("FIELD").value); + } + + return fields; + } +} + +class ParserError extends Error { + constructor( + message: string, + public position: number, + public length: number, + ) { + super(message); + this.name = "ParserError"; + } +} + +export function parseKql(input: string): ParseResult { + try { + const trimmed = input.trim(); + if (trimmed === "") { + return { success: true, query: {} }; + } + + const tokens = tokenize(trimmed); + const parser = new Parser(tokens); + const query = parser.parse(); + return { success: true, query }; + } catch (err) { + if (err instanceof ParserError || err instanceof LexerError) { + return { + success: false, + error: { + message: err.message, + position: err.position, + length: err.length, + }, + }; + } + throw err; + } +} diff --git a/backend/src/search/types.ts b/backend/src/search/types.ts new file mode 100644 index 0000000..aafbb93 --- /dev/null +++ b/backend/src/search/types.ts @@ -0,0 +1,133 @@ +// KQL (Kibana Query Language) AST types for advanced meta search + +// --- Token types (used by lexer) --- + +export type TokenType = + | "FIELD" + | "STRING" + | "NUMBER" + | "WILDCARD" + | "OPERATOR" + | "AND" + | "OR" + | "NOT" + | "LPAREN" + | "RPAREN" + | "PIPE" + | "STATS" + | "BY" + | "COMMA" + | "EXISTS" + | "EOF"; + +export interface Token { + type: TokenType; + value: string; + position: number; +} + +// --- AST node types --- + +export type KqlValue = + | { type: "string"; value: string } + | { type: "number"; value: number } + | { type: "wildcard"; pattern: string }; + +export type ComparisonOperator = + | ":" + | "=" + | "!=" + | ">" + | ">=" + | "<" + | "<="; + +export type KqlExpression = + | { + type: "comparison"; + field: string; + operator: ComparisonOperator; + value: KqlValue; + } + | { type: "and"; left: KqlExpression; right: KqlExpression } + | { type: "or"; left: KqlExpression; right: KqlExpression } + | { type: "not"; expression: KqlExpression } + | { type: "group"; expression: KqlExpression } + | { type: "exists"; field: string }; + +export type AggregateFunction = + | "count" + | "avg" + | "sum" + | "min" + | "max" + | "p50" + | "p95" + | "p99"; + +export interface AggregateExpression { + fn: AggregateFunction; + field?: string; // undefined for count() +} + +export interface KqlAggregation { + functions: AggregateExpression[]; + groupBy?: string[]; +} + +export interface KqlQuery { + filter?: KqlExpression; + aggregation?: KqlAggregation; +} + +// --- Parse result types --- + +export interface ParseError { + message: string; + position: number; + length: number; +} + +export type ParseResult = + | { success: true; query: KqlQuery } + | { success: false; error: ParseError }; + +// --- Compiled query types --- + +export interface CompiledQuery { + /** SQL WHERE clause fragment (without "WHERE" keyword). Empty string if no filter. */ + whereClause: string; + /** Parameterized values for the WHERE clause ($1, $2, ...) */ + params: unknown[]; + /** Aggregation SQL fragments, present when query has `| stats` */ + aggregation?: { + /** SQL SELECT expressions for aggregation functions */ + selectExpressions: { sql: string; alias: string }[]; + /** SQL GROUP BY column expression */ + groupByColumn?: string; + /** The field name used in GROUP BY */ + groupByField?: string; + }; +} + +// --- Field metadata for autocomplete --- + +export type FieldType = + | "text" + | "number" + | "enum" + | "timestamp" + | "jsonb"; + +export interface FieldInfo { + /** Field name as used in KQL queries (e.g., "model", "extraHeaders.x-experiment") */ + name: string; + /** Data type */ + type: FieldType; + /** Human-readable description */ + description: string; + /** Enum values if type is "enum" */ + values?: string[]; + /** Whether the field supports nested paths (e.g., extraHeaders.*) */ + nested?: boolean; +} diff --git a/frontend/src/pages/requests/search-bar.tsx b/frontend/src/pages/requests/search-bar.tsx new file mode 100644 index 0000000..9e9d559 --- /dev/null +++ b/frontend/src/pages/requests/search-bar.tsx @@ -0,0 +1,87 @@ +import { useState } from 'react' +import { useNavigate, useSearch } from '@tanstack/react-router' +import { XIcon } from 'lucide-react' + +import { Button } from '@/components/ui/button' +import { QueryInput } from '@/pages/search/query-input' +import { TimeRangePicker, type TimeRangePreset } from '@/pages/search/time-range-picker' +import { ExportButton } from '@/pages/search/export-button' + +export function SearchBar() { + const { q, range, ...rest } = useSearch({ from: '/requests/' }) + const navigate = useNavigate() + + const [queryText, setQueryText] = useState(q ?? '') + + const handleSubmit = () => { + const trimmed = queryText.trim() + navigate({ + to: '/requests', + search: { + ...rest, + q: trimmed || undefined, + range: range ?? '24h', + page: 1, // Reset to first page on new search + }, + }) + } + + const handleClear = () => { + setQueryText('') + navigate({ + to: '/requests', + search: { + ...rest, + q: undefined, + page: 1, + }, + }) + } + + const handleRangeChange = (value: TimeRangePreset) => { + navigate({ + to: '/requests', + search: { + ...rest, + q, + range: value, + page: 1, + }, + }) + } + + const isSearching = !!q?.trim() + + const now = new Date() + const rangeMs: Record = { + '15m': 15 * 60_000, + '1h': 3600_000, + '4h': 4 * 3600_000, + '12h': 12 * 3600_000, + '24h': 24 * 3600_000, + '7d': 7 * 86400_000, + '30d': 30 * 86400_000, + } + const timeRange = isSearching + ? { + from: new Date(now.getTime() - (rangeMs[range ?? '24h'] ?? 86400_000)).toISOString(), + to: now.toISOString(), + } + : undefined + + return ( +
+ + + {isSearching && ( + <> + + + + )} +
+ ) +} diff --git a/frontend/src/pages/search/aggregation-results.tsx b/frontend/src/pages/search/aggregation-results.tsx new file mode 100644 index 0000000..328dda8 --- /dev/null +++ b/frontend/src/pages/search/aggregation-results.tsx @@ -0,0 +1,56 @@ +import { TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' + +interface AggregationResultsProps { + results: Record[] +} + +export function AggregationResults({ results }: AggregationResultsProps) { + if (results.length === 0) { + return
No aggregation results
+ } + + // Get column names from first result + const columns = Object.keys(results[0]) + + return ( +
+ + + + {columns.map((col) => ( + + {col} + + ))} + + + + {results.map((row, i) => ( + + {columns.map((col) => ( + + {formatAggValue(row[col])} + + ))} + + ))} + +
+
+ ) +} + +function formatAggValue(value: unknown): string { + if (value === null || value === undefined) return '-' + if (typeof value === 'number') { + return Number.isInteger(value) ? String(value) : value.toFixed(2) + } + if (typeof value === 'string') { + // Try to format as number if it looks like one + const num = Number(value) + if (!Number.isNaN(num)) { + return Number.isInteger(num) ? String(num) : num.toFixed(2) + } + } + return String(value) +} diff --git a/frontend/src/pages/search/export-button.tsx b/frontend/src/pages/search/export-button.tsx new file mode 100644 index 0000000..f0624c9 --- /dev/null +++ b/frontend/src/pages/search/export-button.tsx @@ -0,0 +1,69 @@ +import { useState } from 'react' +import { DownloadIcon } from 'lucide-react' + +import { Button } from '@/components/ui/button' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu' + +interface ExportButtonProps { + query: string + timeRange?: { from: string; to: string } + disabled?: boolean +} + +export function ExportButton({ query, timeRange, disabled }: ExportButtonProps) { + const [loading, setLoading] = useState(false) + + const handleExport = async (format: 'csv' | 'json') => { + setLoading(true) + try { + const backendBaseURL = import.meta.env.PROD ? location.origin : import.meta.env.VITE_BASE_URL + const adminSecret = localStorage.getItem('admin-secret') + if (!adminSecret) return + + const response = await fetch(`${backendBaseURL}/api/admin/search/export`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${JSON.parse(adminSecret)}`, + }, + body: JSON.stringify({ query, timeRange, format }), + }) + + if (!response.ok) throw new Error('Export failed') + + const blob = await response.blob() + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = `search-results.${format}` + document.body.appendChild(a) + a.click() + document.body.removeChild(a) + URL.revokeObjectURL(url) + } catch { + // Silently fail — could add toast notification here + } finally { + setLoading(false) + } + } + + return ( + + + + + + handleExport('csv')}>Export as CSV + handleExport('json')}>Export as JSON + + + ) +} diff --git a/frontend/src/pages/search/query-input.tsx b/frontend/src/pages/search/query-input.tsx new file mode 100644 index 0000000..0042caa --- /dev/null +++ b/frontend/src/pages/search/query-input.tsx @@ -0,0 +1,238 @@ +import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react' +import { useQuery } from '@tanstack/react-query' +import { SearchIcon } from 'lucide-react' + +import { api } from '@/lib/api' +import { cn } from '@/lib/utils' +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' + +interface QueryInputProps { + value: string + onChange: (value: string) => void + onSubmit: () => void + className?: string +} + +type FieldInfo = { + name: string + type: string + description: string + values?: string[] + nested?: boolean +} + +const KEYWORDS = ['AND', 'OR', 'NOT', '| stats', 'by'] +const AGG_FUNCTIONS = ['count()', 'avg(', 'sum(', 'min(', 'max(', 'p50(', 'p95(', 'p99('] + +export function QueryInput({ value, onChange, onSubmit, className }: QueryInputProps) { + const [open, setOpen] = useState(false) + const [validationError, setValidationError] = useState(null) + const inputRef = useRef(null) + + // Fetch field metadata for autocomplete + const { data: fieldsData } = useQuery({ + queryKey: ['search-fields'], + queryFn: async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { data, error } = await (api.admin.search.fields as any).get() + if (error) return { fields: [] as FieldInfo[] } + return data as { fields: FieldInfo[] } + }, + staleTime: 60_000, + }) + + const fields = useMemo(() => fieldsData?.fields ?? [], [fieldsData]) + + // Build suggestions based on cursor context + const getSuggestions = useCallback((): { label: string; description?: string; insert: string }[] => { + const cursorText = value.trimEnd() + const lastToken = cursorText.split(/\s+/).pop() || '' + + // After a pipe, suggest "stats" + if (cursorText.endsWith('|') || cursorText.endsWith('| ')) { + return [{ label: 'stats', description: 'Aggregate results', insert: 'stats ' }] + } + + // After "stats", suggest aggregate functions + if (/\|\s*stats\s*$/i.test(cursorText) || /,\s*$/i.test(cursorText)) { + return AGG_FUNCTIONS.map((fn) => ({ + label: fn, + description: 'Aggregate function', + insert: fn, + })) + } + + // After "by", suggest fields + if (/\bby\s*$/i.test(cursorText)) { + return fields.map((f) => ({ + label: f.name, + description: f.description, + insert: f.name + ' ', + })) + } + + // After an operator (:, =, >=, etc.), suggest values for the field + const fieldMatch = cursorText.match(/(\w[\w.]*)\s*(?::|=|!=|>=?|<=?)\s*$/) + if (fieldMatch) { + const fieldName = fieldMatch[1] + const field = fields.find((f) => f.name === fieldName) + if (field?.values && field.values.length > 0) { + return field.values.map((v) => ({ + label: `"${v}"`, + description: `${field.name} value`, + insert: `"${v}" `, + })) + } + return [] + } + + // Default: suggest field names and keywords + const suggestions: { label: string; description?: string; insert: string }[] = [] + + // Filter field names by what user has typed + const lowerToken = lastToken.toLowerCase() + for (const field of fields) { + if (!lowerToken || field.name.toLowerCase().startsWith(lowerToken)) { + suggestions.push({ + label: field.name, + description: field.description, + insert: field.name + ': ', + }) + } + } + + // Add keywords + for (const kw of KEYWORDS) { + if (!lowerToken || kw.toLowerCase().startsWith(lowerToken)) { + suggestions.push({ + label: kw, + description: 'Keyword', + insert: kw + ' ', + }) + } + } + + return suggestions.slice(0, 12) + }, [value, fields]) + + const suggestions = getSuggestions() + + // Validate query on change (debounced) + useEffect(() => { + if (!value.trim()) { + setValidationError(null) + return + } + + const timeout = setTimeout(async () => { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { data, error } = await (api.admin.search.validate as any).post({ + query: value, + }) + if (error || !data) { + setValidationError(null) + return + } + if (data.valid) { + setValidationError(null) + } else if (data.error) { + setValidationError(data.error.message) + } + } catch { + // Ignore validation errors during typing + } + }, 500) + + return () => clearTimeout(timeout) + }, [value]) + + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Enter') { + e.preventDefault() + setOpen(false) + onSubmit() + } + if (e.key === 'Escape') { + setOpen(false) + } + } + + const handleSuggestionClick = (insert: string) => { + // Replace the last partial token with the suggestion + const parts = value.trimEnd().split(/\s+/) + const lastToken = parts[parts.length - 1] || '' + + let newValue: string + if (lastToken && insert.toLowerCase().startsWith(lastToken.toLowerCase())) { + // Replace the partial match + parts[parts.length - 1] = insert + newValue = parts.join(' ') + } else { + // Append + newValue = value.trimEnd() + (value.endsWith(' ') || !value ? '' : ' ') + insert + } + + onChange(newValue) + setOpen(false) + inputRef.current?.focus() + } + + return ( +
+ 0} onOpenChange={setOpen}> + +
+ + { + onChange(e.target.value) + if (e.target.value) setOpen(true) + }} + onFocus={() => { + if (value || suggestions.length > 0) setOpen(true) + }} + onKeyDown={handleKeyDown} + placeholder='model: "gpt-4" AND status: completed' + className={cn( + 'pl-9 font-mono text-sm', + validationError && 'border-destructive focus-visible:ring-destructive', + )} + /> + {validationError && ( +

{validationError}

+ )} +
+ + e.preventDefault()} + > +
+ {suggestions.map((suggestion, i) => ( + + ))} +
+ + + +
+ ) +} diff --git a/frontend/src/pages/search/search-histogram.tsx b/frontend/src/pages/search/search-histogram.tsx new file mode 100644 index 0000000..24b1240 --- /dev/null +++ b/frontend/src/pages/search/search-histogram.tsx @@ -0,0 +1,54 @@ +import { memo } from 'react' +import { format } from 'date-fns' +import { Bar, BarChart, CartesianGrid, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts' + +import { tooltipContentStyle, tooltipItemStyle, tooltipLabelStyle } from '@/pages/overview/charts/chart-styles' + +interface HistogramBucket { + bucket: string | Date + total: string + completed: string + failed: string +} + +interface SearchHistogramProps { + data: HistogramBucket[] +} + +export const SearchHistogram = memo(function SearchHistogram({ data }: SearchHistogramProps) { + const chartData = data.map((item) => ({ + timestamp: typeof item.bucket === 'string' ? item.bucket : item.bucket.toISOString(), + completed: Number(item.completed), + failed: Number(item.failed), + other: Math.max(0, Number(item.total) - Number(item.completed) - Number(item.failed)), + })) + + if (chartData.length === 0) return null + + return ( +
+ + + + format(new Date(value), 'HH:mm')} + className="text-xs" + tickLine={false} + axisLine={false} + /> + + format(new Date(value), 'yyyy-MM-dd HH:mm:ss')} + contentStyle={tooltipContentStyle} + labelStyle={tooltipLabelStyle} + itemStyle={tooltipItemStyle} + /> + + + + + +
+ ) +}) diff --git a/frontend/src/pages/search/time-range-picker.tsx b/frontend/src/pages/search/time-range-picker.tsx new file mode 100644 index 0000000..9efbf27 --- /dev/null +++ b/frontend/src/pages/search/time-range-picker.tsx @@ -0,0 +1,76 @@ +import { useMemo } from 'react' + +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' + +export type TimeRangePreset = '15m' | '1h' | '4h' | '12h' | '24h' | '7d' | '30d' + +const TIME_RANGES: { value: TimeRangePreset; label: string }[] = [ + { value: '15m', label: 'Last 15 min' }, + { value: '1h', label: 'Last 1 hour' }, + { value: '4h', label: 'Last 4 hours' }, + { value: '12h', label: 'Last 12 hours' }, + { value: '24h', label: 'Last 24 hours' }, + { value: '7d', label: 'Last 7 days' }, + { value: '30d', label: 'Last 30 days' }, +] + +interface TimeRangePickerProps { + value: TimeRangePreset + onChange: (value: TimeRangePreset) => void +} + +export function TimeRangePicker({ value, onChange }: TimeRangePickerProps) { + return ( + + ) +} + +/** + * Convert a time range preset to from/to Date objects. + */ +export function useTimeRangeDates(preset: TimeRangePreset): { from: string; to: string } { + return useMemo(() => { + const now = new Date() + const to = now.toISOString() + + const ms: Record = { + '15m': 15 * 60 * 1000, + '1h': 60 * 60 * 1000, + '4h': 4 * 60 * 60 * 1000, + '12h': 12 * 60 * 60 * 1000, + '24h': 24 * 60 * 60 * 1000, + '7d': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + } + + const from = new Date(now.getTime() - ms[preset]).toISOString() + return { from, to } + }, [preset]) +} + +/** + * Get the appropriate bucket size in seconds for a given time range preset. + */ +export function getBucketSeconds(preset: TimeRangePreset): number { + const buckets: Record = { + '15m': 15, // 15s buckets → ~60 bars + '1h': 60, // 1min buckets → 60 bars + '4h': 240, // 4min buckets → 60 bars + '12h': 720, // 12min buckets → 60 bars + '24h': 1800, // 30min buckets → 48 bars + '7d': 10800, // 3h buckets → 56 bars + '30d': 43200, // 12h buckets → 60 bars + } + return buckets[preset] +} diff --git a/frontend/src/routes/requests/index.tsx b/frontend/src/routes/requests/index.tsx index 23249e7..dcfdece 100644 --- a/frontend/src/routes/requests/index.tsx +++ b/frontend/src/routes/requests/index.tsx @@ -1,4 +1,4 @@ -import { queryOptions, useSuspenseQuery } from '@tanstack/react-query' +import { queryOptions, useQuery, useSuspenseQuery } from '@tanstack/react-query' import { createFileRoute } from '@tanstack/react-router' import { zodValidator } from '@tanstack/zod-adapter' import { z } from 'zod' @@ -11,6 +11,8 @@ import { queryClient } from '@/components/app/query-provider' import i18n from '@/i18n' import type { ChatRequest } from '@/pages/requests/columns' import { RequestsDataTable } from '@/pages/requests/data-table' +import { SearchBar } from '@/pages/requests/search-bar' +import { AggregationResults } from '@/pages/search/aggregation-results' const requestsSearchSchema = z.object({ page: z.number().catch(1), @@ -19,6 +21,8 @@ const requestsSearchSchema = z.object({ upstreamId: z.number().optional(), model: z.string().optional(), selectedRequestId: z.number().optional(), + q: z.string().optional(), + range: z.enum(['15m', '1h', '4h', '12h', '24h', '7d', '30d']).catch('24h'), }) type RequestsSearchSchema = z.infer @@ -42,27 +46,142 @@ const requestsQueryOptions = ({ page, pageSize, apiKeyId, upstreamId, model }: R export const Route = createFileRoute('/requests/')({ validateSearch: zodValidator(requestsSearchSchema), - loaderDeps: ({ search: { page, pageSize, apiKeyId, upstreamId, model } }) => ({ + loaderDeps: ({ search: { page, pageSize, apiKeyId, upstreamId, model, q } }) => ({ page, pageSize, apiKeyId, upstreamId, model, + q, }), - loader: ({ deps }) => queryClient.ensureQueryData(requestsQueryOptions(deps)), + loader: ({ deps }) => { + // Only use the standard loader when there's no search query + if (!deps.q) { + return queryClient.ensureQueryData(requestsQueryOptions(deps as RequestsSearchSchema)) + } + return null + }, component: RouteComponent, errorComponent: AppErrorComponent, }) function RouteComponent() { + const { q } = Route.useSearch() + + // When there's a KQL search query, use the search endpoint + const isSearching = !!q?.trim() + + return ( +
+ + {isSearching ? ( + + ) : ( + + )} +
+ ) +} + +function DefaultResults() { const { page, pageSize, apiKeyId, upstreamId, model } = Route.useSearch() const { data: { data, total }, - } = useSuspenseQuery(requestsQueryOptions({ page, pageSize, apiKeyId, upstreamId, model })) + } = useSuspenseQuery(requestsQueryOptions({ page, pageSize, apiKeyId, upstreamId, model } as RequestsSearchSchema)) return ( -
+
-
+ + ) +} + +function SearchResults() { + const { q, range, page, pageSize } = Route.useSearch() + + const now = new Date() + const rangeMs: Record = { + '15m': 15 * 60_000, + '1h': 3600_000, + '4h': 4 * 3600_000, + '12h': 12 * 3600_000, + '24h': 24 * 3600_000, + '7d': 7 * 86400_000, + '30d': 30 * 86400_000, + } + const from = new Date(now.getTime() - (rangeMs[range ?? '24h'] ?? 86400_000)).toISOString() + const to = now.toISOString() + + const { data, isLoading, error } = useQuery({ + queryKey: ['search', { q, range, page, pageSize }], + queryFn: async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { data: result, error: err } = await (api.admin.search as any).post({ + query: q ?? '', + timeRange: { from, to }, + offset: (page - 1) * pageSize, + limit: pageSize, + }) + if (err) throw new Error('Search failed') + return result + }, + enabled: !!q?.trim(), + }) + + if (isLoading) { + return ( +
Searching...
+ ) + } + + if (error) { + return ( +
+ Search error: {error.message} +
+ ) + } + + if (!data) return null + + // Aggregation results + if (data.type === 'aggregation') { + return ( +
+ [] }).results} /> +
+ ) + } + + // Document results — normalize snake_case raw SQL rows to ChatRequest format + const rawRows = (data as { data: Record[]; total: number }) + const normalized = rawRows.data.map((row: Record) => ({ + id: row.id, + apiKeyId: row.api_key_id, + upstreamId: null, + modelId: null, + model: row.model, + status: row.status, + duration: row.duration, + ttft: row.ttft, + promptTokens: row.prompt_tokens, + completionTokens: row.completion_tokens, + createdAt: row.created_at, + updatedAt: row.updated_at, + deleted: false, + rating: row.rating ?? null, + reqId: row.req_id ?? null, + sourceCompletionId: null, + apiFormat: row.api_format ?? null, + cachedResponse: null, + prompt: typeof row.prompt === 'string' ? JSON.parse(row.prompt) : row.prompt, + completion: typeof row.completion === 'string' ? JSON.parse(row.completion) : row.completion, + providerName: row.provider_name ?? null, + })) as unknown as ChatRequest[] + + return ( +
+ +
) } From 5f2034b05d52af2a6aa7d1a698ae832d4233e624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E7=BF=94=E5=AE=87?= Date: Mon, 2 Feb 2026 02:11:29 +0800 Subject: [PATCH 2/5] refactor: move KQL validation to frontend, remove CLAUDE.md/TODOs.md - Copy KQL lexer/parser/types to frontend/src/lib/kql/ for client-side query validation instead of calling backend /search/validate endpoint - Remove backend /search/validate endpoint (no longer needed) - Add CLAUDE.md and TODOs.md to .gitignore Co-Authored-By: Claude Opus 4.5 --- .gitignore | 3 + backend/src/api/admin/search.ts | 16 - frontend/src/lib/kql/index.ts | 2 + frontend/src/lib/kql/lexer.ts | 187 ++++++++++++ frontend/src/lib/kql/parser.ts | 337 ++++++++++++++++++++++ frontend/src/lib/kql/types.ts | 93 ++++++ frontend/src/pages/search/query-input.tsx | 28 +- 7 files changed, 631 insertions(+), 35 deletions(-) create mode 100644 frontend/src/lib/kql/index.ts create mode 100644 frontend/src/lib/kql/lexer.ts create mode 100644 frontend/src/lib/kql/parser.ts create mode 100644 frontend/src/lib/kql/types.ts diff --git a/.gitignore b/.gitignore index d82cdb7..e7b04ec 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ coverage # Docs build output (generated from docs/ workspace) backend/docs/ +CLAUDE.md +TODOs.md + .turbo # Python test code (ignore all except essential files) diff --git a/backend/src/api/admin/search.ts b/backend/src/api/admin/search.ts index 24938cb..32d663f 100644 --- a/backend/src/api/admin/search.ts +++ b/backend/src/api/admin/search.ts @@ -129,22 +129,6 @@ export const adminSearch = new Elysia() }), }, ) - // Validate KQL query - .post( - "/search/validate", - ({ body }) => { - const result = parseKql(body.query); - if (result.success) { - return { valid: true, hasAggregation: !!result.query.aggregation }; - } - return { valid: false, error: result.error }; - }, - { - body: t.Object({ - query: t.String({ maxLength: 2000 }), - }), - }, - ) // Get searchable fields (for autocomplete) .get("/search/fields", async () => { const fields = getSearchableFields(); diff --git a/frontend/src/lib/kql/index.ts b/frontend/src/lib/kql/index.ts new file mode 100644 index 0000000..0f39eca --- /dev/null +++ b/frontend/src/lib/kql/index.ts @@ -0,0 +1,2 @@ +export { parseKql } from "./parser"; +export type { KqlQuery, ParseResult, ParseError } from "./types"; diff --git a/frontend/src/lib/kql/lexer.ts b/frontend/src/lib/kql/lexer.ts new file mode 100644 index 0000000..70e17f5 --- /dev/null +++ b/frontend/src/lib/kql/lexer.ts @@ -0,0 +1,187 @@ +import type { Token, TokenType } from "./types"; + +const KEYWORDS: Record = { + AND: "AND", + OR: "OR", + NOT: "NOT", + EXISTS: "EXISTS", + stats: "STATS", + by: "BY", +}; + +const OPERATORS = new Set([":", "=", "!=", ">", ">=", "<", "<="]); + +export class LexerError extends Error { + constructor( + message: string, + public position: number, + public length: number, + ) { + super(message); + this.name = "LexerError"; + } +} + +export function tokenize(input: string): Token[] { + const tokens: Token[] = []; + let pos = 0; + + const charAt = (i: number): string => input.charAt(i); + + while (pos < input.length) { + // Skip whitespace + if (/\s/.test(charAt(pos))) { + pos++; + continue; + } + + const start = pos; + const ch = charAt(pos); + + // Single-character tokens + if (ch === "(") { + tokens.push({ type: "LPAREN", value: "(", position: start }); + pos++; + continue; + } + if (ch === ")") { + tokens.push({ type: "RPAREN", value: ")", position: start }); + pos++; + continue; + } + if (ch === "|") { + tokens.push({ type: "PIPE", value: "|", position: start }); + pos++; + continue; + } + if (ch === ",") { + tokens.push({ type: "COMMA", value: ",", position: start }); + pos++; + continue; + } + + // Operators: !=, >=, <=, >, <, =, : + if (ch === "!" && pos + 1 < input.length && charAt(pos + 1) === "=") { + tokens.push({ type: "OPERATOR", value: "!=", position: start }); + pos += 2; + continue; + } + if (ch === ">" && pos + 1 < input.length && charAt(pos + 1) === "=") { + tokens.push({ type: "OPERATOR", value: ">=", position: start }); + pos += 2; + continue; + } + if (ch === "<" && pos + 1 < input.length && charAt(pos + 1) === "=") { + tokens.push({ type: "OPERATOR", value: "<=", position: start }); + pos += 2; + continue; + } + if (ch === ">") { + tokens.push({ type: "OPERATOR", value: ">", position: start }); + pos++; + continue; + } + if (ch === "<") { + tokens.push({ type: "OPERATOR", value: "<", position: start }); + pos++; + continue; + } + if (ch === "=") { + tokens.push({ type: "OPERATOR", value: "=", position: start }); + pos++; + continue; + } + if (ch === ":") { + tokens.push({ type: "OPERATOR", value: ":", position: start }); + pos++; + continue; + } + + // Quoted string + if (ch === '"') { + pos++; // skip opening quote + let value = ""; + while (pos < input.length && charAt(pos) !== '"') { + if (charAt(pos) === "\\" && pos + 1 < input.length) { + pos++; // skip backslash + value += charAt(pos); + } else { + value += charAt(pos); + } + pos++; + } + if (pos >= input.length) { + throw new LexerError("Unterminated string", start, pos - start); + } + pos++; // skip closing quote + tokens.push({ type: "STRING", value, position: start }); + continue; + } + + // Number (integer or decimal, optionally negative) + if (/\d/.test(ch) || (ch === "-" && pos + 1 < input.length && /\d/.test(charAt(pos + 1)))) { + let num = ch; + pos++; + while (pos < input.length && /[\d.]/.test(charAt(pos))) { + num += charAt(pos); + pos++; + } + tokens.push({ type: "NUMBER", value: num, position: start }); + continue; + } + + // Wildcard or unquoted identifier/field + if (ch === "*" || /[a-zA-Z_]/.test(ch)) { + let word = ""; + let hasWildcard = false; + while ( + pos < input.length && + /[a-zA-Z0-9_.*-]/.test(charAt(pos)) && + !OPERATORS.has(charAt(pos)) + ) { + // The `:` is an operator, so it stops identifier scanning + if (charAt(pos) === ":") { + break; + } + if (charAt(pos) === "*") { + hasWildcard = true; + } + word += charAt(pos); + pos++; + } + + // Check for keywords (case-sensitive for AND/OR/NOT, case-insensitive for stats/by) + const keyword = KEYWORDS[word] || KEYWORDS[word.toLowerCase()]; + if (keyword && !hasWildcard) { + // Only treat as keyword if it's AND/OR/NOT (uppercase) or stats/by (lowercase) + if ( + word === "AND" || + word === "OR" || + word === "NOT" || + word === "EXISTS" || + word.toLowerCase() === "stats" || + word.toLowerCase() === "by" + ) { + tokens.push({ type: keyword, value: word, position: start }); + continue; + } + } + + if (hasWildcard) { + tokens.push({ type: "WILDCARD", value: word, position: start }); + } else { + tokens.push({ type: "FIELD", value: word, position: start }); + } + continue; + } + + throw new LexerError( + `Unexpected character '${ch}'`, + pos, + 1, + ); + } + + tokens.push({ type: "EOF", value: "", position: pos }); + return tokens; +} diff --git a/frontend/src/lib/kql/parser.ts b/frontend/src/lib/kql/parser.ts new file mode 100644 index 0000000..9293b69 --- /dev/null +++ b/frontend/src/lib/kql/parser.ts @@ -0,0 +1,337 @@ +import { tokenize, LexerError } from "./lexer"; +import type { + Token, + TokenType, + KqlQuery, + KqlExpression, + KqlValue, + ComparisonOperator, + AggregateExpression, + AggregateFunction, + ParseResult, +} from "./types"; + +const AGGREGATE_FUNCTIONS = new Set([ + "count", + "avg", + "sum", + "min", + "max", + "p50", + "p95", + "p99", +]); + +class Parser { + private tokens: Token[]; + private pos: number; + + constructor(tokens: Token[]) { + this.tokens = tokens; + this.pos = 0; + } + + private current(): Token { + // pos is always valid (lexer ends with EOF token) + const token = this.tokens[this.pos]; + if (!token) { + throw new Error("Unexpected end of token stream"); + } + return token; + } + + private peek(type: TokenType): boolean { + return this.current().type === type; + } + + private advance(): Token { + const token = this.current(); + if (token.type !== "EOF") { + this.pos++; + } + return token; + } + + private expect(type: TokenType): Token { + const token = this.current(); + if (token.type !== type) { + throw new ParserError( + `Expected ${type} but found ${token.type}${token.value ? ` '${token.value}'` : ""}`, + token.position, + token.value.length || 1, + ); + } + return this.advance(); + } + + // query = filter_expr (PIPE "stats" agg_list ("by" field_list)?)? + parse(): KqlQuery { + const query: KqlQuery = {}; + + // Parse filter expression if present (not starting with pipe or at EOF) + if (!this.peek("EOF") && !this.peek("PIPE")) { + query.filter = this.parseOrExpr(); + } + + // Parse aggregation pipeline if present + if (this.peek("PIPE")) { + this.advance(); // consume PIPE + this.expect("STATS"); + + const functions = this.parseAggList(); + let groupBy: string[] | undefined; + + if (this.peek("BY")) { + this.advance(); // consume BY + groupBy = this.parseFieldList(); + } + + query.aggregation = { functions, groupBy }; + } + + if (!this.peek("EOF")) { + const token = this.current(); + throw new ParserError( + `Unexpected token '${token.value}'`, + token.position, + token.value.length || 1, + ); + } + + return query; + } + + // or_expr = and_expr ("OR" and_expr)* + private parseOrExpr(): KqlExpression { + let left = this.parseAndExpr(); + + while (this.peek("OR")) { + this.advance(); // consume OR + const right = this.parseAndExpr(); + left = { type: "or", left, right }; + } + + return left; + } + + // and_expr = not_expr ("AND" not_expr)* + private parseAndExpr(): KqlExpression { + let left = this.parseNotExpr(); + + while (this.peek("AND")) { + this.advance(); // consume AND + const right = this.parseNotExpr(); + left = { type: "and", left, right }; + } + + return left; + } + + // not_expr = "NOT" not_expr | primary + private parseNotExpr(): KqlExpression { + if (this.peek("NOT")) { + this.advance(); // consume NOT + const expression = this.parseNotExpr(); + return { type: "not", expression }; + } + + return this.parsePrimary(); + } + + // primary = LPAREN or_expr RPAREN | comparison + private parsePrimary(): KqlExpression { + if (this.peek("LPAREN")) { + this.advance(); // consume LPAREN + const expression = this.parseOrExpr(); + this.expect("RPAREN"); + return { type: "group", expression }; + } + + return this.parseComparison(); + } + + // comparison = FIELD (operator value | "EXISTS") + private parseComparison(): KqlExpression { + const fieldToken = this.current(); + + if (fieldToken.type !== "FIELD") { + throw new ParserError( + `Expected field name but found ${fieldToken.type}${fieldToken.value ? ` '${fieldToken.value}'` : ""}`, + fieldToken.position, + fieldToken.value.length || 1, + ); + } + this.advance(); // consume FIELD + + // Check for EXISTS keyword (e.g., "extraBody EXISTS") + if (this.peek("EXISTS")) { + this.advance(); // consume EXISTS + return { type: "exists", field: fieldToken.value }; + } + + const opToken = this.current(); + if (opToken.type !== "OPERATOR") { + throw new ParserError( + `Expected operator after field '${fieldToken.value}'`, + opToken.position, + opToken.value.length || 1, + ); + } + const operator = opToken.value as ComparisonOperator; + this.advance(); // consume OPERATOR + + const value = this.parseValue(); + + return { + type: "comparison", + field: fieldToken.value, + operator, + value, + }; + } + + // value = STRING | NUMBER | WILDCARD | FIELD (unquoted string treated as string value) + private parseValue(): KqlValue { + const token = this.current(); + + if (token.type === "STRING") { + this.advance(); + return { type: "string", value: token.value }; + } + + if (token.type === "NUMBER") { + this.advance(); + return { type: "number", value: Number(token.value) }; + } + + if (token.type === "WILDCARD") { + this.advance(); + return { type: "wildcard", pattern: token.value }; + } + + // Allow unquoted identifiers as string values (e.g., `status: completed`) + if (token.type === "FIELD") { + this.advance(); + return { type: "string", value: token.value }; + } + + throw new ParserError( + `Expected value after operator`, + token.position, + token.value.length || 1, + ); + } + + // agg_list = agg_fn ("," agg_fn)* + private parseAggList(): AggregateExpression[] { + const functions: AggregateExpression[] = []; + functions.push(this.parseAggFn()); + + while (this.peek("COMMA")) { + this.advance(); // consume COMMA + functions.push(this.parseAggFn()); + } + + return functions; + } + + // agg_fn = FIELD "(" FIELD? ")" + private parseAggFn(): AggregateExpression { + const fnToken = this.current(); + if (fnToken.type !== "FIELD") { + throw new ParserError( + `Expected aggregate function name`, + fnToken.position, + fnToken.value.length || 1, + ); + } + + const fnName = fnToken.value.toLowerCase(); + if (!AGGREGATE_FUNCTIONS.has(fnName as AggregateFunction)) { + throw new ParserError( + `Unknown aggregate function '${fnToken.value}'. Supported: ${[...AGGREGATE_FUNCTIONS].join(", ")}`, + fnToken.position, + fnToken.value.length, + ); + } + this.advance(); // consume function name + + this.expect("LPAREN"); + + let field: string | undefined; + if (!this.peek("RPAREN")) { + const fieldToken = this.expect("FIELD"); + field = fieldToken.value; + } + + this.expect("RPAREN"); + + // Validate count() has no field, others require a field + if (fnName === "count" && field) { + throw new ParserError( + `count() does not take a field argument`, + fnToken.position, + fnToken.value.length, + ); + } + if (fnName !== "count" && !field) { + throw new ParserError( + `${fnName}() requires a field argument`, + fnToken.position, + fnToken.value.length, + ); + } + + return { fn: fnName as AggregateFunction, field }; + } + + // field_list = FIELD ("," FIELD)* + private parseFieldList(): string[] { + const fields: string[] = []; + fields.push(this.expect("FIELD").value); + + while (this.peek("COMMA")) { + this.advance(); // consume COMMA + fields.push(this.expect("FIELD").value); + } + + return fields; + } +} + +class ParserError extends Error { + constructor( + message: string, + public position: number, + public length: number, + ) { + super(message); + this.name = "ParserError"; + } +} + +export function parseKql(input: string): ParseResult { + try { + const trimmed = input.trim(); + if (trimmed === "") { + return { success: true, query: {} }; + } + + const tokens = tokenize(trimmed); + const parser = new Parser(tokens); + const query = parser.parse(); + return { success: true, query }; + } catch (err) { + if (err instanceof ParserError || err instanceof LexerError) { + return { + success: false, + error: { + message: err.message, + position: err.position, + length: err.length, + }, + }; + } + throw err; + } +} diff --git a/frontend/src/lib/kql/types.ts b/frontend/src/lib/kql/types.ts new file mode 100644 index 0000000..17fd246 --- /dev/null +++ b/frontend/src/lib/kql/types.ts @@ -0,0 +1,93 @@ +// KQL (Kibana Query Language) AST types for advanced meta search + +// --- Token types (used by lexer) --- + +export type TokenType = + | "FIELD" + | "STRING" + | "NUMBER" + | "WILDCARD" + | "OPERATOR" + | "AND" + | "OR" + | "NOT" + | "LPAREN" + | "RPAREN" + | "PIPE" + | "STATS" + | "BY" + | "COMMA" + | "EXISTS" + | "EOF"; + +export interface Token { + type: TokenType; + value: string; + position: number; +} + +// --- AST node types --- + +export type KqlValue = + | { type: "string"; value: string } + | { type: "number"; value: number } + | { type: "wildcard"; pattern: string }; + +export type ComparisonOperator = + | ":" + | "=" + | "!=" + | ">" + | ">=" + | "<" + | "<="; + +export type KqlExpression = + | { + type: "comparison"; + field: string; + operator: ComparisonOperator; + value: KqlValue; + } + | { type: "and"; left: KqlExpression; right: KqlExpression } + | { type: "or"; left: KqlExpression; right: KqlExpression } + | { type: "not"; expression: KqlExpression } + | { type: "group"; expression: KqlExpression } + | { type: "exists"; field: string }; + +export type AggregateFunction = + | "count" + | "avg" + | "sum" + | "min" + | "max" + | "p50" + | "p95" + | "p99"; + +export interface AggregateExpression { + fn: AggregateFunction; + field?: string; // undefined for count() +} + +export interface KqlAggregation { + functions: AggregateExpression[]; + groupBy?: string[]; +} + +export interface KqlQuery { + filter?: KqlExpression; + aggregation?: KqlAggregation; +} + +// --- Parse result types --- + +export interface ParseError { + message: string; + position: number; + length: number; +} + +export type ParseResult = + | { success: true; query: KqlQuery } + | { success: false; error: ParseError }; diff --git a/frontend/src/pages/search/query-input.tsx b/frontend/src/pages/search/query-input.tsx index 0042caa..e91a956 100644 --- a/frontend/src/pages/search/query-input.tsx +++ b/frontend/src/pages/search/query-input.tsx @@ -3,6 +3,7 @@ import { useQuery } from '@tanstack/react-query' import { SearchIcon } from 'lucide-react' import { api } from '@/lib/api' +import { parseKql } from '@/lib/kql' import { cn } from '@/lib/utils' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' @@ -119,32 +120,21 @@ export function QueryInput({ value, onChange, onSubmit, className }: QueryInputP const suggestions = getSuggestions() - // Validate query on change (debounced) + // Validate query on change (debounced, purely client-side) useEffect(() => { if (!value.trim()) { setValidationError(null) return } - const timeout = setTimeout(async () => { - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const { data, error } = await (api.admin.search.validate as any).post({ - query: value, - }) - if (error || !data) { - setValidationError(null) - return - } - if (data.valid) { - setValidationError(null) - } else if (data.error) { - setValidationError(data.error.message) - } - } catch { - // Ignore validation errors during typing + const timeout = setTimeout(() => { + const result = parseKql(value) + if (result.success) { + setValidationError(null) + } else { + setValidationError(result.error.message) } - }, 500) + }, 300) return () => clearTimeout(timeout) }, [value]) From d49dbbe10d58f3058031f1558270b8ae2239bc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E7=BF=94=E5=AE=87?= Date: Mon, 2 Feb 2026 02:49:44 +0800 Subject: [PATCH 3/5] fix: address code review issues in KQL search - Sanitize error responses: replace String(err) with server-side logging via createLogger, remove internal details from client responses - Fix parseTimeRange: require both from and to (no silent defaults) - Consolidate duplicated rangeMs lookups into shared TIME_RANGE_MS constant and getTimeRangeISO() utility in time-range-picker.tsx - Fix null safety in aggregation-results.tsx for results[0] - Add SAFETY comment documenting trust boundary for sql.raw() in aggregation queries - Remove unused useTimeRangeDates hook and useMemo import Co-Authored-By: Claude Opus 4.5 --- backend/src/api/admin/search.ts | 17 ++++--- backend/src/db/index.ts | 2 + frontend/src/pages/requests/search-bar.tsx | 17 +------ .../src/pages/search/aggregation-results.tsx | 6 ++- .../src/pages/search/time-range-picker.tsx | 45 +++++++++---------- frontend/src/routes/requests/index.tsx | 14 +----- 6 files changed, 41 insertions(+), 60 deletions(-) diff --git a/backend/src/api/admin/search.ts b/backend/src/api/admin/search.ts index 32d663f..a177571 100644 --- a/backend/src/api/admin/search.ts +++ b/backend/src/api/admin/search.ts @@ -10,17 +10,20 @@ import { searchCompletionsTimeSeries, getDistinctFieldValues, } from "@/db"; +import { createLogger } from "@/utils/logger"; + +const logger = createLogger("search"); function parseTimeRange( from?: string, to?: string, ): { from: Date; to: Date } | undefined { - if (!from && !to) { + if (!from || !to) { return undefined; } return { - from: from ? new Date(from) : new Date(Date.now() - 3600_000), // default: 1h ago - to: to ? new Date(to) : new Date(), + from: new Date(from), + to: new Date(to), }; } @@ -46,9 +49,9 @@ export const adminSearch = new Elysia() const results = await aggregateCompletions(compiled); return { type: "aggregation" as const, results }; } catch (err) { + logger.error("Aggregation failed", { error: err }); return status(500, { error: "Aggregation failed", - details: String(err), }); } } @@ -68,9 +71,9 @@ export const adminSearch = new Elysia() }); return { type: "documents" as const, ...data }; } catch (err) { + logger.error("Search failed", { error: err }); return status(500, { error: "Search failed", - details: String(err), }); } }, @@ -110,9 +113,9 @@ export const adminSearch = new Elysia() ); return { buckets }; } catch (err) { + logger.error("Histogram query failed", { error: err }); return status(500, { error: "Histogram query failed", - details: String(err), }); } }, @@ -206,9 +209,9 @@ export const adminSearch = new Elysia() 'attachment; filename="search-results.json"'; return JSON.stringify(data.data, null, 2); } catch (err) { + logger.error("Export failed", { error: err }); return status(500, { error: "Export failed", - details: String(err), }); } }, diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index 836c5ff..1f77258 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -2041,6 +2041,8 @@ export async function aggregateCompletions( compiled.aggregation; // Build SELECT clause + // SAFETY: selectExpressions and groupByColumn are produced by the compiler + // from the trusted FIELD_REGISTRY whitelist — they never contain user input. const selectParts: string[] = []; if (groupByColumn && groupByField) { selectParts.push(`${groupByColumn} AS "${groupByField}"`); diff --git a/frontend/src/pages/requests/search-bar.tsx b/frontend/src/pages/requests/search-bar.tsx index 9e9d559..d24e998 100644 --- a/frontend/src/pages/requests/search-bar.tsx +++ b/frontend/src/pages/requests/search-bar.tsx @@ -4,7 +4,7 @@ import { XIcon } from 'lucide-react' import { Button } from '@/components/ui/button' import { QueryInput } from '@/pages/search/query-input' -import { TimeRangePicker, type TimeRangePreset } from '@/pages/search/time-range-picker' +import { TimeRangePicker, getTimeRangeISO, type TimeRangePreset } from '@/pages/search/time-range-picker' import { ExportButton } from '@/pages/search/export-button' export function SearchBar() { @@ -52,21 +52,8 @@ export function SearchBar() { const isSearching = !!q?.trim() - const now = new Date() - const rangeMs: Record = { - '15m': 15 * 60_000, - '1h': 3600_000, - '4h': 4 * 3600_000, - '12h': 12 * 3600_000, - '24h': 24 * 3600_000, - '7d': 7 * 86400_000, - '30d': 30 * 86400_000, - } const timeRange = isSearching - ? { - from: new Date(now.getTime() - (rangeMs[range ?? '24h'] ?? 86400_000)).toISOString(), - to: now.toISOString(), - } + ? getTimeRangeISO((range as TimeRangePreset) ?? '24h') : undefined return ( diff --git a/frontend/src/pages/search/aggregation-results.tsx b/frontend/src/pages/search/aggregation-results.tsx index 328dda8..f017cc1 100644 --- a/frontend/src/pages/search/aggregation-results.tsx +++ b/frontend/src/pages/search/aggregation-results.tsx @@ -10,7 +10,11 @@ export function AggregationResults({ results }: AggregationResultsProps) { } // Get column names from first result - const columns = Object.keys(results[0]) + const firstRow = results[0] + if (!firstRow) { + return
No aggregation results
+ } + const columns = Object.keys(firstRow) return (
diff --git a/frontend/src/pages/search/time-range-picker.tsx b/frontend/src/pages/search/time-range-picker.tsx index 9efbf27..19235a8 100644 --- a/frontend/src/pages/search/time-range-picker.tsx +++ b/frontend/src/pages/search/time-range-picker.tsx @@ -1,9 +1,27 @@ -import { useMemo } from 'react' - import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' export type TimeRangePreset = '15m' | '1h' | '4h' | '12h' | '24h' | '7d' | '30d' +/** Milliseconds for each time range preset. */ +export const TIME_RANGE_MS: Record = { + '15m': 15 * 60_000, + '1h': 3600_000, + '4h': 4 * 3600_000, + '12h': 12 * 3600_000, + '24h': 24 * 3600_000, + '7d': 7 * 86400_000, + '30d': 30 * 86400_000, +} + +/** Compute from/to ISO strings for a preset relative to now. */ +export function getTimeRangeISO(preset: TimeRangePreset): { from: string; to: string } { + const now = new Date() + return { + from: new Date(now.getTime() - TIME_RANGE_MS[preset]).toISOString(), + to: now.toISOString(), + } +} + const TIME_RANGES: { value: TimeRangePreset; label: string }[] = [ { value: '15m', label: 'Last 15 min' }, { value: '1h', label: 'Last 1 hour' }, @@ -36,29 +54,6 @@ export function TimeRangePicker({ value, onChange }: TimeRangePickerProps) { ) } -/** - * Convert a time range preset to from/to Date objects. - */ -export function useTimeRangeDates(preset: TimeRangePreset): { from: string; to: string } { - return useMemo(() => { - const now = new Date() - const to = now.toISOString() - - const ms: Record = { - '15m': 15 * 60 * 1000, - '1h': 60 * 60 * 1000, - '4h': 4 * 60 * 60 * 1000, - '12h': 12 * 60 * 60 * 1000, - '24h': 24 * 60 * 60 * 1000, - '7d': 7 * 24 * 60 * 60 * 1000, - '30d': 30 * 24 * 60 * 60 * 1000, - } - - const from = new Date(now.getTime() - ms[preset]).toISOString() - return { from, to } - }, [preset]) -} - /** * Get the appropriate bucket size in seconds for a given time range preset. */ diff --git a/frontend/src/routes/requests/index.tsx b/frontend/src/routes/requests/index.tsx index dcfdece..34fe545 100644 --- a/frontend/src/routes/requests/index.tsx +++ b/frontend/src/routes/requests/index.tsx @@ -13,6 +13,7 @@ import type { ChatRequest } from '@/pages/requests/columns' import { RequestsDataTable } from '@/pages/requests/data-table' import { SearchBar } from '@/pages/requests/search-bar' import { AggregationResults } from '@/pages/search/aggregation-results' +import { getTimeRangeISO, type TimeRangePreset } from '@/pages/search/time-range-picker' const requestsSearchSchema = z.object({ page: z.number().catch(1), @@ -99,18 +100,7 @@ function DefaultResults() { function SearchResults() { const { q, range, page, pageSize } = Route.useSearch() - const now = new Date() - const rangeMs: Record = { - '15m': 15 * 60_000, - '1h': 3600_000, - '4h': 4 * 3600_000, - '12h': 12 * 3600_000, - '24h': 24 * 3600_000, - '7d': 7 * 86400_000, - '30d': 30 * 86400_000, - } - const from = new Date(now.getTime() - (rangeMs[range ?? '24h'] ?? 86400_000)).toISOString() - const to = now.toISOString() + const { from, to } = getTimeRangeISO((range ?? '24h') as TimeRangePreset) const { data, isLoading, error } = useQuery({ queryKey: ['search', { q, range, page, pageSize }], From 1f2518f6f608e90314b47e4d9fd3384c943187e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E7=BF=94=E5=AE=87?= Date: Mon, 2 Feb 2026 02:55:37 +0800 Subject: [PATCH 4/5] fix: address automated review comments in KQL search - Guard jsonb_array_elements against non-array values in compiler - Add explicit error for multiple GROUP BY fields - Fix number lexer to reject multi-dot numbers (e.g. 1.2.3) - Wrap parseTimeRange with try-catch in /search/export endpoint - Use escapeCsvField for all CSV fields in export - Add safe JSON.parse wrapper in search results normalization - Sync search bar text with URL query param on navigation - Add NaN fallback for histogram chart data - Log export errors to console instead of silently swallowing Co-Authored-By: Claude Opus 4.5 --- backend/src/api/admin/search.ts | 67 ++++++++++++++----- backend/src/search/compiler.ts | 11 +-- backend/src/search/lexer.ts | 7 ++ frontend/src/lib/kql/lexer.ts | 5 ++ frontend/src/pages/requests/search-bar.tsx | 7 +- frontend/src/pages/search/export-button.tsx | 4 +- .../src/pages/search/search-histogram.tsx | 6 +- frontend/src/routes/requests/index.tsx | 12 +++- 8 files changed, 89 insertions(+), 30 deletions(-) diff --git a/backend/src/api/admin/search.ts b/backend/src/api/admin/search.ts index a177571..6fc5c9a 100644 --- a/backend/src/api/admin/search.ts +++ b/backend/src/api/admin/search.ts @@ -21,10 +21,26 @@ function parseTimeRange( if (!from || !to) { return undefined; } - return { - from: new Date(from), - to: new Date(to), - }; + const fromDate = new Date(from); + const toDate = new Date(to); + if (Number.isNaN(fromDate.getTime()) || Number.isNaN(toDate.getTime())) { + throw new Error("Invalid timeRange date"); + } + if (fromDate > toDate) { + throw new Error("timeRange.from must be <= timeRange.to"); + } + return { from: fromDate, to: toDate }; +} + +function escapeCsvField(value: unknown): string { + if (value == null) { + return ""; + } + const str = typeof value === "object" ? JSON.stringify(value) : String(value as string | number | boolean); + if (str.includes(",") || str.includes("\n") || str.includes("\r") || str.includes('"')) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; } export const adminSearch = new Elysia() @@ -40,7 +56,12 @@ export const adminSearch = new Elysia() }); } - const timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + let timeRange: { from: Date; to: Date } | undefined; + try { + timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + } catch (err) { + return status(400, { error: err instanceof Error ? err.message : "Invalid timeRange" }); + } const compiled = compileSearch(result.query, { timeRange }); // If the query has aggregation, return aggregation results @@ -103,7 +124,12 @@ export const adminSearch = new Elysia() }); } - const timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + let timeRange: { from: Date; to: Date } | undefined; + try { + timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + } catch (err) { + return status(400, { error: err instanceof Error ? err.message : "Invalid timeRange" }); + } const compiled = compileSearch(result.query, { timeRange }); try { @@ -160,7 +186,12 @@ export const adminSearch = new Elysia() }); } - const timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + let timeRange: { from: Date; to: Date } | undefined; + try { + timeRange = parseTimeRange(body.timeRange?.from, body.timeRange?.to); + } catch (err) { + return status(400, { error: err instanceof Error ? err.message : "Invalid timeRange" }); + } const compiled = compileSearch(result.query, { timeRange }); try { @@ -187,17 +218,17 @@ export const adminSearch = new Elysia() ]; const rows = data.data.map((row) => [ - row.id, - `"${(row.model || "").replace(/"/g, '""')}"`, - row.status, - row.duration, - row.ttft, - row.prompt_tokens, - row.completion_tokens, - row.created_at, - `"${(row.provider_name || "").replace(/"/g, '""')}"`, - row.api_format || "", - row.rating ?? "", + escapeCsvField(row.id), + escapeCsvField(row.model), + escapeCsvField(row.status), + escapeCsvField(row.duration), + escapeCsvField(row.ttft), + escapeCsvField(row.prompt_tokens), + escapeCsvField(row.completion_tokens), + escapeCsvField(row.created_at), + escapeCsvField(row.provider_name), + escapeCsvField(row.api_format), + escapeCsvField(row.rating), ].join(","), ); return [headers.join(","), ...rows].join("\n"); diff --git a/backend/src/search/compiler.ts b/backend/src/search/compiler.ts index 18f015c..d3654aa 100644 --- a/backend/src/search/compiler.ts +++ b/backend/src/search/compiler.ts @@ -269,8 +269,9 @@ class SqlCompiler { // Array-rooted JSONB (e.g., toolCalls — completion is an array of messages) if (mapping.jsonbRootIsArray) { + const safeUnwrapped = `CASE WHEN jsonb_typeof(${unwrapped}) = 'array' THEN ${unwrapped} ELSE '[]'::jsonb END`; if (parts.length === 1) { - return `EXISTS (SELECT 1 FROM jsonb_array_elements(${unwrapped}) _elem WHERE _elem->'${mapping.jsonbRootKey}' IS NOT NULL)`; + return `EXISTS (SELECT 1 FROM jsonb_array_elements(${safeUnwrapped}) _elem WHERE _elem->'${mapping.jsonbRootKey}' IS NOT NULL)`; } const pathSegments = parts.slice(1); validateJsonbPath(pathSegments); @@ -279,7 +280,7 @@ class SqlCompiler { pathSql += `->'${pathSegments[i]}'`; } pathSql += `->>'${pathSegments[pathSegments.length - 1]}'`; - return `EXISTS (SELECT 1 FROM jsonb_array_elements(${unwrapped}) _elem WHERE ${pathSql} IS NOT NULL)`; + return `EXISTS (SELECT 1 FROM jsonb_array_elements(${safeUnwrapped}) _elem WHERE ${pathSql} IS NOT NULL)`; } if (parts.length === 1) { @@ -345,7 +346,7 @@ class SqlCompiler { } } - return `EXISTS (SELECT 1 FROM jsonb_array_elements(${unwrapped}) _msg, jsonb_array_elements(_msg->'${mapping.jsonbRootKey}') _tc WHERE ${comparison})`; + return `EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(${unwrapped}) = 'array' THEN ${unwrapped} ELSE '[]'::jsonb END) _msg, jsonb_array_elements(CASE WHEN jsonb_typeof(_msg->'${mapping.jsonbRootKey}') = 'array' THEN _msg->'${mapping.jsonbRootKey}' ELSE '[]'::jsonb END) _tc WHERE ${comparison})`; } private compileComparison(expr: { @@ -549,7 +550,9 @@ class SqlCompiler { let groupByField: string | undefined; if (agg.groupBy && agg.groupBy.length > 0) { - // Currently support single GROUP BY field + if (agg.groupBy.length > 1) { + throw new CompilerError("Only a single GROUP BY field is supported"); + } const [field] = agg.groupBy; if (!field) { throw new CompilerError("Empty GROUP BY field"); diff --git a/backend/src/search/lexer.ts b/backend/src/search/lexer.ts index 70e17f5..ba27fe5 100644 --- a/backend/src/search/lexer.ts +++ b/backend/src/search/lexer.ts @@ -122,7 +122,14 @@ export function tokenize(input: string): Token[] { if (/\d/.test(ch) || (ch === "-" && pos + 1 < input.length && /\d/.test(charAt(pos + 1)))) { let num = ch; pos++; + let hasDot = false; while (pos < input.length && /[\d.]/.test(charAt(pos))) { + if (charAt(pos) === ".") { + if (hasDot) { + break; // Stop at second dot to reject "1.2.3" + } + hasDot = true; + } num += charAt(pos); pos++; } diff --git a/frontend/src/lib/kql/lexer.ts b/frontend/src/lib/kql/lexer.ts index 70e17f5..2547af7 100644 --- a/frontend/src/lib/kql/lexer.ts +++ b/frontend/src/lib/kql/lexer.ts @@ -122,7 +122,12 @@ export function tokenize(input: string): Token[] { if (/\d/.test(ch) || (ch === "-" && pos + 1 < input.length && /\d/.test(charAt(pos + 1)))) { let num = ch; pos++; + let hasDot = false; while (pos < input.length && /[\d.]/.test(charAt(pos))) { + if (charAt(pos) === ".") { + if (hasDot) break; // Stop at second dot to reject "1.2.3" + hasDot = true; + } num += charAt(pos); pos++; } diff --git a/frontend/src/pages/requests/search-bar.tsx b/frontend/src/pages/requests/search-bar.tsx index d24e998..2649752 100644 --- a/frontend/src/pages/requests/search-bar.tsx +++ b/frontend/src/pages/requests/search-bar.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useEffect, useState } from 'react' import { useNavigate, useSearch } from '@tanstack/react-router' import { XIcon } from 'lucide-react' @@ -13,6 +13,11 @@ export function SearchBar() { const [queryText, setQueryText] = useState(q ?? '') + // Sync queryText with URL q param on back/forward navigation + useEffect(() => { + setQueryText(q ?? '') + }, [q]) + const handleSubmit = () => { const trimmed = queryText.trim() navigate({ diff --git a/frontend/src/pages/search/export-button.tsx b/frontend/src/pages/search/export-button.tsx index f0624c9..701bc94 100644 --- a/frontend/src/pages/search/export-button.tsx +++ b/frontend/src/pages/search/export-button.tsx @@ -45,8 +45,8 @@ export function ExportButton({ query, timeRange, disabled }: ExportButtonProps) a.click() document.body.removeChild(a) URL.revokeObjectURL(url) - } catch { - // Silently fail — could add toast notification here + } catch (err) { + console.error('Export failed', err) } finally { setLoading(false) } diff --git a/frontend/src/pages/search/search-histogram.tsx b/frontend/src/pages/search/search-histogram.tsx index 24b1240..6f32740 100644 --- a/frontend/src/pages/search/search-histogram.tsx +++ b/frontend/src/pages/search/search-histogram.tsx @@ -18,9 +18,9 @@ interface SearchHistogramProps { export const SearchHistogram = memo(function SearchHistogram({ data }: SearchHistogramProps) { const chartData = data.map((item) => ({ timestamp: typeof item.bucket === 'string' ? item.bucket : item.bucket.toISOString(), - completed: Number(item.completed), - failed: Number(item.failed), - other: Math.max(0, Number(item.total) - Number(item.completed) - Number(item.failed)), + completed: Number(item.completed) || 0, + failed: Number(item.failed) || 0, + other: Math.max(0, (Number(item.total) || 0) - (Number(item.completed) || 0) - (Number(item.failed) || 0)), })) if (chartData.length === 0) return null diff --git a/frontend/src/routes/requests/index.tsx b/frontend/src/routes/requests/index.tsx index 34fe545..52d828d 100644 --- a/frontend/src/routes/requests/index.tsx +++ b/frontend/src/routes/requests/index.tsx @@ -97,6 +97,14 @@ function DefaultResults() { ) } +function safeJsonParse(value: string): unknown { + try { + return JSON.parse(value) + } catch { + return null + } +} + function SearchResults() { const { q, range, page, pageSize } = Route.useSearch() @@ -164,8 +172,8 @@ function SearchResults() { sourceCompletionId: null, apiFormat: row.api_format ?? null, cachedResponse: null, - prompt: typeof row.prompt === 'string' ? JSON.parse(row.prompt) : row.prompt, - completion: typeof row.completion === 'string' ? JSON.parse(row.completion) : row.completion, + prompt: typeof row.prompt === 'string' ? safeJsonParse(row.prompt) : row.prompt, + completion: typeof row.completion === 'string' ? safeJsonParse(row.completion) : row.completion, providerName: row.provider_name ?? null, })) as unknown as ChatRequest[] From 946a182613823028961176baafd34e39b25f7857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E7=BF=94=E5=AE=87?= Date: Mon, 2 Feb 2026 02:57:08 +0800 Subject: [PATCH 5/5] fix: return 400 for CompilerError in search endpoints compileSearch throws CompilerError for user input errors (unknown fields, invalid enum values, etc.). Wrap all three endpoints with try-catch to return 400 instead of letting it bubble up as 500. Co-Authored-By: Claude Opus 4.5 --- backend/src/api/admin/search.ts | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/backend/src/api/admin/search.ts b/backend/src/api/admin/search.ts index 6fc5c9a..0c2df51 100644 --- a/backend/src/api/admin/search.ts +++ b/backend/src/api/admin/search.ts @@ -62,7 +62,15 @@ export const adminSearch = new Elysia() } catch (err) { return status(400, { error: err instanceof Error ? err.message : "Invalid timeRange" }); } - const compiled = compileSearch(result.query, { timeRange }); + let compiled; + try { + compiled = compileSearch(result.query, { timeRange }); + } catch (err) { + return status(400, { + error: "Invalid query", + details: err instanceof Error ? err.message : "Compilation failed", + }); + } // If the query has aggregation, return aggregation results if (compiled.aggregation) { @@ -130,7 +138,15 @@ export const adminSearch = new Elysia() } catch (err) { return status(400, { error: err instanceof Error ? err.message : "Invalid timeRange" }); } - const compiled = compileSearch(result.query, { timeRange }); + let compiled; + try { + compiled = compileSearch(result.query, { timeRange }); + } catch (err) { + return status(400, { + error: "Invalid query", + details: err instanceof Error ? err.message : "Compilation failed", + }); + } try { const buckets = await searchCompletionsTimeSeries( @@ -192,7 +208,15 @@ export const adminSearch = new Elysia() } catch (err) { return status(400, { error: err instanceof Error ? err.message : "Invalid timeRange" }); } - const compiled = compileSearch(result.query, { timeRange }); + let compiled; + try { + compiled = compileSearch(result.query, { timeRange }); + } catch (err) { + return status(400, { + error: "Invalid query", + details: err instanceof Error ? err.message : "Compilation failed", + }); + } try { // Fetch all results (up to 10000 for export)