release: dev → main — pgserve storage, benchmarks, sessions, config CLI

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@automagik/rlmx",
-  "version": "0.260330.13",
+  "version": "0.260331.1",
   "description": "RLM algorithm CLI for coding agents — prompt externalization, Python REPL with symbolic recursion, code-driven navigation",
   "type": "module",
   "publishConfig": {
@@ -17,7 +17,7 @@
     "examples"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && cp src/benchmark-data.json dist/src/",
     "dev": "tsc --watch",
     "clean": "rm -rf dist",
     "test": "node --test dist/tests/*.test.js",
@@ -27,13 +27,16 @@
   },
   "dependencies": {
     "@mariozechner/pi-ai": "0.64.0",
-    "js-yaml": "^4.1.1"
+    "js-yaml": "^4.1.1",
+    "pg": "^8.20.0",
+    "pgserve": "^1.1.6"
   },
   "devDependencies": {
     "@commitlint/cli": "^19.0.0",
     "@commitlint/config-conventional": "^19.0.0",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.0.0",
+    "@types/pg": "^8.11.0",
     "husky": "^9.0.0",
     "typescript": "^5.7.0"
   },

python/gemini_batteries.py

@@ -22,7 +22,7 @@ def web_search(query: str) -> str:
 
     Returns search results as text. Only available with provider: google.
     """
-    results = llm_bridge._send_request("web_search", [query])
+    results = llm_bridge.send_request("web_search", [query])
     return results[0] if results else "Error: no response from web_search"
 
 
@@ -31,7 +31,7 @@ def fetch_url(url: str) -> str:
 
     Returns page content as text. Only available with provider: google.
     """
-    results = llm_bridge._send_request("fetch_url", [url])
+    results = llm_bridge.send_request("fetch_url", [url])
     return results[0] if results else "Error: no response from fetch_url"
 
 
@@ -42,5 +42,5 @@ def generate_image(prompt: str, aspect_ratio: str = "16:9", size: str = "2K") ->
     Only available with provider: google.
     """
     full_prompt = f"{prompt} [aspect_ratio={aspect_ratio}, size={size}]"
-    results = llm_bridge._send_request("generate_image", [full_prompt])
+    results = llm_bridge.send_request("generate_image", [full_prompt])
     return results[0] if results else "Error: no response from generate_image"

python/llm_bridge.py

@@ -50,6 +50,11 @@ def _send_request(request_type: str, prompts: list, model=None) -> list:
             return [f"Error: invalid JSON response: {e}"] * len(prompts)
 
 
+def send_request(request_type: str, prompts: list, model=None) -> list:
+    """Public interface for sending IPC requests to Node.js parent process."""
+    return _send_request(request_type, prompts, model)
+
+
 def llm_query(prompt: str, model=None) -> str:
     """Query the LLM with a single prompt. Returns the response string."""
     results = _send_request("llm_query", [prompt], model)

python/load_dataset.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Load Oolong Synth dataset from HuggingFace and output as JSON."""
+import json
+import sys
+from datasets import load_dataset
+
+
+def main():
+    samples = int(sys.argv[1]) if len(sys.argv) > 1 else 5
+    idx = int(sys.argv[2]) if len(sys.argv) > 2 else -1
+
+    ds = load_dataset("oolongbench/oolong-synth", split="test")
+
+    if idx >= 0:
+        items = [ds[idx]]
+    else:
+        items = list(ds.select(range(min(samples, len(ds)))))
+
+    output = []
+    for item in items:
+        # Oolong Synth uses "context_window_text" for the context field
+        context = item.get("context_window_text") or item.get("context", "")
+        answer = item.get("answer", "")
+        # answer can be a list — join if so
+        if isinstance(answer, list):
+            answer = ", ".join(str(a) for a in answer)
+        output.append({
+            "id": f"oolong-{item.get('id', 'unknown')}",
+            "name": item.get("question", "")[:50],
+            "question": item["question"],
+            "context": context,
+            "expected": answer,
+            "category": item.get("task_group", "oolong"),
+        })
+
+    json.dump(output, sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

python/pg_batteries.py

@@ -0,0 +1,158 @@
+"""
+pg_batteries.py — PostgreSQL storage query functions for rlmx.
+
+Provides pg_search(), pg_slice(), pg_time(), pg_count(), pg_query()
+that communicate with the Node.js PgStorage via the IPC bridge.
+
+Available when storage mode is active.
+"""
+
+import json
+
+# IPC bridge — resolved at call time from REPL namespace globals
+import llm_bridge
+
+
+def _pg_request(request_type, params=None):
+    """Send a pg_* request to Node.js PgStorage and return parsed result."""
+    payload = json.dumps(params) if params else "{}"
+    results = llm_bridge.send_request(request_type, [payload])
+    if not results:
+        return None
+    raw = results[0]
+    if raw.startswith("Error:"):
+        raise RuntimeError(raw)
+    try:
+        return json.loads(raw)
+    except (json.JSONDecodeError, TypeError):
+        return raw
+
+
+def _truncate_output(items, label="results"):
+    """Truncate output if > 2000 chars, showing first 5 items as stub."""
+    if not isinstance(items, list):
+        text = str(items)
+        if len(text) <= 2000:
+            return text
+        return text[:2000] + "\n... [truncated]"
+
+    full = json.dumps(items, indent=2)
+    if len(full) <= 2000:
+        return full
+
+    n = len(items)
+    preview_items = items[:5]
+    lines = [f"[{n} {label}, showing first {min(5, n)}]"]
+    for item in preview_items:
+        if isinstance(item, dict):
+            content = item.get("content", "")
+            preview = content[:100] + "..." if len(content) > 100 else content
+            shown = {k: (preview if k == "content" else v) for k, v in item.items()}
+            lines.append(json.dumps(shown))
+        else:
+            lines.append(str(item)[:100])
+    lines.append("...")
+    if items and isinstance(items[0], dict) and "line_num" in items[0]:
+        first_ln = items[0]["line_num"]
+        lines.append(f'Use pg_slice({first_ln}, {first_ln + 10}) to see full content')
+    return "\n".join(lines)
+
+
+def pg_search(pattern, limit=20):
+    """Full-text search in stored context. Returns matching records ranked by relevance.
+
+    Args:
+        pattern: Search terms (words joined with AND)
+        limit: Max results (default 20)
+
+    Returns:
+        List of {line_num, source, content, rank} dicts
+    """
+    result = _pg_request("pg_search", {"pattern": pattern, "limit": limit})
+    if isinstance(result, list):
+        return _truncate_output(result, "results")
+    return result
+
+
+def pg_slice(start, end):
+    """Get context lines by range. Returns content for lines [start, end).
+
+    Args:
+        start: Starting line number (inclusive)
+        end: Ending line number (exclusive)
+
+    Returns:
+        String with source and content of the requested lines
+    """
+    result = _pg_request("pg_slice", {"start": start, "end": end})
+    if isinstance(result, list):
+        lines = []
+        for r in result:
+            if isinstance(r, dict):
+                src = r.get("source", "")
+                content = r.get("content", "")
+                lines.append(f"[{src}] {content}" if src else content)
+            else:
+                lines.append(str(r))
+        content = "\n".join(lines)
+        if len(content) > 2000:
+            return content[:2000] + f"\n... [truncated, {len(result)} lines total]"
+        return content
+    return result
+
+
+def pg_sources():
+    """List distinct source files in the stored context.
+
+    Returns:
+        List of source file paths
+    """
+    result = _pg_request("pg_query", {
+        "sql": "SELECT DISTINCT source FROM records WHERE source IS NOT NULL ORDER BY source"
+    })
+    if isinstance(result, list):
+        return [r.get("source", "") if isinstance(r, dict) else str(r) for r in result]
+    return result
+
+
+def pg_time(from_time, to_time):
+    """Filter context records by timestamp range.
+
+    Args:
+        from_time: Start time (e.g. '01:00' or '2024-01-01T01:00:00')
+        to_time: End time
+
+    Returns:
+        List of {line_num, timestamp, content} dicts
+    """
+    result = _pg_request("pg_time", {"from": from_time, "to": to_time})
+    if isinstance(result, list):
+        return _truncate_output(result, "results")
+    return result
+
+
+def pg_count():
+    """Count total records in stored context.
+
+    Returns:
+        Integer count
+    """
+    result = _pg_request("pg_count", {})
+    if isinstance(result, dict):
+        return result.get("count", 0)
+    return result
+
+
+def pg_query(sql):
+    """Execute raw SQL query (read-only) against the context database.
+
+    Args:
+        sql: SQL query string
+
+    Returns:
+        List of result row dicts
+    """
+    result = _pg_request("pg_query", {"sql": sql})
+    if isinstance(result, list):
+        return _truncate_output(result, "rows")
+    return result

src/batch.ts

@@ -11,6 +11,7 @@ import { rlmLoop, type RLMOptions } from "./rlm.js";
 import type { RlmxConfig } from "./config.js";
 import type { LoadedContext } from "./context.js";
 import { isGoogleProvider } from "./gemini.js";
+import { validateContextSize } from "./cache.js";
 
 interface BatchResult {
   question: string;
@@ -36,6 +37,8 @@ export interface BatchOptions extends Partial<RLMOptions> {
   parallel?: number;
   /** Use Gemini Batch API for 50% cost reduction. Requires provider: google. */
   batchApi?: boolean;
+  /** When true, use pgserve storage for large context handling. */
+  storageMode?: boolean;
 }
 
 /**
@@ -96,13 +99,18 @@ export async function runBatch(
       break;
     }
 
-    // Run each question through rlmLoop with cache enabled
+    // Determine cache/storage mode for this batch
+    const useCache = options.cache ?? true;
+    const useStorage = options.storageMode ?? false;
+
+    // Run each question through rlmLoop
     const result = await rlmLoop(question, context, config, {
       maxIterations: options.maxIterations,
       timeout: options.timeout,
       verbose: options.verbose,
       output: "text", // batch always captures text internally
-      cache: true, // batch always uses cache
+      cache: useCache,
+      storageMode: useStorage,
     });
 
     totalCost += result.usage.totalCost;