feat: optimize web UI performance for large chat histories

agent.py

@@ -1,4 +1,4 @@
-import asyncio, random, string, threading
+import asyncio, random, string, threading, time
 import nest_asyncio
 
 nest_asyncio.apply()
@@ -86,16 +86,39 @@ def __init__(
         self.log = log or Log.Log()
         self.log.context = self
         self.paused = paused
-        self.streaming_agent = streaming_agent
+        self._streaming_agent = streaming_agent
         self.task: DeferredTask | None = None
         self.created_at = created_at or datetime.now(timezone.utc)
         self.type = type
         AgentContext._counter += 1
         self.no = AgentContext._counter
         self.last_message = last_message or datetime.now(timezone.utc)
 
+        # Lazy deserialization support: these are set by persist_chat._deserialize_context
+        # when loading from disk to defer the expensive agent/history deserialization.
+        self._raw_agents: list | None = None
+        self._raw_streaming_agent_no: int = 0
+
         # initialize agent at last (context is complete now)
-        self.agent0 = agent0 or Agent(0, self.config, self)
+        self._agent0 = agent0 or Agent(0, self.config, self)
+
+    @property
+    def agent0(self):
+        self._ensure_hydrated()
+        return self._agent0
+
+    @agent0.setter
+    def agent0(self, value):
+        self._agent0 = value
+
+    @property
+    def streaming_agent(self):
+        self._ensure_hydrated()
+        return self._streaming_agent
+
+    @streaming_agent.setter
+    def streaming_agent(self, value):
+        self._streaming_agent = value
 
     @staticmethod
     def get(id: str):
@@ -237,6 +260,12 @@ def nudge(self):
         self.task = self.communicate(UserMessage(self.agent0.read_prompt("fw.msg_nudge.md")))
         return self.task
 
+    def _ensure_hydrated(self):
+        """Ensure agents and history are fully deserialized (lazy hydration)."""
+        if self._raw_agents is not None:
+            from python.helpers.persist_chat import hydrate_context_agents
+            hydrate_context_agents(self)
+
     def get_agent(self):
         return self.streaming_agent or self.agent0
 
@@ -293,6 +322,12 @@ async def _process_chain(self, agent: "Agent", msg: "UserMessage|str", user=True
             return response
         except Exception as e:
             agent.handle_critical_exception(e)
+        finally:
+            if user:
+                from python.helpers.state_snapshot import touch_chat_list
+                from python.helpers.state_monitor_integration import mark_dirty_all
+                touch_chat_list()
+                mark_dirty_all(reason="process_chain_end")
 
 
 @dataclass
@@ -663,10 +698,20 @@ def get_data(self, field: str):
     def set_data(self, field: str, value):
         self.data[field] = value
 
+    _last_msg_touch: float = 0.0
+    _MSG_TOUCH_INTERVAL: float = 5.0
+
     def hist_add_message(
         self, ai: bool, content: history.MessageContent, tokens: int = 0
     ):
-        self.last_message = datetime.now(timezone.utc)
+        now = datetime.now(timezone.utc)
+        self.last_message = now
+        self.context.last_message = now
+        now_mono = time.time()
+        if now_mono - Agent._last_msg_touch >= Agent._MSG_TOUCH_INTERVAL:
+            Agent._last_msg_touch = now_mono
+            from python.helpers.state_snapshot import touch_chat_list
+            touch_chat_list()
         # Allow extensions to process content before adding to history
         content_data = {"content": content}
         asyncio.run(

docs/superpowers/specs/2026-03-26-webui-performance-optimization-design.md

@@ -0,0 +1,159 @@
+# Web UI Performance Optimization
+
+## Problem
+
+The Agent Zero web interface becomes slow and memory-heavy with many chats or long chat histories. Root causes:
+
+1. Every `state_push` includes the full `contexts[]` and `tasks[]` arrays — even when the chat list hasn't changed.
+2. Switching chats triggers `forceFull` (`log_from: 0`), sending all log items in a single JSON payload and rendering them synchronously.
+3. `load_tmp_chats()` at startup fully deserializes every `chat.json` including heavy `History` objects.
+4. `Log._notify_state_monitor()` calls `mark_dirty_all` on every new log item, pushing to all connected clients regardless of which chat they're viewing.
+5. Frontend renders the entire log array synchronously on mass render, causing UI freezes.
+
+## Constraints
+
+- 10–30 chats, 100+ messages per chat.
+- All three pain points matter: initial load, chat switching, long-session degradation.
+- Only the web UI consumes the API — no backward-compatibility requirement.
+
+## Design
+
+Five coordinated changes across backend and frontend.
+
+### 1. Conditional chat list in snapshots
+
+**Goal:** Stop sending `contexts[]`/`tasks[]` when the list hasn't changed.
+
+Backend holds a module-level `_chat_list_updated_at: float` timestamp, updated by operations that change the chat list: create, remove, rename, running-status change.
+
+`StateRequestV1` gains a field `chat_list_since: float`. The client sends the timestamp of its last received chat list update. On first load, hard refresh, or new tab, the client sends `chat_list_since: 0`, which always triggers a full list. `build_snapshot_from_request` compares: if `chat_list_since >= _chat_list_updated_at`, it sets `contexts` and `tasks` to `None` in the snapshot and skips the `AgentContext.all()` iteration + sort entirely.
+
+`SnapshotV1` gains `chat_list_updated_at: float`. The client stores this and passes it back in subsequent requests.
+
+Frontend `applySnapshot` checks: if `contexts` is `null`, leave the sidebar list unchanged. Otherwise apply as today.
+
+Both `contexts` and `tasks` follow the same conditional logic — they are either both included or both `null`. The `has_earlier_logs` field in the snapshot and the `has_more` field in the `/chat_logs` response represent the same concept (whether older log items exist) for their respective contexts.
+
+**Files:** `state_snapshot.py`, `index.js`, `sync-store.js`, `chats-store.js`, `tasks-store.js`.
+
+### 2. Log pagination on chat load
+
+**Goal:** Load only the tail of the conversation when switching chats; fetch older messages on demand.
+
+#### Backend
+
+`Log.output()` gains an optional `tail: int | None` parameter. When `tail` is set and `start == 0` (full load), the method collects all unique log item numbers from `updates`, takes only the last `tail` items, and returns their outputs. When `start != 0` (incremental push), `tail` is ignored. The snapshot includes a new field `has_earlier_logs: bool` indicating whether there are log items before the returned window.
+
+Default tail size: `INITIAL_LOG_TAIL = 50`.
+
+Incremental pushes (`log_from > 0`) are unaffected — they continue returning all updates since the cursor, which is already incremental and small during streaming.
+
+#### New endpoint: `POST /chat_logs`
+
+Request body:
+
+```json
+{
+  "context_id": "abc123",
+  "before": 42,
+  "limit": 50
+}
+```
+
+- `before`: the `no` of the oldest currently loaded log item.
+- `limit`: max items to return (default 50).
+
+Response:
+
+```json
+{
+  "logs": [ ... ],
+  "has_more": true
+}
+```
+
+Returns log items with `no < before`, sorted oldest-to-newest. `has_more` indicates whether there are still earlier items.
+
+#### Frontend
+
+On chat switch, `setMessages` receives at most 50 items. If `has_earlier_logs` is true, a "Load earlier messages" indicator appears at the top of `#chat-history`. Clicking it (or scrolling to top) triggers `POST /chat_logs`. Returned items are prepended to the DOM with scroll-position preservation (`scrollHeight` delta applied to `scrollTop`).
+
+**Files:** `log.py`, `state_snapshot.py`, new `python/api/chat_logs.py`, `index.js`, `messages.js`.
+
+### 3. Lazy deserialization at startup
+
+**Goal:** Reduce server startup time and initial RAM by deferring heavy `History` deserialization.
+
+`load_tmp_chats()` currently calls `_deserialize_context()` for each chat, which includes `_deserialize_agents()` → `History.deserialize_history()`. This is the most expensive part per chat.
+
+Split into two phases:
+
+1. **Phase 1 (startup):** Read `chat.json`, parse top-level JSON, deserialize metadata fields (`id`, `name`, `type`, `created_at`, `last_message`) and `log`. Store the `agents` JSON fragment as a raw string in `AgentContext._raw_agents: str | None`. Skip `_deserialize_agents()`.
+
+2. **Phase 2 (on demand):** When a chat is actually accessed (user opens it, agent starts working), call `AgentContext._ensure_hydrated()`. This deserializes `_raw_agents` into full Agent/History objects and sets `_raw_agents = None` to free the raw string.
+
+`AgentContext.output()` (used for the sidebar list) works without hydration — it only needs metadata and log fields available after phase 1.
+
+Any code path that accesses `agent0` or the agent chain must call `_ensure_hydrated()` first. Key entry points: `use_context()` (before message processing), `chat_export`, `persist_chat.save_tmp_chat`.
+
+**Files:** `persist_chat.py`, `agent.py`.
+
+### 4. Scoped dirty signals
+
+**Goal:** Stop broadcasting push updates to clients not viewing the active chat.
+
+Currently `Log._notify_state_monitor()` calls `mark_dirty_all` when a new log item is created. This was necessary because `contexts[]` (with updated `log_version`) was included in every push — other tabs needed it to update their sidebar.
+
+After change 1 (conditional chat list), sidebar metadata is decoupled from log pushes. Therefore:
+
+- `Log._notify_state_monitor()` switches from `mark_dirty_all` to `mark_dirty_for_context`. Only clients projecting this context receive a push.
+- `Log._notify_state_monitor_for_context_update()` (existing item updates during streaming) remains unchanged — already uses `mark_dirty_for_context`.
+- Operations that actually change the chat list (create, remove, rename, running-status toggle) call `mark_dirty_all` and update `_chat_list_updated_at`. These are infrequent events.
+
+**Files:** `log.py`, API handlers that mutate the chat list (`chat_create.py`, `chat_remove.py`, `chat_reset.py`, and others that trigger `mark_dirty_all` today).
+
+### 5. Batched DOM rendering
+
+**Goal:** Eliminate UI freezes during mass render of large chat histories.
+
+When `_massRender` is true (chat switch, initial load), instead of synchronously iterating all messages in `setMessages`, process them in batches of 20 via `requestAnimationFrame`. The first batch renders immediately so the user sees content instantly; subsequent batches render across frames without blocking input.
+
+The prepend path for `/chat_logs` responses inserts DOM nodes before existing content and preserves scroll position by measuring `scrollHeight` before and after insertion and adjusting `scrollTop`.
+
+A loading indicator at the top of `#chat-history` appears when `has_earlier_logs` is true and disappears when all earlier logs have been loaded or the request completes.
+
+**Files:** `messages.js`, `index.js`, minimal CSS for the loading indicator.
+
+## Files changed (summary)
+
+| File | Changes |
+|------|---------|
+| `python/helpers/state_snapshot.py` | `chat_list_since` in request, conditional `contexts`/`tasks`, `chat_list_updated_at` + `has_earlier_logs` in snapshot, `tail` parameter passthrough |
+| `python/helpers/log.py` | `tail` param in `output()`, switch `_notify_state_monitor` to `mark_dirty_for_context` |
+| `python/helpers/state_monitor.py` | No structural changes; existing `mark_dirty_for_context` and `mark_dirty_all` used as-is |
+| `python/helpers/persist_chat.py` | Two-phase deserialization: metadata-only in `load_tmp_chats`, lazy `_deserialize_agents` |
+| `agent.py` | `_raw_agents` field, `_ensure_hydrated()` method; `output()` works without hydration, guards on paths accessing `agent0`/chain |
+| `python/api/chat_logs.py` | New endpoint for paginated log history |
+| `python/api/chat_create.py`, `chat_remove.py`, `chat_reset.py` | Update `_chat_list_updated_at` on list mutations |
+| `run_ui.py` | No edit needed — new `chat_logs.py` is auto-discovered by `load_classes_from_folder` in `run_ui.py` |
+| `webui/index.js` | Track `chat_list_updated_at`, pass `chat_list_since`, handle `has_earlier_logs`, call `/chat_logs` |
+| `webui/js/messages.js` | Batched `requestAnimationFrame` rendering, prepend logic with scroll preservation |
+| `webui/components/sync/sync-store.js` | Pass `chat_list_since` in state request payload |
+| `webui/components/sidebar/chats/chats-store.js` | Handle `null` contexts gracefully |
+| `webui/components/sidebar/tasks/tasks-store.js` | Handle `null` tasks gracefully |
+
+## Testing
+
+- **Unit tests** for `Log.output(tail=N)`: verify correct tail slicing, `has_earlier_logs` flag, edge cases (empty log, tail > total items).
+- **Unit tests** for `build_snapshot_from_request`: verify `contexts` is `None` when `chat_list_since` is current, full list when stale.
+- **Unit tests** for lazy deserialization: verify `output()` works without hydration, `_ensure_hydrated()` produces valid agents.
+- **Unit tests** for `/chat_logs` endpoint: pagination correctness, boundary conditions.
+- **Integration test**: start server with multiple chats, verify startup time improvement (no full History deserialization).
+- **Manual testing**: verify chat switching speed, scroll-up loading, no regressions in streaming, sidebar updates on chat create/remove.
+
+## Out of scope
+
+- Virtual scrolling / DOM virtualization (disproportionate complexity for 10–30 chats).
+- SQLite index for chat metadata (unnecessary at current scale).
+- LRU eviction of inactive `AgentContext` from memory (can be added later if needed).
+- Chat list pagination in the sidebar (30 items is fine).

docs/superpowers/specs/2026-03-27-sidebar-chat-list-ux-design.md

@@ -0,0 +1,97 @@
+# Sidebar Chat List UX Improvements
+
+## Problem
+
+Three usability issues in the sidebar chat list:
+
+1. **Unread indicator overlaps project color** — the blue "unread" dot overrides the project color ball, losing project identity.
+2. **No timestamp** — no way to see when a chat was last active without opening it.
+3. **Static sort order** — chats are sorted by creation time. Active chats stay buried, making it hard to find recently active conversations.
+
+## Design
+
+### Chat item layout
+
+```
+[project-ball] [chat-name] [unread-dot?] [timestamp] [close-btn]
+```
+
+- `project-ball` — always shows project color (never overridden by unread state)
+- `chat-name` — truncated with ellipsis, `flex: 1`
+- `unread-dot` — 6px blue circle, shown only when `context.unread === true` and chat is not selected
+- `timestamp` — relative time since `last_message`, right-aligned
+- `close-btn` — existing delete button, visible on hover
+
+### 1. Unread indicator
+
+**Current**: `.chat-unread .project-color-ball` CSS rule overrides background to blue.
+
+**Change**: Remove the `.chat-unread .project-color-ball` CSS rules. Add a separate `<span class="unread-dot">` element after the chat name, controlled by `x-show="context.unread && context.id !== $store.chats.selected"`.
+
+Styling:
+- `width: 6px; height: 6px; border-radius: 50%`
+- `background-color: #4fc3f7` (dark mode), `#0288d1` (light mode)
+- `flex-shrink: 0`
+
+Keep `.chat-unread .chat-name { font-weight: 600 }` — bold name is a good secondary signal.
+
+### 2. Relative timestamp
+
+Add `<span class="chat-timestamp">` after the unread dot. Shows time since `last_message` in compact format.
+
+Format rules:
+| Elapsed | Display |
+|---|---|
+| < 60s | `Ns` (e.g. `5s`) |
+| < 60m | `Nm` (e.g. `10m`) |
+| < 24h | `Nh` (e.g. `5h`) |
+| < 7d | `Nd` (e.g. `3d`) |
+| >= 7d | `Nw` (e.g. `2w`) |
+
+`title` attribute shows full localized datetime on hover (using `Date.toLocaleString()`).
+
+The helper function `formatRelativeTime(isoString)` lives in `chats-store.js` (or inline in the template via Alpine `x-text`). It computes relative time from `context.last_message`.
+
+Timestamps update on every `applyContexts` call (snapshot push, every ~2-5s during active sessions). No separate `setInterval` timer needed — staleness of a few seconds is acceptable.
+
+Styling:
+- `font-size: var(--font-size-xs, 0.7em)`
+- `color: var(--color-secondary); opacity: 0.6`
+- `flex-shrink: 0; margin-left: auto; padding: 0 4px`
+- `white-space: nowrap`
+
+### 3. Sort by last update
+
+**Current** (line 47 of `chats-store.js`):
+```javascript
+this.contexts = contextsList.sort(
+  (a, b) => (b.created_at || 0) - (a.created_at || 0)
+);
+```
+
+**Change**: Sort by `last_message` descending. The `last_message` field is an ISO datetime string from `AgentContext.output()`. Convert to comparable values:
+```javascript
+this.contexts = contextsList.sort((a, b) => {
+  const ta = a.last_message ? new Date(a.last_message).getTime() : 0;
+  const tb = b.last_message ? new Date(b.last_message).getTime() : 0;
+  return tb - ta;
+});
+```
+
+Chats with more recent activity float to the top.
+
+## Files changed
+
+| File | Change |
+|---|---|
+| `webui/components/sidebar/chats/chats-list.html` | Add unread-dot span, timestamp span; remove `.chat-unread .project-color-ball` CSS; add new CSS for `.unread-dot` and `.chat-timestamp` |
+| `webui/components/sidebar/chats/chats-store.js` | Change sort key from `created_at` to `last_message`; add `formatRelativeTime()` helper |
+
+## Testing
+
+Manual verification:
+1. Create a new chat, send a message — verify timestamp shows `1s`, `2s`, etc.
+2. Switch away and back — verify unread dot appears to the right of the name, project color ball keeps its color.
+3. Send messages in different chats — verify most recently active chat moves to the top.
+4. Hover over timestamp — verify full datetime tooltip.
+5. Light mode — verify unread dot uses `#0288d1`.

python/api/chat_create.py

@@ -27,7 +27,9 @@ async def process(self, input: Input, request: Request) -> Output:
             #     new_context.set_output_data(projects.CONTEXT_DATA_KEY_PROJECT, current_data_2)
 
         # New context should appear in other tabs' chat lists via state_push.
+        from python.helpers.state_snapshot import touch_chat_list
         from python.helpers.state_monitor_integration import mark_dirty_all
+        touch_chat_list()
         mark_dirty_all(reason="api.chat_create.CreateChat")
 
         return {

python/api/chat_logs.py

@@ -0,0 +1,18 @@
+from python.helpers.api import ApiHandler, Input, Output, Request, Response
+from agent import AgentContext
+
+
+class ChatLogs(ApiHandler):
+    async def process(self, input: Input, request: Request) -> Output:
+        context_id = input.get("context_id", "")
+        if not context_id:
+            raise Exception("No context_id provided")
+
+        before = int(input.get("before", 0))
+        limit = int(input.get("limit", 50))
+
+        context = AgentContext.get(context_id)
+        if not context:
+            return {"logs": [], "has_more": False}
+
+        return context.log.get_items_before(before, limit)

python/api/chat_remove.py

@@ -26,7 +26,9 @@ async def process(self, input: Input, request: Request) -> Output:
             await scheduler.remove_task_by_uuid(task.uuid)
 
         # Context removal affects global chat/task lists in all tabs.
+        from python.helpers.state_snapshot import touch_chat_list
         from python.helpers.state_monitor_integration import mark_dirty_all
+        touch_chat_list()
         mark_dirty_all(reason="api.chat_remove.RemoveChat")
 
         return {