Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion echo/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,6 @@ __queuestorage__echo/server/dembrane/workspace_script.py
tools/translator
echo-gitops/

cypress/reports/
cypress/reports/

cookies.txt
40 changes: 20 additions & 20 deletions echo/.vscode/sessions.json
Original file line number Diff line number Diff line change
@@ -1,68 +1,68 @@
{
"$schema": "https://cdn.statically.io/gh/nguyenngoclongdev/cdn/main/schema/v11/terminal-keeper.json",
"theme": "tribe",
"active": "default",
"keepExistingTerminals": false,
"sessions": {
"default": [
{
"autoExecuteCommands": true,
"name": "server",
"icon": "server",
"commands": [
"cd server",
"./run.sh"
]
],
"icon": "server",
"name": "server"
},
[
{
"autoExecuteCommands": true,
"name": "workers",
"icon": "gear",
"commands": [
"cd server",
"./run-worker.sh"
]
],
"icon": "gear",
"name": "workers"
},
{
"autoExecuteCommands": true,
"name": "workers-cpu",
"icon": "gear",
"commands": [
"cd server",
"./run-worker-cpu.sh"
]
],
"icon": "gear",
"name": "workers-cpu"
},
{
"autoExecuteCommands": true,
"name": "scheduler",
"icon": "clock",
"commands": [
"cd server",
"./run-scheduler.sh"
]
],
"icon": "clock",
"name": "scheduler"
}
],
[
{
"autoExecuteCommands": true,
"name": "admin-dashboard",
"icon": "browser",
"commands": [
"cd frontend",
"pnpm run dev"
]
],
"icon": "browser",
"name": "admin-dashboard"
},
{
"autoExecuteCommands": true,
"name": "participant-portal",
"icon": "browser",
"commands": [
"cd frontend",
"pnpm run participant:dev"
]
],
"icon": "browser",
"name": "participant-portal"
}
]
]
}
},
"theme": "tribe"
}
79 changes: 40 additions & 39 deletions echo/.vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,41 +1,42 @@
{
"files.eol": "\n",
"debug.internalConsoleOptions": "neverOpen",
// py
"cursorpyright.analysis.typeCheckingMode": "off",
"python.languageServer": "None",
"python.defaultInterpreterPath": "./server/.venv/bin/python",
"ruff.lint.enable": true,
"ruff.configuration": "./server/pyproject.toml",
"mypy.runUsingActiveInterpreter": true,
"mypy.targets": ["./server/dembrane"],
"mypy.configFile": "./server/pyproject.toml",
"[python]": {
"editor.defaultFormatter": "charliermarsh.ruff",
"editor.useTabStops": true,
"editor.tabSize": 4,
"editor.formatOnSave": true
},
"python.testing.pytestArgs": ["server"],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true,
"python.testing.autoTestDiscoverOnSaveEnabled": true,
// ts
"[typescript]": {
"editor.defaultFormatter": "biomejs.biome",
"editor.formatOnSave": true,
"editor.useTabStops": true
},
"[typescriptreact]": {
"editor.defaultFormatter": "biomejs.biome",
"editor.useTabStops": true,
"editor.formatOnSave": true
},
"biome.enabled": true,
"biome.lsp.bin": "frontend/node_modules/.bin/biome",
"biome.configurationPath": "frontend/biome.json",
"editor.codeActionsOnSave": {
"source.fixAll.biome": "always",
"source.action.organizeImports.biome": "always"
}
"[python]": {
"editor.defaultFormatter": "charliermarsh.ruff",
"editor.formatOnSave": true,
"editor.tabSize": 4,
"editor.useTabStops": true
},
// ts
"[typescript]": {
"editor.defaultFormatter": "biomejs.biome",
"editor.formatOnSave": true,
"editor.useTabStops": true
},
"[typescriptreact]": {
"editor.defaultFormatter": "biomejs.biome",
"editor.formatOnSave": true,
"editor.useTabStops": true
},
"biome.configurationPath": "frontend/biome.json",
"biome.enabled": true,
"biome.lsp.bin": "frontend/node_modules/.bin/biome",
// py
"cursorpyright.analysis.typeCheckingMode": "off",
"debug.internalConsoleOptions": "neverOpen",
"editor.codeActionsOnSave": {
"source.action.organizeImports.biome": "always",
"source.fixAll.biome": "always"
},
"files.eol": "\n",
"mypy.configFile": "./server/pyproject.toml",
"mypy.runUsingActiveInterpreter": true,
"mypy.targets": ["./server/dembrane"],
"python.defaultInterpreterPath": "./server/.venv/bin/python",
"python.languageServer": "None",
"python.testing.autoTestDiscoverOnSaveEnabled": true,
"python.testing.pytestArgs": ["server"],
"python.testing.pytestEnabled": true,
"python.testing.unittestEnabled": false,
"ruff.configuration": "./server/pyproject.toml",
"ruff.lint.enable": true,
"terminal.integrated.enableVisualBell": true
}
93 changes: 76 additions & 17 deletions echo/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,35 +158,94 @@ bash echo/server/scripts/agentic/latest_runs.sh --chat-id <chat_uuid> --limit 1
| `frontend/src/config.ts` | Frontend feature flags |
| `server/dembrane/settings.py` | Backend configuration |
| `docs/frontend_translations.md` | Translation workflow |
| `docs/branching_and_releases.md` | Branching strategy, release process, hotfixes |
| `docs/database_migrations.md` | Directus data migration steps |

## Code Style

- Frontend: TypeScript, React, Mantine UI
- Backend: Python 3.11+, FastAPI, Pydantic
- Use existing patterns in the codebase as reference

## Dev Notes
## Branching Strategy & Deployment

### Recent Changes (testing branch)
- Copy guide enforcement: "context limit" → "selection too large"
- Translations updated for all 6 languages
- Suggestions use faster model (`TEXT_FAST` instead of `MULTI_MODAL_PRO`)
- Stream status shows inline under "Thinking..." instead of toast
- Webhooks (conversation-level notifications)
See [docs/branching_and_releases.md](docs/branching_and_releases.md) for the full guide including hotfix process, release checklist, and ASCII diagrams.

### Tech Debt / Known Issues
Quick reference:
- **Feature flow**: branch off `main` → (optional) merge to `testing` → PR to `main` → auto-deploys to Echo Next
- **Releases**: tagged from `main` every ~2 weeks → auto-deploys to production
- **Hotfixes**: branch off release tag → fix → new release → cherry-pick into main
- **Project management**: Linear (`ECHO-xxx` tickets, two-week cycles)
- **GitOps**: `dembrane/echo-gitops` (Terraform + Helm + Argo CD)

## Architecture Notes

### High-Level Stack

```
Frontend (React/Vite/Mantine) → Backend API (FastAPI) → Directus (headless CMS/DB)
↕ ↕
Dramatiq Workers PostgreSQL
(gevent + standard)
Redis (pub/sub, task broker, caching)
Agent Service (LangGraph, port 8001)
```
Comment on lines +185 to +194
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Add a language identifier to the fenced block.

The architecture ASCII block is missing a fence language and triggers markdown linting.

✅ Suggested markdown fix
-```
+```text
 Frontend (React/Vite/Mantine)  →  Backend API (FastAPI)  →  Directus (headless CMS/DB)
                                        ↕                          ↕
                                Dramatiq Workers           PostgreSQL
                                (gevent + standard)
                                        ↕
                                     Redis (pub/sub, task broker, caching)
                                        ↕
                                Agent Service (LangGraph, port 8001)
</details>

<details>
<summary>🧰 Tools</summary>

<details>
<summary>🪛 markdownlint-cli2 (0.22.0)</summary>

[warning] 185-185: Fenced code blocks should have a language specified

(MD040, fenced-code-language)

</details>

</details>

<details>
<summary>🤖 Prompt for AI Agents</summary>

Verify each finding against the current code and only fix it if needed.

In @echo/AGENTS.md around lines 185 - 194, The fenced ASCII architecture block
that begins with "Frontend (React/Vite/Mantine) → Backend API (FastAPI) →
Directus (headless CMS/DB)" is missing a fence language which triggers markdown
linting; add a language identifier (e.g., "text") immediately after the opening
triple backticks so the block starts with "```text" and leave the closing triple
backticks unchanged to satisfy the linter and preserve formatting.


</details>

<!-- fingerprinting:phantom:poseidon:hawk:ecb4e5f5-abbe-41ba-9503-68dfaf572288 -->

<!-- This is an auto-generated comment by CodeRabbit -->


- **Directus** is the data layer — all collections (projects, conversations, reports, etc.) live there
- **FastAPI** handles API routes, SSE streaming, and orchestration
- **Dramatiq** handles background work: transcription, summarization, report generation
- **Redis** is used for task brokering, pub/sub (SSE progress), and caching
- **LiteLLM** routes all LLM calls with automatic failover between deployments

### Report Generation Pipeline

Report generation runs **synchronously** in Dramatiq network-queue workers (no asyncio — this was a deliberate choice after recurring event-loop corruption bugs):

1. Fetch conversations for the project
2. Fan-out summarization of individual conversations via `dramatiq.group()`
3. Poll Redis for group completion
4. Refetch conversations with summaries
5. Fetch full transcripts via `gevent.pool.Pool` (concurrent I/O)
6. Build prompt with token budget management
7. Call LLM via `router_completion()` (sync litellm, uses `MULTI_MODAL_PRO`)

Key files:
- `server/dembrane/report_generation.py` — main pipeline
- `server/dembrane/report_events.py` — Redis pub/sub for real-time SSE progress
- `server/prompt_templates/system_report.{lang}.jinja` — per-language prompt templates (written IN the target language)

### BFF Pattern

Backend For Frontend endpoints under `/bff/` aggregate data the frontend needs in a single call. This is the preferred pattern over having the frontend make multiple Directus SDK calls directly.

Example: `/bff/projects/home` bundles pinned projects, paginated project list, search results, and admin info into one response.

### Transcription Pipeline

Two-step process:
1. **AssemblyAI** (`universal-3-pro`) for raw speech-to-text — supports en, es, pt, fr, de, it. Dutch ("nl") requires `universal-2` fallback.
2. **Gemini correction** — fixes transcripts, normalizes hotwords, PII redaction, adds recording feedback

Production uses webhook mode (`ASSEMBLYAI_WEBHOOK_URL`); polling is only a fallback path.

### Agent Service

The `agent/` directory contains the agentic chat service (LangGraph-based). It runs as a separate FastAPI service on port 8001. Agentic chat streams via `POST /api/agentic/runs/{run_id}/stream` — no Dramatiq dispatch. See `agent/README.md`.

## Tech Debt / Known Issues
- Some mypy errors in `llm_router.py` and `settings.py` (pre-existing, non-blocking)
Comment on lines +237 to 238
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Add a blank line after the heading for markdownlint compliance.

## Tech Debt / Known Issues should be followed by a blank line before the list.

✅ Suggested markdown fix
 ## Tech Debt / Known Issues
+
 - Some mypy errors in `llm_router.py` and `settings.py` (pre-existing, non-blocking)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
## Tech Debt / Known Issues
- Some mypy errors in `llm_router.py` and `settings.py` (pre-existing, non-blocking)
## Tech Debt / Known Issues
- Some mypy errors in `llm_router.py` and `settings.py` (pre-existing, non-blocking)
🧰 Tools
🪛 markdownlint-cli2 (0.22.0)

[warning] 237-237: Headings should be surrounded by blank lines
Expected: 1; Actual: 0; Below

(MD022, blanks-around-headings)

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@echo/AGENTS.md` around lines 237 - 238, Add a blank line immediately after
the markdown heading "## Tech Debt / Known Issues" so the subsequent list "-
Some mypy errors in `llm_router.py` and `settings.py`..." is separated by an
empty line; update the AGENTS.md section containing that heading (look for the
exact heading text) to satisfy markdownlint by inserting one newline between the
heading and the list.


## Deployment Process
## Deployment Checklist

### Merging to Main (for echo-next environment)
### Before Merging to Main

1. **Compare branches**: `git log main..testing --oneline`
2. **Check for new env vars**: Look for new `Field()` definitions in `settings.py` and new exports in `config.ts`
3. **Update deployment env vars** if needed (see checklist below)
4. **Push Directus schema** if there were database changes
5. **Create PR**: `testing` → `main`
6. **Deploy** after merge
1. **Check for new env vars**: Look for new `Field()` definitions in `settings.py` and new exports in `config.ts`
2. **Update deployment env vars** if needed (see checklist below)
3. **Push Directus schema** if there were database changes (see `docs/database_migrations.md`)
4. **Create PR** from feature branch to `main`
5. After merge → auto-deploys to Echo Next

### Environment Variables Checklist

Expand Down Expand Up @@ -225,4 +284,4 @@ LLM__MULTI_MODAL_FAST_2__GCP_SA_JSON=${GCP_SA_JSON}
LLM__MULTI_MODAL_FAST_2__VERTEX_LOCATION=europe-west1
```

Model groups: `TEXT_FAST`, `MULTI_MODAL_PRO`, `MULTI_MODAL_FAST`
Model groups: `MULTI_MODAL_PRO` (Gemini 2.5 Pro — chat, reports, transcript correction), `MULTI_MODAL_FAST` (Gemini 2.5 Flash — suggestions, verification, lightweight tasks), `TEXT_FAST` (Azure GPT-4.1 — being deprecated)
Binary file not shown.
1 change: 0 additions & 1 deletion echo/cypress/cypress/downloads/transcript-1771916490192

This file was deleted.

Binary file not shown.
Binary file not shown.
15 changes: 12 additions & 3 deletions echo/cypress/support/functions/report/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,26 +43,35 @@ export const generateReport = (langCode = 'en') => {
// ============= Report Actions =============

/**
* Clicks the share button (mobile)
* Opens the report actions menu (three-dot menu)
*/
const openReportActionsMenu = () => {
cy.get('[data-testid="report-actions-menu"]').should('be.visible').click();
};

/**
* Clicks the share button (in actions menu dropdown)
*/
export const shareReport = () => {
cy.log('Sharing Report');
openReportActionsMenu();
cy.get('[data-testid="report-share-button"]').should('be.visible').click();
};

/**
* Copies the report link
* Copies the report link (inline icon button, only when published)
*/
export const copyReportLink = () => {
cy.log('Copying Report Link');
cy.get('[data-testid="report-copy-link-button"]').should('be.visible').click();
};

/**
* Prints the report
* Prints the report (in actions menu dropdown, only when published)
*/
export const printReport = () => {
cy.log('Printing Report');
openReportActionsMenu();
cy.get('[data-testid="report-print-button"]').should('be.visible').click();
};

Expand Down
5 changes: 5 additions & 0 deletions echo/directus/sync/collections/folders.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,10 @@
"name": "Public",
"parent": null,
"_syncId": "74232676-80e7-4f8c-8012-c0d59e6d0a24"
},
{
"name": "avatars",
"parent": null,
"_syncId": "da1c3f3e-4398-4dda-950e-9123c0873fbb"
}
]
7 changes: 7 additions & 0 deletions echo/directus/sync/collections/permissions.json
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,13 @@
}
}
},
{
"folder": {
"name": {
"_contains": "avatars"
}
}
},
{
"folder": {
"name": {
Expand Down
28 changes: 28 additions & 0 deletions echo/directus/sync/snapshot/collections/prompt_template.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"collection": "prompt_template",
"meta": {
"accountability": "all",
"archive_app_filter": true,
"archive_field": null,
"archive_value": null,
"collapse": "open",
"collection": "prompt_template",
"color": null,
"display_template": null,
"group": null,
"hidden": false,
"icon": null,
"item_duplication_fields": null,
"note": null,
"preview_url": null,
"singleton": false,
"sort": null,
"sort_field": null,
"translations": null,
"unarchive_value": null,
"versioning": false
},
"schema": {
"name": "prompt_template"
}
}
Loading
Loading