ECHO-164 cron job to collect unfinished conversations

.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
   push:
     branches:
       - main
+  merge_group:
+    branches:
+      - main
 
 jobs:
   ci-check-server:

.github/workflows/dev-deploy-gitops-backends.yaml

@@ -4,6 +4,9 @@ on:
   push:
     branches:
       - main
+  merge_group:
+    branches:
+      - main
 
 jobs:
   dev-deploy-gitops-backends:

.github/workflows/dev-deploy-vercel-dashboard.yml

@@ -6,6 +6,9 @@ on:
   push:
     branches:
       - main
+  merge_group:
+    branches:
+      - main
 
 jobs:
   dev-deploy-vercel-dashboard:

.github/workflows/dev-deploy-vercel-portal.yml

@@ -6,6 +6,9 @@ on:
   push:
     branches:
       - main
+  merge_group:
+    branches:
+      - main
 
 jobs:
   dev-deploy-vercel-portal:

echo/.gitignore

@@ -15,6 +15,8 @@ directus/.env
 node_modules
 .next
 
+celerybeat-schedule.db
+
 __blobstorage__
 __queuestorage__echo/server/dembrane/workspace_script.py
 

echo/.vscode/launch.json

@@ -2,16 +2,26 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python: ProcessTracker Debug",
-      "type": "python",
+      "name": "Python Debugger: Module",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "dembrane.tasks",
+      "cwd": "${workspaceFolder}/server",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/server"
+      },
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Python: Conversation Utils Debug",
+      "type": "debugpy",
       "request": "launch",
-      "program": "dembrane/audio_lightrag/tests/unit_tests/directus_etl_pipeline_test.py",
+      "program": "dembrane/conversation_utils.py",
       "cwd": "${workspaceFolder}/server",
       "env": {
         "PYTHONPATH": "${workspaceFolder}/server"
       },
-      "console": "integratedTerminal",
-      "justMyCode": "false"
+      "console": "integratedTerminal"
     },
     {
       "name": "Python: Contextual Chunk ETL Pipeline Debug",
@@ -22,19 +32,18 @@
       "env": {
         "PYTHONPATH": "${workspaceFolder}/server"
       },
-      "console": "integratedTerminal",
-      "justMyCode": "false"
+      "console": "integratedTerminal"
     },
     {
       "name": "Python Debugger: transcribe",
       "type": "debugpy",
       "request": "launch",
       "module": "dembrane.transcribe",
-      "cwd": "${workspaceFolder}/server"
+      "cwd": "${workspaceFolder}/server",
+      "env": {
         "PYTHONPATH": "${workspaceFolder}/server"
       },
-      "console": "integratedTerminal",
-      "justMyCode": "false"
+      "console": "integratedTerminal"
     },
     {
       "name": "Python: LightRAG Embedding Debug",
@@ -45,8 +54,7 @@
       "env": {
         "PYTHONPATH": "${workspaceFolder}/server"
       },
-      "console": "integratedTerminal",
-      "justMyCode": "false"
+      "console": "integratedTerminal"
     },
     {
       "name": "Python: LightRAG Test",
@@ -57,8 +65,18 @@
       "env": {
         "PYTHONPATH": "${workspaceFolder}/server"
       },
-      "console": "integratedTerminal",
-      "justMyCode": "false"
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Python: Conversation Utils Debug",
+      "type": "python",
+      "request": "launch",
+      "program": "dembrane/conversation_utils.py",
+      "cwd": "${workspaceFolder}/server",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/server"
+      },
+      "console": "integratedTerminal"
     }
   ]
-}
+}

echo/.vscode/settings.json

@@ -35,6 +35,26 @@
             "source .venv/bin/activate",
             "./run-worker.sh"
           ]
+        },
+        {
+          "name": "workers-cpu",
+          "commands": [
+            "cd server",
+            "source .venv/bin/activate",
+            "./run-worker-cpu.sh"
+          ]
+        }
+      ]
+    },
+    {
+      "splitTerminals": [
+        {
+          "name": "scheduler",
+          "commands": [
+            "cd server",
+            "source .venv/bin/activate",
+            "./run-scheduler.sh"
+          ]
         }
       ]
     },

echo/directus/directus-sync.config.js

@@ -4,10 +4,4 @@ module.exports = {
   // OR needs to be run in the same container as the directus server
   dumpPath: './sync',
   preserveIds: ['roles', 'policies', 'dashboards', 'panels'],
-  exclude: {
-    collections: [
-      "lightrag_chunk_graph_map","lightrag_doc_chunks","lightrag_doc_full","lightrag_doc_status",
-      "lightrag_llm_cache","lightrag_vdb_entity","lightrag_vdb_relation","lightrag_vdb_transcript"  // This regex pattern will match any collection that starts with "lightrag_"
-    ]
-  }
 };

echo/directus/sync/snapshot/fields/aspect/centroid_embedding.json

@@ -0,0 +1,24 @@
+{
+  "collection": "aspect",
+  "field": "centroid_embedding",
+  "type": "unknown",
+  "meta": null,
+  "schema": {
+    "name": "centroid_embedding",
+    "table": "aspect",
+    "data_type": "vector",
+    "default_value": null,
+    "max_length": null,
+    "numeric_precision": null,
+    "numeric_scale": null,
+    "is_nullable": true,
+    "is_unique": false,
+    "is_indexed": false,
+    "is_primary_key": false,
+    "is_generated": false,
+    "generation_expression": null,
+    "has_auto_increment": false,
+    "foreign_key_table": null,
+    "foreign_key_column": null
+  }
+}

echo/directus/sync/snapshot/fields/conversation/is_finished.json

@@ -0,0 +1,45 @@
+{
+  "collection": "conversation",
+  "field": "is_finished",
+  "type": "boolean",
+  "meta": {
+    "collection": "conversation",
+    "conditions": null,
+    "display": null,
+    "display_options": null,
+    "field": "is_finished",
+    "group": null,
+    "hidden": false,
+    "interface": "boolean",
+    "note": null,
+    "options": null,
+    "readonly": false,
+    "required": false,
+    "sort": 24,
+    "special": [
+      "cast-boolean"
+    ],
+    "translations": null,
+    "validation": null,
+    "validation_message": null,
+    "width": "full"
+  },
+  "schema": {
+    "name": "is_finished",
+    "table": "conversation",
+    "data_type": "boolean",
+    "default_value": false,
+    "max_length": null,
+    "numeric_precision": null,
+    "numeric_scale": null,
+    "is_nullable": true,
+    "is_unique": false,
+    "is_indexed": false,
+    "is_primary_key": false,
+    "is_generated": false,
+    "generation_expression": null,
+    "has_auto_increment": false,
+    "foreign_key_table": null,
+    "foreign_key_column": null
+  }
+}

echo/directus/sync/snapshot/fields/conversation_chunk/timestamp.json

@@ -16,9 +16,7 @@
     "readonly": false,
     "required": false,
     "sort": 7,
-    "special": [
-      "date-created"
-    ],
+    "special": null,
     "translations": null,
     "validation": null,
     "validation_message": null,

echo/directus/sync/snapshot/fields/project/is_enhanced_audio_processing_enabled.json

@@ -0,0 +1,45 @@
+{
+  "collection": "project",
+  "field": "is_enhanced_audio_processing_enabled",
+  "type": "boolean",
+  "meta": {
+    "collection": "project",
+    "conditions": null,
+    "display": null,
+    "display_options": null,
+    "field": "is_enhanced_audio_processing_enabled",
+    "group": null,
+    "hidden": false,
+    "interface": "boolean",
+    "note": null,
+    "options": null,
+    "readonly": false,
+    "required": false,
+    "sort": 25,
+    "special": [
+      "cast-boolean"
+    ],
+    "translations": null,
+    "validation": null,
+    "validation_message": null,
+    "width": "full"
+  },
+  "schema": {
+    "name": "is_enhanced_audio_processing_enabled",
+    "table": "project",
+    "data_type": "boolean",
+    "default_value": true,
+    "max_length": null,
+    "numeric_precision": null,
+    "numeric_scale": null,
+    "is_nullable": true,
+    "is_unique": false,
+    "is_indexed": false,
+    "is_primary_key": false,
+    "is_generated": false,
+    "generation_expression": null,
+    "has_auto_increment": false,
+    "foreign_key_table": null,
+    "foreign_key_column": null
+  }
+}

echo/directus/sync/snapshot/fields/project_chat/auto_select.json

@@ -32,7 +32,7 @@
     "max_length": null,
     "numeric_precision": null,
     "numeric_scale": null,
-    "is_nullable": false,
+    "is_nullable": true,
     "is_unique": false,
     "is_indexed": false,
     "is_primary_key": false,