Feat/sandbox provisioner shared storage

.env.template

@@ -6,6 +6,38 @@ RUN_CANCEL_KEY_TTL_SECONDS=1800
 LANGGRAPH_CHECKPOINTER_BACKEND=postgres
 VITE_USE_RUNS_API=false
 
+# Sandbox (deerFlow-style provisioner)
+SANDBOX_PROVIDER=provisioner
+SANDBOX_PROVISIONER_URL=http://sandbox-provisioner:8002
+SANDBOX_VIRTUAL_PATH_PREFIX=/mnt/user-data
+SANDBOX_EXEC_TIMEOUT_SECONDS=180
+SANDBOX_MAX_OUTPUT_BYTES=262144
+SANDBOX_KEEPALIVE_INTERVAL_SECONDS=30
+SANDBOX_IDLE_TIMEOUT_SECONDS=120
+SANDBOX_IDLE_CHECK_INTERVAL_SECONDS=10
+# sandbox-provisioner backend: memory | local | docker | kubernetes
+SANDBOX_PROVISIONER_BACKEND=local
+# local/docker backend defaults (deerFlow local_backend style)
+# SANDBOX_CONTAINER_PORT=8080
+# SANDBOX_DOCKER_SANDBOX_HOST=host.docker.internal
+# Docker provisioner options (used when SANDBOX_PROVISIONER_BACKEND=local/docker)
+# SANDBOX_DOCKER_NETWORK=yuxi-know_app-network
+# SANDBOX_DOCKER_THREADS_HOST_PATH=
+# SANDBOX_DOCKER_SKILLS_HOST_PATH=
+# SANDBOX_DOCKER_SANDBOX_PREFIX=yuxi-sandbox
+# Optional proxy for sandbox-provisioner container
+# SANDBOX_HTTP_PROXY=http://host.docker.internal:7897
+# SANDBOX_HTTPS_PROXY=http://host.docker.internal:7897
+# K8s provisioner options (used when SANDBOX_PROVISIONER_BACKEND=kubernetes)
+# SANDBOX_K8S_NAMESPACE=yuxi-know
+# SANDBOX_IMAGE=enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest
+# SANDBOX_SKILLS_HOST_PATH=/app/saves/skills
+# SANDBOX_THREADS_HOST_PATH=/app/saves/threads
+# SANDBOX_NODE_HOST=host.docker.internal
+# KUBECONFIG_PATH=/root/.kube/config
+# Memory backend sandbox url template, supports {sandbox_id}
+# MEMORY_SANDBOX_URL_TEMPLATE=http://agent-sandbox:8000
+
 # region model_provider
 SILICONFLOW_API_KEY=  # 推荐使用硅基流动免费服务 https://cloud.siliconflow.cn/i/Eo5yTHGJ
 TAVILY_API_KEY=  # 获取搜索服务的 api key 请访问 https://app.tavily.com/

AGENTS.md

@@ -37,7 +37,7 @@ make format        # 格式化代码
 # 直接在容器内执行命令
 docker compose exec api uv run python test/your_script.py  # 放在 test 文件夹
 ```
-
+安装依赖太慢时可以使用代理端口 7897
 注意：
 - Python 代码要符合 Python 的规范，符合 pythonic 风格
 - 尽量使用较新的语法，避免使用旧版本的语法（版本兼容到 3.12+）

docker-compose.prod.yml

@@ -32,8 +32,13 @@ services:
       - MINIO_URI=${MINIO_URI:-http://milvus-minio:9000}
       - MODEL_DIR_IN_DOCKER=/models
       - RUNNING_IN_DOCKER=true
-      - NO_PROXY=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,api.siliconflow.cn
-      - no_proxy=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,api.siliconflow.cn
+      - SANDBOX_PROVIDER=${SANDBOX_PROVIDER:-provisioner}
+      - SANDBOX_PROVISIONER_URL=${SANDBOX_PROVISIONER_URL:-http://sandbox-provisioner:8002}
+      - SANDBOX_VIRTUAL_PATH_PREFIX=${SANDBOX_VIRTUAL_PATH_PREFIX:-/mnt/user-data}
+      - SANDBOX_EXEC_TIMEOUT_SECONDS=${SANDBOX_EXEC_TIMEOUT_SECONDS:-180}
+      - SANDBOX_MAX_OUTPUT_BYTES=${SANDBOX_MAX_OUTPUT_BYTES:-262144}
+      - NO_PROXY=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
+      - no_proxy=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
     command: uv run --no-dev uvicorn server.main:app --host 0.0.0.0 --port 5050
     restart: unless-stopped
     healthcheck:
@@ -51,6 +56,8 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
+      sandbox-provisioner:
+        condition: service_healthy
 
   worker:
     build:
@@ -85,8 +92,13 @@ services:
       - MINIO_URI=${MINIO_URI:-http://milvus-minio:9000}
       - MODEL_DIR_IN_DOCKER=/models
       - RUNNING_IN_DOCKER=true
-      - NO_PROXY=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,api.siliconflow.cn
-      - no_proxy=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,api.siliconflow.cn
+      - SANDBOX_PROVIDER=${SANDBOX_PROVIDER:-provisioner}
+      - SANDBOX_PROVISIONER_URL=${SANDBOX_PROVISIONER_URL:-http://sandbox-provisioner:8002}
+      - SANDBOX_VIRTUAL_PATH_PREFIX=${SANDBOX_VIRTUAL_PATH_PREFIX:-/mnt/user-data}
+      - SANDBOX_EXEC_TIMEOUT_SECONDS=${SANDBOX_EXEC_TIMEOUT_SECONDS:-180}
+      - SANDBOX_MAX_OUTPUT_BYTES=${SANDBOX_MAX_OUTPUT_BYTES:-262144}
+      - NO_PROXY=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
+      - no_proxy=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
     command: uv run --no-dev arq server.worker_main.WorkerSettings
     restart: unless-stopped
     depends_on:
@@ -98,6 +110,46 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
+      sandbox-provisioner:
+        condition: service_healthy
+
+  sandbox-provisioner:
+    build:
+      context: ./docker/sandbox_provisioner
+      dockerfile: Dockerfile
+    container_name: sandbox-provisioner
+    volumes:
+      - ./saves:/app/saves
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - app-network
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      - PROVISIONER_BACKEND=${SANDBOX_PROVISIONER_BACKEND:-local}
+      - K8S_NAMESPACE=${SANDBOX_K8S_NAMESPACE:-yuxi-know}
+      - SANDBOX_IMAGE=${SANDBOX_IMAGE:-enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest}
+      - SANDBOX_CONTAINER_PORT=${SANDBOX_CONTAINER_PORT:-8080}
+      - SKILLS_HOST_PATH=${SANDBOX_SKILLS_HOST_PATH:-/app/saves/skills}
+      - THREADS_HOST_PATH=${SANDBOX_THREADS_HOST_PATH:-/app/saves/threads}
+      - NODE_HOST=${SANDBOX_NODE_HOST:-host.docker.internal}
+      - KUBECONFIG_PATH=${KUBECONFIG_PATH:-}
+      - MEMORY_SANDBOX_URL_TEMPLATE=${MEMORY_SANDBOX_URL_TEMPLATE:-http://agent-sandbox:8000}
+      - DOCKER_NETWORK=${SANDBOX_DOCKER_NETWORK:-yuxi-know_app-network}
+      - DOCKER_THREADS_HOST_PATH=${SANDBOX_DOCKER_THREADS_HOST_PATH:-}
+      - DOCKER_SKILLS_HOST_PATH=${SANDBOX_DOCKER_SKILLS_HOST_PATH:-}
+      - DOCKER_SANDBOX_PREFIX=${SANDBOX_DOCKER_SANDBOX_PREFIX:-yuxi-sandbox}
+      - DOCKER_SANDBOX_HOST=${SANDBOX_DOCKER_SANDBOX_HOST:-host.docker.internal}
+      - SANDBOX_IDLE_TIMEOUT_SECONDS=${SANDBOX_IDLE_TIMEOUT_SECONDS:-120}
+      - SANDBOX_IDLE_CHECK_INTERVAL_SECONDS=${SANDBOX_IDLE_CHECK_INTERVAL_SECONDS:-10}
+      - SANDBOX_EXEC_TIMEOUT_SECONDS=${SANDBOX_EXEC_TIMEOUT_SECONDS:-180}
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8002/health').read()"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 10s
+    restart: unless-stopped
 
   web:
     build:

docker-compose.yml

@@ -3,7 +3,7 @@ services:
     build:
       context: .
       dockerfile: docker/api.Dockerfile
-    image: yuxi-api:0.5.dev
+    image: yuxi-api:0.5.2.dev
     container_name: api-dev
     working_dir: /app
     volumes:
@@ -41,8 +41,13 @@ services:
       - MINIO_URI=${MINIO_URI:-http://milvus-minio:9000}
       - MODEL_DIR_IN_DOCKER=/models
       - RUNNING_IN_DOCKER=true
-      - NO_PROXY=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,api.siliconflow.cn
-      - no_proxy=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,api.siliconflow.cn
+      - SANDBOX_PROVIDER=${SANDBOX_PROVIDER:-provisioner}
+      - SANDBOX_PROVISIONER_URL=${SANDBOX_PROVISIONER_URL:-http://sandbox-provisioner:8002}
+      - SANDBOX_VIRTUAL_PATH_PREFIX=${SANDBOX_VIRTUAL_PATH_PREFIX:-/mnt/user-data}
+      - SANDBOX_EXEC_TIMEOUT_SECONDS=${SANDBOX_EXEC_TIMEOUT_SECONDS:-180}
+      - SANDBOX_MAX_OUTPUT_BYTES=${SANDBOX_MAX_OUTPUT_BYTES:-262144}
+      - NO_PROXY=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
+      - no_proxy=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
     # endregion api_envs
     command: uv run --no-dev uvicorn server.main:app --host 0.0.0.0 --port 5050 --reload
     restart: unless-stopped
@@ -61,12 +66,14 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
+      sandbox-provisioner:
+        condition: service_healthy
 
   worker:
     build:
       context: .
       dockerfile: docker/api.Dockerfile
-    image: yuxi-api:0.5.dev
+    image: yuxi-api:0.5.2.dev
     container_name: worker-dev
     working_dir: /app
     volumes:
@@ -103,8 +110,13 @@ services:
       - MINIO_URI=${MINIO_URI:-http://milvus-minio:9000}
       - MODEL_DIR_IN_DOCKER=/models
       - RUNNING_IN_DOCKER=true
-      - NO_PROXY=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,api.siliconflow.cn
-      - no_proxy=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,api.siliconflow.cn
+      - SANDBOX_PROVIDER=${SANDBOX_PROVIDER:-provisioner}
+      - SANDBOX_PROVISIONER_URL=${SANDBOX_PROVISIONER_URL:-http://sandbox-provisioner:8002}
+      - SANDBOX_VIRTUAL_PATH_PREFIX=${SANDBOX_VIRTUAL_PATH_PREFIX:-/mnt/user-data}
+      - SANDBOX_EXEC_TIMEOUT_SECONDS=${SANDBOX_EXEC_TIMEOUT_SECONDS:-180}
+      - SANDBOX_MAX_OUTPUT_BYTES=${SANDBOX_MAX_OUTPUT_BYTES:-262144}
+      - NO_PROXY=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
+      - no_proxy=localhost,127.0.0.1,host.docker.internal,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio,mineru,paddlex,sandbox-provisioner,api.siliconflow.cn
     command: uv run --no-dev arq server.worker_main.WorkerSettings
     restart: unless-stopped
     depends_on:
@@ -116,13 +128,63 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
+      sandbox-provisioner:
+        condition: service_healthy
+
+  sandbox-provisioner:
+    build:
+      context: ./docker/sandbox_provisioner
+      dockerfile: Dockerfile
+    image: yuxi-sandbox-provisioner:0.5.2.dev
+    container_name: sandbox-provisioner
+    volumes:
+      - ./saves:/app/saves
+      - ./docker/sandbox_provisioner/app.py:/app/app.py:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    ports:
+      - "8002:8002"
+    networks:
+      - app-network
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      - PROVISIONER_BACKEND=${SANDBOX_PROVISIONER_BACKEND:-local}
+      - K8S_NAMESPACE=${SANDBOX_K8S_NAMESPACE:-yuxi-know}
+      - SANDBOX_IMAGE=${SANDBOX_IMAGE:-enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest}
+      - SANDBOX_CONTAINER_PORT=${SANDBOX_CONTAINER_PORT:-8080}
+      - SKILLS_HOST_PATH=${SANDBOX_SKILLS_HOST_PATH:-/app/saves/skills}
+      - THREADS_HOST_PATH=${SANDBOX_THREADS_HOST_PATH:-/app/saves/threads}
+      - NODE_HOST=${SANDBOX_NODE_HOST:-host.docker.internal}
+      - KUBECONFIG_PATH=${KUBECONFIG_PATH:-}
+      - MEMORY_SANDBOX_URL_TEMPLATE=${MEMORY_SANDBOX_URL_TEMPLATE:-http://agent-sandbox:8000}
+      - HTTP_PROXY=${SANDBOX_HTTP_PROXY:-}
+      - HTTPS_PROXY=${SANDBOX_HTTPS_PROXY:-}
+      - NO_PROXY=localhost,127.0.0.1,host.docker.internal
+      - DOCKER_NETWORK=${SANDBOX_DOCKER_NETWORK:-yuxi-know_app-network}
+      - DOCKER_THREADS_HOST_PATH=${SANDBOX_DOCKER_THREADS_HOST_PATH:-}
+      - DOCKER_SKILLS_HOST_PATH=${SANDBOX_DOCKER_SKILLS_HOST_PATH:-}
+      - DOCKER_SANDBOX_PREFIX=${SANDBOX_DOCKER_SANDBOX_PREFIX:-yuxi-sandbox}
+      - DOCKER_SANDBOX_HOST=${SANDBOX_DOCKER_SANDBOX_HOST:-host.docker.internal}
+      - SANDBOX_IDLE_TIMEOUT_SECONDS=${SANDBOX_IDLE_TIMEOUT_SECONDS:-120}
+      - SANDBOX_IDLE_CHECK_INTERVAL_SECONDS=${SANDBOX_IDLE_CHECK_INTERVAL_SECONDS:-10}
+      - SANDBOX_EXEC_TIMEOUT_SECONDS=${SANDBOX_EXEC_TIMEOUT_SECONDS:-180}
+    command: >
+      sh -lc "python -c 'import docker' >/dev/null 2>&1 || pip install --no-cache-dir 'docker>=7.1.0';
+      uvicorn app:app --host 0.0.0.0 --port 8002"
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8002/health').read()"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 10s
+    restart: unless-stopped
 
   web:
     build:
       context: .
       dockerfile: docker/web.Dockerfile
       target: development
-    image: yuxi-web:0.5.dev
+    image: yuxi-web:0.5.2.dev
     container_name: web-dev
     volumes:
       - ./web/src:/app/src
@@ -265,6 +327,8 @@ services:
   redis:
     image: redis:7-alpine
     container_name: redis
+    ports:
+      - "6379:6379"
     command: redis-server --appendonly yes
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]

docker/sandbox_provisioner/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+COPY app.py /app/app.py
+
+EXPOSE 8002
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8002"]