diff --git a/.github/workflows/ci-servers.yml b/.github/workflows/ci-servers.yml deleted file mode 100644 index 29340a2c..00000000 --- a/.github/workflows/ci-servers.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: ci-servers - -on: - # Run on PRs (to check if the images can be built) - pull_request: - branches: - - main - # Run on pushes to main (to publish the images) - push: - branches: - - main - -jobs: - ci-servers: - runs-on: ubuntu-latest - strategy: - matrix: - image: - # Directus - - name: dbr-echo-directus - context: ./echo/directus - dockerfile: Dockerfile - tag: dbr-echo-directus - build_args: "" - # Shared image for server and worker - - name: dbr-echo-server - context: ./echo/server - dockerfile: Dockerfile - tag: dbr-echo-server - build_args: "" - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - # This enables the creation of a builder instance with persistent cache - driver-opts: | - image=moby/buildkit:latest - network=host - - - name: Log in to DigitalOcean Container Registry - # Login step runs on both PR and push events - uses: docker/login-action@v2 - with: - registry: registry.digitalocean.com - username: ${{ secrets.DO_REGISTRY_USERNAME }} - password: ${{ secrets.DO_REGISTRY_TOKEN }} - - - name: Build (also push sometimes) ${{ matrix.image.name }} - uses: docker/build-push-action@v3 - with: - context: ${{ matrix.image.context }} - file: ${{ matrix.image.context }}/${{ matrix.image.dockerfile }} - # Only push if the event is a push event to main. - push: ${{ github.event_name == 'push' }} - tags: registry.digitalocean.com/dbr-cr/${{ matrix.image.tag }}:${{ github.sha }} - build-args: ${{ matrix.image.build_args }} - # Enhanced cache settings - cache-from: | - type=gha,scope=${{ matrix.image.name }} - type=registry,ref=registry.digitalocean.com/dbr-cr/${{ matrix.image.tag }}:latest - cache-to: type=gha,scope=${{ matrix.image.name }},mode=max \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..17a9b80c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,141 @@ +name: ci + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + ci-check-server: + name: ci-check-server + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: "3.11" + + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('server/requirements.lock') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: pip install -r echo/server/requirements.lock + + - name: Run mypy + uses: tsuyoshicho/action-mypy@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-review + level: warning + workdir: echo/server + + - name: Run ruff + uses: chartboost/ruff-action@v1 + with: + src: echo/server + + ci-check-frontend: + name: ci-check-frontend + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./echo/frontend/ + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: "22" + + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 10 + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + name: Setup pnpm cache + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Lint and build + run: pnpm run build + + ci-build-and-push-servers: + name: ci-build-servers + needs: [ci-check-server, ci-check-frontend] + runs-on: ubuntu-latest + strategy: + matrix: + image: + # Directus + - name: dbr-echo-directus + context: ./echo/directus + dockerfile: Dockerfile + tag: dbr-echo-directus + build_args: "" + # Shared image for server and worker + - name: dbr-echo-server + context: ./echo/server + dockerfile: Dockerfile + tag: dbr-echo-server + build_args: "" + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + # This enables the creation of a builder instance with persistent cache + driver-opts: | + image=moby/buildkit:latest + network=host + + - name: Log in to DigitalOcean Container Registry + uses: docker/login-action@v2 + with: + registry: registry.digitalocean.com + username: ${{ secrets.DO_REGISTRY_USERNAME }} + password: ${{ secrets.DO_REGISTRY_TOKEN }} + + - name: Build (also push sometimes) ${{ matrix.image.name }} + uses: docker/build-push-action@v3 + with: + context: ${{ matrix.image.context }} + file: ${{ matrix.image.context }}/${{ matrix.image.dockerfile }} + # Only push if the event is a push event to main. + push: ${{ github.event_name == 'push' }} + tags: registry.digitalocean.com/dbr-cr/${{ matrix.image.tag }}:${{ github.sha }} + build-args: ${{ matrix.image.build_args }} + # Enhanced cache settings + cache-from: | + type=gha,scope=${{ matrix.image.name }} + type=registry,ref=registry.digitalocean.com/dbr-cr/${{ matrix.image.tag }}:latest + cache-to: type=gha,scope=${{ matrix.image.name }},mode=max \ No newline at end of file diff --git a/.github/workflows/dev-deploy-vercel-dashboard.yml b/.github/workflows/dev-deploy-vercel-dashboard.yml index 412757fe..3c0a9c87 100644 --- a/.github/workflows/dev-deploy-vercel-dashboard.yml +++ b/.github/workflows/dev-deploy-vercel-dashboard.yml @@ -1,4 +1,4 @@ -name: dev-deploy-dashboard-vercel +name: dev-deploy-vercel-dashboard env: VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_DASHBOARD }} @@ -8,7 +8,7 @@ on: - main jobs: - dev-deploy-dashboard-vercel: + dev-deploy-vercel-dashboard: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/prod-deploy-vercel-dashboard.yml b/.github/workflows/prod-deploy-vercel-dashboard.yml new file mode 100644 index 00000000..3c0a9c87 --- /dev/null +++ b/.github/workflows/prod-deploy-vercel-dashboard.yml @@ -0,0 +1,43 @@ +name: dev-deploy-vercel-dashboard +env: + VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_DASHBOARD }} +on: + push: + branches: + - main + +jobs: + dev-deploy-vercel-dashboard: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - uses: pnpm/action-setup@v2 + name: Install pnpm + with: + version: 10 + run_install: true + cwd: echo/frontend + + - name: Install Vercel CLI + run: pnpm add --global vercel@latest + working-directory: echo/frontend + + - name: Pull Vercel Environment Information + run: vercel pull --yes --environment=staging --token=${{ secrets.VERCEL_TOKEN }} + working-directory: echo/frontend + + - name: Build Project Artifacts + run: vercel build --target=staging --token=${{ secrets.VERCEL_TOKEN }} + working-directory: echo/frontend + + - name: Deploy Project Artifacts to Vercel + run: vercel deploy --prebuilt --target=staging --token=${{ secrets.VERCEL_TOKEN }} + working-directory: echo/frontend \ No newline at end of file diff --git a/.github/workflows/prod-deploy-vercel-portal.yml b/.github/workflows/prod-deploy-vercel-portal.yml new file mode 100644 index 00000000..8b45a89d --- /dev/null +++ b/.github/workflows/prod-deploy-vercel-portal.yml @@ -0,0 +1,43 @@ +name: dev-deploy-vercel-portal +env: + VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_PORTAL }} +on: + push: + branches: + - main + +jobs: + dev-deploy-vercel-portal: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - uses: pnpm/action-setup@v2 + name: Install pnpm + with: + version: 10 + run_install: true + cwd: echo/frontend + + - name: Install Vercel CLI + run: pnpm add --global vercel@latest + working-directory: echo/frontend + + - name: Pull Vercel Environment Information + run: vercel pull --yes --environment=staging --token=${{ secrets.VERCEL_TOKEN }} + working-directory: echo/frontend + + - name: Build Project Artifacts + run: vercel build --target=staging --token=${{ secrets.VERCEL_TOKEN }} + working-directory: echo/frontend + + - name: Deploy Project Artifacts to Vercel + run: vercel deploy --prebuilt --target=staging --token=${{ secrets.VERCEL_TOKEN }} + working-directory: echo/frontend \ No newline at end of file diff --git a/README.md b/README.md index d1856eea..c777cc89 100644 --- a/README.md +++ b/README.md @@ -98,4 +98,6 @@ Thanks to everyone in the community that has contributed to this project! - \ No newline at end of file + + +![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/Dembrane/echo?utm_source=oss&utm_medium=github&utm_campaign=Dembrane%2Fecho&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) \ No newline at end of file diff --git a/echo/readme.md b/echo/readme.md index 574f685e..9ca3f96c 100644 --- a/echo/readme.md +++ b/echo/readme.md @@ -1,5 +1,7 @@ # Dembrane ECHO +![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/Dembrane/echo?utm_source=oss&utm_medium=github&utm_campaign=Dembrane%2Fecho&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) + ## Architecture Data Storage: diff --git a/echo/server/dembrane/api/participant.py b/echo/server/dembrane/api/participant.py index 67b66c5c..d5ce4c9e 100644 --- a/echo/server/dembrane/api/participant.py +++ b/echo/server/dembrane/api/participant.py @@ -1,4 +1,3 @@ -import os from typing import List, Optional, Annotated from logging import getLogger from datetime import datetime @@ -7,10 +6,9 @@ from pydantic import BaseModel from sqlalchemy.orm import joinedload +from dembrane.s3 import save_to_s3_from_file_like from dembrane.tasks import task_finish_conversation_hook, task_process_conversation_chunk from dembrane.utils import generate_uuid -from dembrane.config import AUDIO_CHUNKS_DIR -from dembrane.s3 import save_to_s3_from_file_like from dembrane.schemas import ( ProjectTagSchema, ConversationChunkSchema,