diff --git a/.bumpversion.toml b/.bumpversion.toml new file mode 100644 index 0000000..235db6a --- /dev/null +++ b/.bumpversion.toml @@ -0,0 +1,54 @@ +[tool.bumpversion] +# 当前版本号（自动从 pyproject.toml 读取） +current_version = "0.7.0" + +# 版本解析格式（支持 major.minor.patch） +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" + +# 版本序列化格式 +serialize = ["{major}.{minor}.{patch}"] + +# 搜索和替换的格式 +search = "{current_version}" +replace = "{new_version}" + +# Git 集成 +commit = true +commit_args = "" +tag = true +tag_name = "v{new_version}" +tag_message = "Release version {new_version}" +sign_tags = false + +# 允许 dirty 工作区（如果需要） +allow_dirty = false + +# 提交信息模板 +message = "chore: bump version from {current_version} to {new_version}" + +# 需要更新版本号的文件列表 +[[tool.bumpversion.files]] +filename = "pyproject.toml" +search = 'version = "{current_version}"' +replace = 'version = "{new_version}"' + +[[tool.bumpversion.files]] +filename = "src/__version__.py" +search = '__version__ = "{current_version}"' +replace = '__version__ = "{new_version}"' + +# Note: docs/changelog.md is now automatically generated by scripts/generate-changelog.py +# It's called by scripts/bump-version.sh before bump-my-version runs + +# 版本部分定义 +[tool.bumpversion.parts.major] +# major 版本从 0 开始，递增到任意数字 +values = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + +[tool.bumpversion.parts.minor] +# minor 版本从 0 开始，递增到任意数字 +values = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + +[tool.bumpversion.parts.patch] +# patch 版本从 0 开始，递增到任意数字 +values = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..8de2b7b --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,246 @@ +name: Build and Push Docker Images + +on: + push: + branches: + - main + tags: + - 'v*' + pull_request: + branches: + - main + workflow_dispatch: + +env: + DOCKER_USER: royisme + DOCKER_REGISTRY: docker.io + +jobs: + validate-version: + name: Validate Version Consistency + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Validate version consistency + run: | + echo "=== Version Validation ===" + + # Get version from pyproject.toml + PROJECT_VERSION=$(grep '^version = ' pyproject.toml | cut -d'"' -f2) + echo "pyproject.toml version: $PROJECT_VERSION" + + # Get version from __version__.py + VERSION_PY=$(grep '__version__ = ' src/__version__.py | cut -d'"' -f2) + echo "__version__.py version: $VERSION_PY" + + # Validate Python version file + if [[ "$PROJECT_VERSION" != "$VERSION_PY" ]]; then + echo "❌ Error: Version mismatch!" + echo " pyproject.toml: $PROJECT_VERSION" + echo " __version__.py: $VERSION_PY" + exit 1 + fi + + # If this is a tag push, validate tag version + if [[ $GITHUB_REF == refs/tags/* ]]; then + TAG_VERSION=${GITHUB_REF#refs/tags/v} + echo "Git tag version: v$TAG_VERSION" + + if [[ "$PROJECT_VERSION" != "$TAG_VERSION" ]]; then + echo "❌ Error: Version mismatch with tag!" + echo " pyproject.toml: $PROJECT_VERSION" + echo " Git tag: $TAG_VERSION" + exit 1 + fi + fi + + echo "✅ All versions consistent: $PROJECT_VERSION" + + build-minimal: + needs: validate-version + name: Build Minimal Image + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_USER }}/codebase-rag + tags: | + type=ref,event=branch,suffix=-minimal + type=ref,event=pr,suffix=-minimal + type=semver,pattern={{version}},suffix=-minimal + type=semver,pattern={{major}}.{{minor}},suffix=-minimal + type=raw,value=minimal,enable={{is_default_branch}} + type=raw,value=minimal-latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.minimal + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + build-standard: + needs: validate-version + name: Build Standard Image + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_USER }}/codebase-rag + tags: | + type=ref,event=branch,suffix=-standard + type=ref,event=pr,suffix=-standard + type=semver,pattern={{version}},suffix=-standard + type=semver,pattern={{major}}.{{minor}},suffix=-standard + type=raw,value=standard,enable={{is_default_branch}} + type=raw,value=standard-latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.standard + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + build-full: + needs: validate-version + name: Build Full Image + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_USER }}/codebase-rag + tags: | + type=ref,event=branch,suffix=-full + type=ref,event=pr,suffix=-full + type=semver,pattern={{version}},suffix=-full + type=semver,pattern={{major}}.{{minor}},suffix=-full + type=raw,value=full,enable={{is_default_branch}} + type=raw,value=full-latest,enable={{is_default_branch}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.full + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + create-release: + name: Create GitHub Release + if: startsWith(github.ref, 'refs/tags/v') + needs: [build-minimal, build-standard, build-full] + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + generate_release_notes: true + body: | + ## Docker Images + + ### Minimal (Code Graph only) + ```bash + docker pull royisme/codebase-rag:minimal + docker pull royisme/codebase-rag:${{ github.ref_name }}-minimal + ``` + + ### Standard (Code Graph + Memory) + ```bash + docker pull royisme/codebase-rag:standard + docker pull royisme/codebase-rag:${{ github.ref_name }}-standard + ``` + + ### Full (All Features) + ```bash + docker pull royisme/codebase-rag:full + docker pull royisme/codebase-rag:${{ github.ref_name }}-full + docker pull royisme/codebase-rag:latest + ``` + + ## Quick Start + + See [documentation](https://code-graph.vantagecraft.dev) for detailed setup instructions. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + notify: + name: Notify Build Status + needs: [build-minimal, build-standard, build-full] + if: always() + runs-on: ubuntu-latest + steps: + - name: Build Summary + run: | + echo "## 🐳 Docker Build Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Minimal**: ✅ Built" >> $GITHUB_STEP_SUMMARY + echo "- **Standard**: ✅ Built" >> $GITHUB_STEP_SUMMARY + echo "- **Full**: ✅ Built" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Images available at: https://hub.docker.com/r/royisme/codebase-rag" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml new file mode 100644 index 0000000..875323f --- /dev/null +++ b/.github/workflows/docs-deploy.yml @@ -0,0 +1,73 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + paths: + - 'docs/**' + - 'mkdocs.yml' + - '.github/workflows/docs-deploy.yml' + pull_request: + branches: + - main + paths: + - 'docs/**' + - 'mkdocs.yml' + workflow_dispatch: + +permissions: + contents: write + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for git info plugin + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install mkdocs-material + pip install mkdocs-minify-plugin + pip install mkdocs-git-revision-date-localized-plugin + + - name: Build documentation + run: mkdocs build --strict + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + - name: Notify deployment + run: | + echo "📚 Documentation deployed successfully!" + echo "🔗 URL: https://code-graph.vantagecraft.dev" diff --git a/GITHUB_PAGES_TROUBLESHOOTING.md b/GITHUB_PAGES_TROUBLESHOOTING.md new file mode 100644 index 0000000..9bbb657 --- /dev/null +++ b/GITHUB_PAGES_TROUBLESHOOTING.md @@ -0,0 +1,347 @@ +# GitHub Pages 部署诊断和配置指南 + +## 问题：编译成功但没有部署 + +### 原因分析 + +你的 `.github/workflows/docs-deploy.yml` 配置了： + +```yaml +deploy: + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' +``` + +**这意味着**： +- ✅ 在 main 分支会部署 +- ❌ 在 feature 分支只会 build，不会 deploy +- ❌ PR 只会 build，不会 deploy + +### 当前状态检查 + +1. **查看你当前在哪个分支** + ```bash + git branch + # 如果显示 claude/fix-docker-env-config-*，那就是在 feature 分支 + ``` + +2. **查看 GitHub Actions 运行记录** + - 访问：https://github.com/royisme/codebase-rag/actions + - 点击最近的 "Deploy Documentation" workflow + - 检查是否有 "deploy" job + - 如果只有 "build" job，说明条件不满足 + +## 解决方案 + +### 方案 1：合并到 main 分支（推荐） + +```bash +# 1. 确保当前分支所有更改已提交 +git status + +# 2. 切换到 main 分支 +git checkout main + +# 3. 合并你的 feature 分支 +git merge claude/fix-docker-env-config-011CUqY1Y431FvqPZW6YAEhT + +# 4. 推送到远程 +git push origin main + +# 5. GitHub Actions 会自动触发，这次会执行 deploy job +``` + +**验证**： +- 访问 Actions 页面 +- 应该看到 "build" 和 "deploy" 两个 job +- deploy job 完成后，会显示部署 URL + +### 方案 2：手动触发 workflow + +如果你还在 feature 分支，但想测试部署： + +```bash +# 在 GitHub 网站上 +1. 访问：https://github.com/royisme/codebase-rag/actions/workflows/docs-deploy.yml +2. 点击 "Run workflow" 按钮 +3. 选择 "main" 分支 +4. 点击 "Run workflow" +``` + +## GitHub Pages 设置配置 + +### 必须配置的设置 + +1. **访问仓库设置** + ``` + https://github.com/royisme/codebase-rag/settings/pages + ``` + +2. **Source 设置** + - ✅ 选择 "GitHub Actions" + - ❌ 不要选择 "Deploy from a branch" + + ![Source Setting](https://docs.github.com/assets/cb-49683/mw-1440/images/help/pages/publishing-source-drop-down.webp) + +3. **Custom domain 设置** + + **需要配置！** 因为你有 `docs/CNAME` 文件： + + ``` + Custom domain: code-graph.vantagecraft.dev + ☑ Enforce HTTPS + ``` + + **为什么需要？** + - 你的 `docs/CNAME` 文件内容是 `code-graph.vantagecraft.dev` + - 这告诉 GitHub Pages 你想使用自定义域名 + - 必须在 Settings 中也配置这个域名 + +### DNS 配置（必须） + +在你的域名服务商（vantagecraft.dev）配置： + +**CNAME 记录**： +``` +类型: CNAME +名称: code-graph +目标: royisme.github.io +TTL: 3600 (或自动) +``` + +**如何验证**： +```bash +# 检查 DNS 是否生效 +nslookup code-graph.vantagecraft.dev + +# 或者 +dig code-graph.vantagecraft.dev + +# 应该显示： +# code-graph.vantagecraft.dev. IN CNAME royisme.github.io. +``` + +### 完整配置步骤 + +#### Step 1: 配置 DNS（在域名服务商） + +``` +记录类型: CNAME +主机记录: code-graph +记录值: royisme.github.io +TTL: 默认或3600 +``` + +保存后等待 5-10 分钟生效。 + +#### Step 2: 配置 GitHub Pages + +1. 访问：https://github.com/royisme/codebase-rag/settings/pages + +2. **Source 设置**： + - Source: GitHub Actions ✅ + +3. **Custom domain 设置**： + - 输入：`code-graph.vantagecraft.dev` + - 点击 Save + - 等待 DNS 验证（可能需要几分钟） + - 验证成功后，勾选 "Enforce HTTPS" + +#### Step 3: 触发部署 + +```bash +# 方法 1: 合并到 main 分支（推荐） +git checkout main +git merge your-feature-branch +git push origin main + +# 方法 2: 手动触发 +# 在 GitHub Actions 页面点击 "Run workflow" + +# 方法 3: 修改文档触发 +echo "test" >> docs/index.md +git add docs/index.md +git commit -m "docs: trigger deployment" +git push origin main +``` + +#### Step 4: 验证部署 + +1. **查看 GitHub Actions** + - https://github.com/royisme/codebase-rag/actions + - 应该看到 "build" 和 "deploy" 两个 job + - deploy job 状态应该是绿色 ✅ + +2. **查看 Pages 设置** + - https://github.com/royisme/codebase-rag/settings/pages + - 应该显示："Your site is live at https://code-graph.vantagecraft.dev" + +3. **访问网站** + - https://code-graph.vantagecraft.dev + - 应该能看到文档 + +## 常见问题排查 + +### 问题 1: deploy job 不执行 + +**症状**：只有 build job，没有 deploy job + +**原因**： +- 不在 main 分支 +- 是 Pull Request + +**解决**： +```bash +git checkout main +git push origin main +``` + +### 问题 2: DNS check failed + +**症状**：GitHub Pages 显示 "DNS check unsuccessful" + +**原因**：DNS 记录未生效或配置错误 + +**解决**： +```bash +# 1. 检查 DNS +dig code-graph.vantagecraft.dev + +# 2. 确保返回 CNAME 记录指向 royisme.github.io +# 3. 等待 DNS 传播（5-60分钟） +# 4. 在 GitHub Pages 设置中点击 "Remove" 再重新添加域名 +``` + +### 问题 3: 404 Not Found + +**症状**：访问域名显示 404 + +**原因**： +- 部署未完成 +- CNAME 文件缺失 +- 域名配置不一致 + +**解决**： +```bash +# 1. 确认 docs/CNAME 文件存在 +cat docs/CNAME +# 应该显示：code-graph.vantagecraft.dev + +# 2. 确认 GitHub Pages 设置中的 Custom domain 与 CNAME 一致 + +# 3. 重新构建 +git commit --allow-empty -m "chore: trigger rebuild" +git push origin main +``` + +### 问题 4: HTTPS 证书问题 + +**症状**："Certificate error" 或 "Not secure" + +**原因**：GitHub 还在生成 HTTPS 证书 + +**解决**： +- 等待 1-24 小时 +- GitHub 会自动从 Let's Encrypt 获取证书 +- 在此期间可以用 HTTP 访问：http://code-graph.vantagecraft.dev + +### 问题 5: 部署成功但内容是旧的 + +**症状**：网站内容没更新 + +**解决**： +```bash +# 清除浏览器缓存 +# 或强制刷新：Ctrl+Shift+R (Windows/Linux) 或 Cmd+Shift+R (Mac) + +# 或等待 CDN 缓存过期（通常 10 分钟） +``` + +## 最佳实践 + +### 1. 开发流程 + +```bash +# Feature 分支开发 +git checkout -b feature/docs-update +# ... 修改文档 ... +git commit -m "docs: update guide" +git push origin feature/docs-update + +# 创建 PR → 在 PR 中会 build（但不 deploy） +# 合并到 main → 自动 deploy + +# 或者直接在 main 分支开发（小改动） +git checkout main +# ... 修改 ... +git commit -m "docs: fix typo" +git push origin main # 自动触发 deploy +``` + +### 2. 快速测试部署 + +如果想快速看到部署效果： + +```bash +# 1. 空提交触发部署 +git commit --allow-empty -m "docs: trigger deployment" +git push origin main + +# 2. 或修改任意文档 +echo "" >> docs/index.md +git add docs/index.md +git commit -m "docs: trigger deployment" +git push origin main +``` + +### 3. 监控部署状态 + +```bash +# 使用 GitHub CLI +gh run list --workflow=docs-deploy.yml + +# 查看最新运行 +gh run view --log + +# 或在浏览器中查看 +open https://github.com/royisme/codebase-rag/actions +``` + +## 配置检查清单 + +使用这个清单确保所有配置正确： + +- [ ] **DNS 配置** + - [ ] CNAME 记录：code-graph → royisme.github.io + - [ ] DNS 已生效（用 dig/nslookup 验证） + +- [ ] **GitHub Pages 设置** + - [ ] Source: GitHub Actions + - [ ] Custom domain: code-graph.vantagecraft.dev + - [ ] DNS check: ✅ (绿色对勾) + - [ ] Enforce HTTPS: ☑ (勾选) + +- [ ] **代码仓库** + - [ ] docs/CNAME 文件存在，内容正确 + - [ ] .github/workflows/docs-deploy.yml 存在 + - [ ] 在 main 分支 + +- [ ] **GitHub Actions** + - [ ] Workflow 权限正确（pages: write） + - [ ] 最近一次运行包含 deploy job + - [ ] deploy job 状态：✅ Success + +- [ ] **访问验证** + - [ ] https://code-graph.vantagecraft.dev 可访问 + - [ ] HTTPS 证书有效 + - [ ] 内容显示正确 + +## 需要帮助？ + +如果按照以上步骤仍有问题，请提供： + +1. 当前分支名：`git branch` +2. GitHub Actions 运行日志截图 +3. GitHub Pages 设置页面截图 +4. DNS 查询结果：`dig code-graph.vantagecraft.dev` + +我会帮你进一步诊断！ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..18fcbd2 --- /dev/null +++ b/Makefile @@ -0,0 +1,279 @@ +# Makefile for Code Graph Knowledge System +# Provides convenient commands for Docker operations + +.PHONY: help docker-minimal docker-standard docker-full docker-full-with-ollama \ + docker-build-minimal docker-build-standard docker-build-full docker-build-all \ + docker-push docker-pull docker-clean docker-logs docker-stop \ + dev-minimal dev-standard dev-full docs-serve docs-build docs-deploy + +# Docker Hub username +DOCKER_USER ?= royisme + +# Default target +help: + @echo "Code Graph Knowledge System - Docker Commands" + @echo "==============================================" + @echo "" + @echo "Quick Start:" + @echo " make docker-minimal - Start minimal deployment (Code Graph only, no LLM)" + @echo " make docker-standard - Start standard deployment (+ Memory, needs Embedding)" + @echo " make docker-full - Start full deployment (all features, needs LLM)" + @echo " make docker-full-with-ollama - Start full deployment with local Ollama" + @echo "" + @echo "Build Commands:" + @echo " make docker-build-minimal - Build minimal image" + @echo " make docker-build-standard - Build standard image" + @echo " make docker-build-full - Build full image" + @echo " make docker-build-all - Build all images" + @echo "" + @echo "Management:" + @echo " make docker-stop - Stop all services" + @echo " make docker-clean - Stop and remove all containers/volumes" + @echo " make docker-logs - Show logs from all services" + @echo " make docker-push - Push all images to Docker Hub" + @echo " make docker-pull - Pull all images from Docker Hub" + @echo "" + @echo "Development:" + @echo " make dev-minimal - Start minimal in dev mode (mounted code)" + @echo " make dev-standard - Start standard in dev mode" + @echo " make dev-full - Start full in dev mode" + @echo "" + @echo "Documentation:" + @echo " make docs-serve - Serve documentation locally" + @echo " make docs-build - Build documentation" + @echo " make docs-deploy - Deploy documentation to vantagecraft.dev" + @echo "" + +# ============================================ +# Deployment Commands +# ============================================ + +docker-minimal: + @echo "🚀 Starting Minimal deployment (Code Graph only)..." + @echo " ✓ No LLM or Embedding required" + @echo " ✓ Code Graph tools available" + @echo "" + docker-compose -f docker/docker-compose.minimal.yml up -d + @echo "" + @echo "✅ Minimal deployment started!" + @echo " API: http://localhost:8000" + @echo " Neo4j Browser: http://localhost:7474" + @echo "" + @echo "Check status: make docker-logs" + +docker-standard: + @echo "🚀 Starting Standard deployment (Code Graph + Memory)..." + @echo " ⚠️ Embedding provider required (check .env)" + @echo " ✓ Code Graph + Memory Store" + @echo "" + docker-compose -f docker/docker-compose.standard.yml up -d + @echo "" + @echo "✅ Standard deployment started!" + @echo " API: http://localhost:8000" + @echo " Neo4j Browser: http://localhost:7474" + +docker-full: + @echo "🚀 Starting Full deployment (All features)..." + @echo " ⚠️ LLM + Embedding required (check .env)" + @echo " ✓ Code Graph + Memory + Knowledge RAG" + @echo "" + docker-compose -f docker/docker-compose.full.yml up -d + @echo "" + @echo "✅ Full deployment started!" + @echo " API: http://localhost:8000" + @echo " Neo4j Browser: http://localhost:7474" + +docker-full-with-ollama: + @echo "🚀 Starting Full deployment with local Ollama..." + @echo " ✓ Ollama will be started in Docker" + @echo " ✓ All features enabled" + @echo "" + docker-compose -f docker/docker-compose.full.yml --profile with-ollama up -d + @echo "" + @echo "✅ Full deployment with Ollama started!" + @echo " API: http://localhost:8000" + @echo " Neo4j Browser: http://localhost:7474" + @echo " Ollama: http://localhost:11434" + @echo "" + @echo "⏳ Ollama may take a few minutes to download models..." + @echo " Check: docker logs codebase-rag-ollama-full -f" + +# ============================================ +# Build Commands +# ============================================ + +docker-build-minimal: + @echo "🔨 Building minimal image..." + docker-compose -f docker/docker-compose.minimal.yml build + @echo "✅ Minimal image built: royisme/codebase-rag:minimal" + +docker-build-standard: + @echo "🔨 Building standard image..." + docker-compose -f docker/docker-compose.standard.yml build + @echo "✅ Standard image built: royisme/codebase-rag:standard" + +docker-build-full: + @echo "🔨 Building full image..." + docker-compose -f docker/docker-compose.full.yml build + @echo "✅ Full image built: royisme/codebase-rag:full" + +docker-build-all: docker-build-minimal docker-build-standard docker-build-full + @echo "" + @echo "✅ All images built successfully!" + +# ============================================ +# Docker Hub Commands +# ============================================ + +docker-push: docker-build-all + @echo "📤 Pushing images to Docker Hub..." + docker tag royisme/codebase-rag:minimal royisme/codebase-rag:minimal-latest + docker tag royisme/codebase-rag:standard royisme/codebase-rag:standard-latest + docker tag royisme/codebase-rag:full royisme/codebase-rag:full-latest + docker push royisme/codebase-rag:minimal + docker push royisme/codebase-rag:minimal-latest + docker push royisme/codebase-rag:standard + docker push royisme/codebase-rag:standard-latest + docker push royisme/codebase-rag:full + docker push royisme/codebase-rag:full-latest + @echo "✅ All images pushed to Docker Hub!" + +docker-pull: + @echo "📥 Pulling images from Docker Hub..." + docker pull royisme/codebase-rag:minimal + docker pull royisme/codebase-rag:standard + docker pull royisme/codebase-rag:full + @echo "✅ All images pulled!" + +# ============================================ +# Management Commands +# ============================================ + +docker-stop: + @echo "🛑 Stopping all services..." + -docker-compose -f docker/docker-compose.minimal.yml down + -docker-compose -f docker/docker-compose.standard.yml down + -docker-compose -f docker/docker-compose.full.yml down + @echo "✅ All services stopped" + +docker-clean: + @echo "🧹 Cleaning up all containers and volumes..." + @read -p "This will remove all data. Continue? [y/N] " confirm; \ + if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \ + docker-compose -f docker/docker-compose.minimal.yml down -v; \ + docker-compose -f docker/docker-compose.standard.yml down -v; \ + docker-compose -f docker/docker-compose.full.yml down -v; \ + echo "✅ Cleanup complete"; \ + else \ + echo "❌ Cleanup cancelled"; \ + fi + +docker-logs: + @echo "📋 Showing logs from all services..." + @echo " Press Ctrl+C to exit" + @echo "" + @if docker ps | grep -q codebase-rag-mcp-minimal; then \ + docker-compose -f docker/docker-compose.minimal.yml logs -f; \ + elif docker ps | grep -q codebase-rag-mcp-standard; then \ + docker-compose -f docker/docker-compose.standard.yml logs -f; \ + elif docker ps | grep -q codebase-rag-mcp-full; then \ + docker-compose -f docker/docker-compose.full.yml logs -f; \ + else \ + echo "❌ No services running. Start with: make docker-minimal"; \ + fi + +# ============================================ +# Development Mode +# ============================================ + +docker-compose.dev.yml: + @echo "Creating dev compose file..." + @echo "version: '3.8'" > docker/docker-compose.dev.yml + @echo "services:" >> docker/docker-compose.dev.yml + @echo " mcp:" >> docker/docker-compose.dev.yml + @echo " volumes:" >> docker/docker-compose.dev.yml + @echo " - .:/app:delegated # Mount source code" >> docker/docker-compose.dev.yml + @echo " environment:" >> docker/docker-compose.dev.yml + @echo " - DEBUG=true" >> docker/docker-compose.dev.yml + @echo " - PYTHONDONTWRITEBYTECODE=1" >> docker/docker-compose.dev.yml + +dev-minimal: docker-compose.dev.yml + @echo "🔧 Starting minimal in development mode..." + docker-compose -f docker/docker-compose.minimal.yml -f docker/docker-compose.dev.yml up + +dev-standard: docker-compose.dev.yml + @echo "🔧 Starting standard in development mode..." + docker-compose -f docker/docker-compose.standard.yml -f docker/docker-compose.dev.yml up + +dev-full: docker-compose.dev.yml + @echo "🔧 Starting full in development mode..." + docker-compose -f docker/docker-compose.full.yml -f docker/docker-compose.dev.yml up + +# ============================================ +# Documentation Commands +# ============================================ + +docs-serve: + @echo "📚 Serving documentation locally..." + @if ! command -v mkdocs &> /dev/null; then \ + echo "❌ MkDocs not installed. Installing..."; \ + pip install mkdocs-material mkdocs-i18n; \ + fi + mkdocs serve + +docs-build: + @echo "🔨 Building documentation..." + @if ! command -v mkdocs &> /dev/null; then \ + echo "❌ MkDocs not installed. Installing..."; \ + pip install mkdocs-material mkdocs-i18n; \ + fi + mkdocs build + +docs-deploy: + @echo "🚀 Deploying documentation to vantagecraft.dev..." + @echo " Building documentation..." + mkdocs build + @echo "✅ Documentation built in site/ directory" + @echo "" + @echo "📝 Next steps for vantagecraft.dev deployment:" + @echo " 1. Upload site/ contents to your web server" + @echo " 2. Configure DNS: code-graph.vantagecraft.dev -> your server" + @echo " 3. Set up SSL certificate (recommended: Let's Encrypt)" + @echo "" + @echo " Or use GitHub Pages:" + @echo " - mkdocs gh-deploy" + +# ============================================ +# Utility Commands +# ============================================ + +health-check: + @echo "🏥 Checking service health..." + @echo "" + @echo "Neo4j:" + @curl -s http://localhost:7474 > /dev/null && echo " ✅ Running" || echo " ❌ Not running" + @echo "API:" + @curl -s http://localhost:8000/api/v1/health > /dev/null && echo " ✅ Running" || echo " ❌ Not running" + @if docker ps | grep -q ollama; then \ + echo "Ollama:"; \ + curl -s http://localhost:11434/api/tags > /dev/null && echo " ✅ Running" || echo " ❌ Not running"; \ + fi + +init-env: + @echo "📝 Initializing environment file..." + @echo "Which deployment mode? [minimal/standard/full]" + @read mode; \ + if [ "$$mode" = "minimal" ]; then \ + cp docker/.env.template/.env.minimal .env; \ + echo "✅ Created .env for minimal deployment"; \ + elif [ "$$mode" = "standard" ]; then \ + cp docker/.env.template/.env.standard .env; \ + echo "✅ Created .env for standard deployment"; \ + echo "⚠️ Don't forget to configure EMBEDDING_PROVIDER"; \ + elif [ "$$mode" = "full" ]; then \ + cp docker/.env.template/.env.full .env; \ + echo "✅ Created .env for full deployment"; \ + echo "⚠️ Don't forget to configure LLM_PROVIDER and EMBEDDING_PROVIDER"; \ + else \ + echo "❌ Invalid mode. Choose: minimal, standard, or full"; \ + fi diff --git a/docker-compose.yml b/docker-compose.yml index 01f2d62..698b3c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,23 @@ +# Default Docker Compose - Points to Minimal deployment +# For other modes, use: +# - docker-compose -f docker/docker-compose.standard.yml up +# - docker-compose -f docker/docker-compose.full.yml up +# Or use Makefile commands: +# - make docker-minimal +# - make docker-standard +# - make docker-full + version: '3.8' services: - # Neo4j Database neo4j: image: neo4j:5.15-community container_name: codebase-rag-neo4j ports: - - "7474:7474" # HTTP - - "7687:7687" # Bolt + - "${NEO4J_HTTP_PORT:-7474}:7474" + - "${NEO4J_BOLT_PORT:-7687}:7687" environment: - - NEO4J_AUTH=neo4j/password123 + - NEO4J_AUTH=${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-password} - NEO4J_PLUGINS=["apoc"] - NEO4J_dbms_security_procedures_unrestricted=apoc.* - NEO4J_dbms_security_procedures_allowlist=apoc.* @@ -22,86 +30,41 @@ services: - neo4j_import:/var/lib/neo4j/import - neo4j_plugins:/plugins healthcheck: - test: ["CMD-SHELL", "cypher-shell -u neo4j -p password123 'RETURN 1' || exit 1"] + test: ["CMD-SHELL", "cypher-shell -u ${NEO4J_USER:-neo4j} -p ${NEO4J_PASSWORD:-password} 'RETURN 1' || exit 1"] interval: 10s timeout: 5s retries: 5 start_period: 30s networks: - - codebase-rag-network - restart: unless-stopped - - # Ollama (Optional - for local LLM) - ollama: - image: ollama/ollama:latest - container_name: codebase-rag-ollama - ports: - - "11434:11434" - volumes: - - ollama_data:/root/.ollama - environment: - - OLLAMA_HOST=0.0.0.0 - networks: - - codebase-rag-network + - codebase-rag restart: unless-stopped - profiles: - - with-ollama - # Application - app: + mcp: build: context: . - dockerfile: Dockerfile - container_name: codebase-rag-app + dockerfile: docker/Dockerfile.minimal + image: royisme/codebase-rag:minimal + container_name: codebase-rag-mcp ports: - - "8000:8000" + - "${APP_PORT:-8000}:8000" environment: - # Neo4j Configuration + - APP_NAME=${APP_NAME:-Code Graph Knowledge System} + - DEBUG=${DEBUG:-false} + - HOST=0.0.0.0 + - PORT=8000 + - DEPLOYMENT_MODE=minimal + - ENABLE_KNOWLEDGE_RAG=false + - ENABLE_AUTO_EXTRACTION=false + - ENABLE_MEMORY_SEARCH=false - NEO4J_URI=bolt://neo4j:7687 - - NEO4J_USER=neo4j - - NEO4J_PASSWORD=password123 - - NEO4J_DATABASE=neo4j - - # LLM Provider (ollama, openai, gemini, openrouter) - - LLM_PROVIDER=ollama - - EMBEDDING_PROVIDER=ollama - - # Ollama Configuration (if using ollama) - - OLLAMA_BASE_URL=http://ollama:11434 - - OLLAMA_MODEL=llama3.2 - - OLLAMA_EMBEDDING_MODEL=nomic-embed-text - - # OpenAI Configuration (if using openai) - # - OPENAI_API_KEY=your-key-here - # - OPENAI_MODEL=gpt-4 - # - OPENAI_EMBEDDING_MODEL=text-embedding-3-small - - # Gemini Configuration (if using gemini) - # - GOOGLE_API_KEY=your-key-here - # - GEMINI_MODEL=gemini-pro - # - GEMINI_EMBEDDING_MODEL=models/embedding-001 - - # Application Configuration - - APP_NAME=Code Graph Knowledge System - - APP_VERSION=0.5.0 - - LOG_LEVEL=INFO - - ENABLE_MONITORING=true - - # Timeouts - - CONNECTION_TIMEOUT=30 - - OPERATION_TIMEOUT=300 - - LARGE_DOCUMENT_TIMEOUT=600 - - # Chunking - - CHUNK_SIZE=512 - - CHUNK_OVERLAP=50 - - # Search - - TOP_K=10 - - VECTOR_DIMENSION=384 + - NEO4J_USER=${NEO4J_USER:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-password} + - NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + - CONNECTION_TIMEOUT=${CONNECTION_TIMEOUT:-30} + - OPERATION_TIMEOUT=${OPERATION_TIMEOUT:-120} volumes: + - ${REPOS_PATH:-./repos}:/repos - ./data:/data - - /tmp/repos:/tmp/repos - ./logs:/app/logs depends_on: neo4j: @@ -113,7 +76,7 @@ services: retries: 3 start_period: 40s networks: - - codebase-rag-network + - codebase-rag restart: unless-stopped volumes: @@ -125,9 +88,7 @@ volumes: driver: local neo4j_plugins: driver: local - ollama_data: - driver: local networks: - codebase-rag-network: + codebase-rag: driver: bridge diff --git a/docker/.env.template/.env.full b/docker/.env.template/.env.full new file mode 100644 index 0000000..992c768 --- /dev/null +++ b/docker/.env.template/.env.full @@ -0,0 +1,79 @@ +# Full Deployment Configuration +# All Features - LLM + Embedding required + +# Application +APP_NAME=Code Graph Knowledge System +DEBUG=false +APP_PORT=8000 +ENABLE_MONITORING=true + +# Neo4j Configuration +NEO4J_HTTP_PORT=7474 +NEO4J_BOLT_PORT=7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=your_secure_password_here +NEO4J_DATABASE=neo4j + +# Repository path +REPOS_PATH=./repos + +# LLM Provider (required for full features) +# Options: ollama, openai, gemini, openrouter +LLM_PROVIDER=ollama +EMBEDDING_PROVIDER=ollama + +# ============================================ +# Ollama Configuration (local or docker) +# ============================================ +# Use this if running Ollama in Docker (with-ollama profile): +OLLAMA_BASE_URL=http://ollama:11434 +# Use this if running Ollama on your host machine (outside Docker): +# OLLAMA_BASE_URL=http://host.docker.internal:11434 + +OLLAMA_PORT=11434 +OLLAMA_MODEL=llama3.2 +OLLAMA_EMBEDDING_MODEL=nomic-embed-text + +# ============================================ +# OpenAI Configuration (alternative) +# ============================================ +# LLM_PROVIDER=openai +# EMBEDDING_PROVIDER=openai +# OPENAI_API_KEY=your_openai_api_key_here +# OPENAI_MODEL=gpt-4 +# OPENAI_EMBEDDING_MODEL=text-embedding-3-small +# OPENAI_BASE_URL=https://api.openai.com/v1 + +# ============================================ +# Google Gemini Configuration (alternative) +# ============================================ +# LLM_PROVIDER=gemini +# EMBEDDING_PROVIDER=gemini +# GOOGLE_API_KEY=your_google_api_key_here +# GEMINI_MODEL=gemini-pro +# GEMINI_EMBEDDING_MODEL=models/embedding-001 + +# ============================================ +# OpenRouter Configuration (alternative) +# ============================================ +# LLM_PROVIDER=openrouter +# EMBEDDING_PROVIDER=openrouter +# OPENROUTER_API_KEY=your_openrouter_api_key_here +# OPENROUTER_MODEL=openai/gpt-3.5-turbo +# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 +# OPENROUTER_MAX_TOKENS=2048 + +# Model Parameters +TEMPERATURE=0.1 +MAX_TOKENS=2048 + +# RAG Settings +CHUNK_SIZE=512 +CHUNK_OVERLAP=50 +TOP_K=10 +VECTOR_DIMENSION=384 + +# Timeouts (in seconds) +CONNECTION_TIMEOUT=30 +OPERATION_TIMEOUT=300 +LARGE_DOCUMENT_TIMEOUT=600 diff --git a/docker/.env.template/.env.minimal b/docker/.env.template/.env.minimal new file mode 100644 index 0000000..0ee172e --- /dev/null +++ b/docker/.env.template/.env.minimal @@ -0,0 +1,23 @@ +# Minimal Deployment Configuration +# Code Graph Only - No LLM or Embedding required + +# Application +APP_NAME=Code Graph Knowledge System +DEBUG=false +APP_PORT=8000 + +# Neo4j Configuration +NEO4J_HTTP_PORT=7474 +NEO4J_BOLT_PORT=7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=your_secure_password_here +NEO4J_DATABASE=neo4j + +# Repository path (local path to mount) +REPOS_PATH=./repos + +# Timeouts (in seconds) +CONNECTION_TIMEOUT=30 +OPERATION_TIMEOUT=120 + +# Note: Minimal mode does not require LLM or Embedding configuration diff --git a/docker/.env.template/.env.standard b/docker/.env.template/.env.standard new file mode 100644 index 0000000..4d2546d --- /dev/null +++ b/docker/.env.template/.env.standard @@ -0,0 +1,42 @@ +# Standard Deployment Configuration +# Code Graph + Memory Store - Embedding required + +# Application +APP_NAME=Code Graph Knowledge System +DEBUG=false +APP_PORT=8000 + +# Neo4j Configuration +NEO4J_HTTP_PORT=7474 +NEO4J_BOLT_PORT=7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=your_secure_password_here +NEO4J_DATABASE=neo4j + +# Repository path +REPOS_PATH=./repos + +# Embedding Provider (required for memory vector search) +# Options: ollama, openai, gemini, huggingface +EMBEDDING_PROVIDER=ollama + +# Ollama Configuration (if using local Ollama on host) +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_EMBEDDING_MODEL=nomic-embed-text + +# OpenAI Configuration (alternative) +# EMBEDDING_PROVIDER=openai +# OPENAI_API_KEY=your_openai_api_key_here +# OPENAI_EMBEDDING_MODEL=text-embedding-3-small + +# Gemini Configuration (alternative) +# EMBEDDING_PROVIDER=gemini +# GOOGLE_API_KEY=your_google_api_key_here +# GEMINI_EMBEDDING_MODEL=models/embedding-001 + +# Vector Settings +VECTOR_DIMENSION=384 + +# Timeouts +CONNECTION_TIMEOUT=30 +OPERATION_TIMEOUT=120 diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 0000000..262d191 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,71 @@ +# Base Docker image for Code Graph Knowledge System +# Multi-stage build for optimized image size + +FROM python:3.13-slim as builder + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + git \ + curl \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install uv for faster dependency management +RUN pip install uv + +# Set work directory +WORKDIR /app + +# Copy dependency files +COPY pyproject.toml ./ +COPY README.md ./ + +# Install Python dependencies +RUN uv pip install --system -e . + +# ============================================ +# Final stage +# ============================================ +FROM python:3.13-slim + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PATH="/app:${PATH}" + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Create non-root user +RUN useradd -m -u 1000 appuser && \ + mkdir -p /app /data /repos && \ + chown -R appuser:appuser /app /data /repos + +# Set work directory +WORKDIR /app + +# Copy Python packages from builder +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +# Copy application code +COPY --chown=appuser:appuser . . + +# Switch to non-root user +USER appuser + +# Expose port +EXPOSE 8000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8000/api/v1/health || exit 1 diff --git a/docker/Dockerfile.full b/docker/Dockerfile.full new file mode 100644 index 0000000..04d8962 --- /dev/null +++ b/docker/Dockerfile.full @@ -0,0 +1,63 @@ +# Full Docker image - All features (LLM + Embedding required) +FROM python:3.13-slim as builder + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install uv + +WORKDIR /app + +# Copy source files needed for package installation +COPY pyproject.toml ./ +COPY api ./api +COPY core ./core +COPY services ./services +COPY monitoring ./monitoring +COPY mcp_tools ./mcp_tools +COPY start.py start_mcp.py mcp_server.py config.py main.py ./ + +# Install the package and its dependencies +RUN uv pip install --system . + +# ============================================ +# Final stage +# ============================================ +FROM python:3.13-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + DEPLOYMENT_MODE=full + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -u 1000 appuser && \ + mkdir -p /app /data /repos && \ + chown -R appuser:appuser /app /data /repos + +WORKDIR /app + +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +COPY --chown=appuser:appuser . . + +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8000/api/v1/health || exit 1 + +# Start in full mode +CMD ["python", "start_mcp.py", "--mode=full"] diff --git a/docker/Dockerfile.minimal b/docker/Dockerfile.minimal new file mode 100644 index 0000000..bc39713 --- /dev/null +++ b/docker/Dockerfile.minimal @@ -0,0 +1,64 @@ +# Minimal Docker image - Code Graph only (No LLM required) +FROM python:3.13-slim as builder + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + git \ + curl \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install uv + +WORKDIR /app + +# Copy source files needed for package installation +COPY pyproject.toml ./ +COPY api ./api +COPY core ./core +COPY services ./services +COPY monitoring ./monitoring +COPY mcp_tools ./mcp_tools +COPY start.py start_mcp.py mcp_server.py config.py main.py ./ + +# Install the package and its dependencies +RUN uv pip install --system . + +# ============================================ +# Final stage +# ============================================ +FROM python:3.13-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + DEPLOYMENT_MODE=minimal + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -u 1000 appuser && \ + mkdir -p /app /data /repos && \ + chown -R appuser:appuser /app /data /repos + +WORKDIR /app + +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +COPY --chown=appuser:appuser . . + +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8000/api/v1/health || exit 1 + +# Start in minimal mode +CMD ["python", "start_mcp.py", "--mode=minimal"] diff --git a/docker/Dockerfile.standard b/docker/Dockerfile.standard new file mode 100644 index 0000000..d7e6ba7 --- /dev/null +++ b/docker/Dockerfile.standard @@ -0,0 +1,63 @@ +# Standard Docker image - Code Graph + Memory (Embedding required) +FROM python:3.13-slim as builder + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install uv + +WORKDIR /app + +# Copy source files needed for package installation +COPY pyproject.toml ./ +COPY api ./api +COPY core ./core +COPY services ./services +COPY monitoring ./monitoring +COPY mcp_tools ./mcp_tools +COPY start.py start_mcp.py mcp_server.py config.py main.py ./ + +# Install the package and its dependencies +RUN uv pip install --system . + +# ============================================ +# Final stage +# ============================================ +FROM python:3.13-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + DEPLOYMENT_MODE=standard + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -u 1000 appuser && \ + mkdir -p /app /data /repos && \ + chown -R appuser:appuser /app /data /repos + +WORKDIR /app + +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +COPY --chown=appuser:appuser . . + +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8000/api/v1/health || exit 1 + +# Start in standard mode +CMD ["python", "start_mcp.py", "--mode=standard"] diff --git a/docker/docker-compose.full.yml b/docker/docker-compose.full.yml new file mode 100644 index 0000000..ceeb807 --- /dev/null +++ b/docker/docker-compose.full.yml @@ -0,0 +1,160 @@ +version: '3.8' + +# Full deployment - All features +# LLM + Embedding required + +services: + # Neo4j Database + neo4j: + image: neo4j:5.15-community + container_name: codebase-rag-neo4j-full + ports: + - "${NEO4J_HTTP_PORT:-7474}:7474" + - "${NEO4J_BOLT_PORT:-7687}:7687" + environment: + - NEO4J_AUTH=${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-password} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.* + - NEO4J_dbms_security_procedures_allowlist=apoc.* + - NEO4J_dbms_memory_heap_initial__size=512m + - NEO4J_dbms_memory_heap_max__size=2G + - NEO4J_dbms_memory_pagecache_size=512m + volumes: + - neo4j_full_data:/data + - neo4j_full_logs:/logs + - neo4j_full_import:/var/lib/neo4j/import + - neo4j_full_plugins:/plugins + healthcheck: + test: ["CMD-SHELL", "cypher-shell -u ${NEO4J_USER:-neo4j} -p ${NEO4J_PASSWORD:-password} 'RETURN 1' || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + networks: + - codebase-rag-full + restart: unless-stopped + + # Ollama (optional, for local LLM) + ollama: + image: ollama/ollama:latest + container_name: codebase-rag-ollama-full + ports: + - "${OLLAMA_PORT:-11434}:11434" + volumes: + - ollama_full_data:/root/.ollama + environment: + - OLLAMA_HOST=0.0.0.0 + networks: + - codebase-rag-full + restart: unless-stopped + profiles: + - with-ollama + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + # MCP Server - Full mode + mcp: + build: + context: .. + dockerfile: docker/Dockerfile.full + image: royisme/codebase-rag:full + container_name: codebase-rag-mcp-full + ports: + - "${APP_PORT:-8000}:8000" + environment: + # Application + - APP_NAME=${APP_NAME:-Code Graph Knowledge System} + - DEBUG=${DEBUG:-false} + - HOST=0.0.0.0 + - PORT=8000 + + # Deployment mode + - DEPLOYMENT_MODE=full + - ENABLE_KNOWLEDGE_RAG=true + - ENABLE_AUTO_EXTRACTION=true + - ENABLE_MEMORY_SEARCH=true + - ENABLE_MONITORING=${ENABLE_MONITORING:-true} + + # Neo4j Configuration + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USER=${NEO4J_USER:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-password} + - NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + + # LLM Provider + - LLM_PROVIDER=${LLM_PROVIDER:-ollama} + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-ollama} + + # Ollama Configuration (for with-ollama profile) + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://ollama:11434} + - OLLAMA_MODEL=${OLLAMA_MODEL:-llama3.2} + - OLLAMA_EMBEDDING_MODEL=${OLLAMA_EMBEDDING_MODEL:-nomic-embed-text} + + # OpenAI Configuration (alternative) + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - OPENAI_MODEL=${OPENAI_MODEL:-gpt-4} + - OPENAI_EMBEDDING_MODEL=${OPENAI_EMBEDDING_MODEL:-text-embedding-3-small} + + # Gemini Configuration (alternative) + - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} + - GEMINI_MODEL=${GEMINI_MODEL:-gemini-pro} + - GEMINI_EMBEDDING_MODEL=${GEMINI_EMBEDDING_MODEL:-models/embedding-001} + + # OpenRouter Configuration (alternative) + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} + - OPENROUTER_MODEL=${OPENROUTER_MODEL:-openai/gpt-3.5-turbo} + - OPENROUTER_BASE_URL=${OPENROUTER_BASE_URL:-https://openrouter.ai/api/v1} + + # Model Parameters + - TEMPERATURE=${TEMPERATURE:-0.1} + - MAX_TOKENS=${MAX_TOKENS:-2048} + + # RAG Settings + - CHUNK_SIZE=${CHUNK_SIZE:-512} + - CHUNK_OVERLAP=${CHUNK_OVERLAP:-50} + - TOP_K=${TOP_K:-10} + - VECTOR_DIMENSION=${VECTOR_DIMENSION:-384} + + # Timeouts + - CONNECTION_TIMEOUT=${CONNECTION_TIMEOUT:-30} + - OPERATION_TIMEOUT=${OPERATION_TIMEOUT:-300} + - LARGE_DOCUMENT_TIMEOUT=${LARGE_DOCUMENT_TIMEOUT:-600} + volumes: + - ${REPOS_PATH:-./repos}:/repos + - ./data:/data + - ./logs:/app/logs + depends_on: + neo4j: + condition: service_healthy + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - codebase-rag-full + restart: unless-stopped + +volumes: + neo4j_full_data: + driver: local + neo4j_full_logs: + driver: local + neo4j_full_import: + driver: local + neo4j_full_plugins: + driver: local + ollama_full_data: + driver: local + +networks: + codebase-rag-full: + driver: bridge diff --git a/docker/docker-compose.minimal.yml b/docker/docker-compose.minimal.yml new file mode 100644 index 0000000..90df7a1 --- /dev/null +++ b/docker/docker-compose.minimal.yml @@ -0,0 +1,97 @@ +version: '3.8' + +# Minimal deployment - Code Graph only +# No LLM or Embedding required + +services: + # Neo4j Database + neo4j: + image: neo4j:5.15-community + container_name: codebase-rag-neo4j-minimal + ports: + - "${NEO4J_HTTP_PORT:-7474}:7474" # HTTP + - "${NEO4J_BOLT_PORT:-7687}:7687" # Bolt + environment: + - NEO4J_AUTH=${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-password} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.* + - NEO4J_dbms_security_procedures_allowlist=apoc.* + - NEO4J_dbms_memory_heap_initial__size=512m + - NEO4J_dbms_memory_heap_max__size=2G + - NEO4J_dbms_memory_pagecache_size=512m + volumes: + - neo4j_minimal_data:/data + - neo4j_minimal_logs:/logs + - neo4j_minimal_import:/var/lib/neo4j/import + - neo4j_minimal_plugins:/plugins + healthcheck: + test: ["CMD-SHELL", "cypher-shell -u ${NEO4J_USER:-neo4j} -p ${NEO4J_PASSWORD:-password} 'RETURN 1' || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + networks: + - codebase-rag-minimal + restart: unless-stopped + + # MCP Server - Minimal mode + mcp: + build: + context: .. + dockerfile: docker/Dockerfile.minimal + image: royisme/codebase-rag:minimal + container_name: codebase-rag-mcp-minimal + ports: + - "${APP_PORT:-8000}:8000" + environment: + # Application + - APP_NAME=${APP_NAME:-Code Graph Knowledge System} + - DEBUG=${DEBUG:-false} + - HOST=0.0.0.0 + - PORT=8000 + + # Deployment mode + - DEPLOYMENT_MODE=minimal + - ENABLE_KNOWLEDGE_RAG=false + - ENABLE_AUTO_EXTRACTION=false + - ENABLE_MEMORY_SEARCH=false + + # Neo4j Configuration + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USER=${NEO4J_USER:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-password} + - NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + + # Timeouts + - CONNECTION_TIMEOUT=${CONNECTION_TIMEOUT:-30} + - OPERATION_TIMEOUT=${OPERATION_TIMEOUT:-120} + volumes: + - ${REPOS_PATH:-./repos}:/repos # Mount repository path + - ./data:/data + - ./logs:/app/logs + depends_on: + neo4j: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - codebase-rag-minimal + restart: unless-stopped + +volumes: + neo4j_minimal_data: + driver: local + neo4j_minimal_logs: + driver: local + neo4j_minimal_import: + driver: local + neo4j_minimal_plugins: + driver: local + +networks: + codebase-rag-minimal: + driver: bridge diff --git a/docker/docker-compose.standard.yml b/docker/docker-compose.standard.yml new file mode 100644 index 0000000..9cebbd0 --- /dev/null +++ b/docker/docker-compose.standard.yml @@ -0,0 +1,111 @@ +version: '3.8' + +# Standard deployment - Code Graph + Memory Store +# Embedding required, LLM optional + +services: + # Neo4j Database + neo4j: + image: neo4j:5.15-community + container_name: codebase-rag-neo4j-standard + ports: + - "${NEO4J_HTTP_PORT:-7474}:7474" + - "${NEO4J_BOLT_PORT:-7687}:7687" + environment: + - NEO4J_AUTH=${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-password} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.* + - NEO4J_dbms_security_procedures_allowlist=apoc.* + - NEO4J_dbms_memory_heap_initial__size=512m + - NEO4J_dbms_memory_heap_max__size=2G + - NEO4J_dbms_memory_pagecache_size=512m + volumes: + - neo4j_standard_data:/data + - neo4j_standard_logs:/logs + - neo4j_standard_import:/var/lib/neo4j/import + - neo4j_standard_plugins:/plugins + healthcheck: + test: ["CMD-SHELL", "cypher-shell -u ${NEO4J_USER:-neo4j} -p ${NEO4J_PASSWORD:-password} 'RETURN 1' || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + networks: + - codebase-rag-standard + restart: unless-stopped + + # MCP Server - Standard mode + mcp: + build: + context: .. + dockerfile: docker/Dockerfile.standard + image: royisme/codebase-rag:standard + container_name: codebase-rag-mcp-standard + ports: + - "${APP_PORT:-8000}:8000" + environment: + # Application + - APP_NAME=${APP_NAME:-Code Graph Knowledge System} + - DEBUG=${DEBUG:-false} + - HOST=0.0.0.0 + - PORT=8000 + + # Deployment mode + - DEPLOYMENT_MODE=standard + - ENABLE_KNOWLEDGE_RAG=false + - ENABLE_AUTO_EXTRACTION=false + - ENABLE_MEMORY_SEARCH=true + + # Neo4j Configuration + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USER=${NEO4J_USER:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-password} + - NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + + # Embedding Provider (required for memory search) + - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-ollama} + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} + - OLLAMA_EMBEDDING_MODEL=${OLLAMA_EMBEDDING_MODEL:-nomic-embed-text} + + # OpenAI Embedding (alternative) + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - OPENAI_EMBEDDING_MODEL=${OPENAI_EMBEDDING_MODEL:-text-embedding-3-small} + + # Timeouts + - CONNECTION_TIMEOUT=${CONNECTION_TIMEOUT:-30} + - OPERATION_TIMEOUT=${OPERATION_TIMEOUT:-120} + + # Vector settings + - VECTOR_DIMENSION=${VECTOR_DIMENSION:-384} + volumes: + - ${REPOS_PATH:-./repos}:/repos + - ./data:/data + - ./logs:/app/logs + depends_on: + neo4j: + condition: service_healthy + extra_hosts: + - "host.docker.internal:host-gateway" # Access host services (e.g., Ollama on host) + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - codebase-rag-standard + restart: unless-stopped + +volumes: + neo4j_standard_data: + driver: local + neo4j_standard_logs: + driver: local + neo4j_standard_import: + driver: local + neo4j_standard_plugins: + driver: local + +networks: + codebase-rag-standard: + driver: bridge diff --git a/docs/CNAME b/docs/CNAME new file mode 100644 index 0000000..0e88ad8 --- /dev/null +++ b/docs/CNAME @@ -0,0 +1 @@ +code-graph.vantagecraft.dev diff --git a/docs/api/mcp-tools.md b/docs/api/mcp-tools.md new file mode 100644 index 0000000..dac4011 --- /dev/null +++ b/docs/api/mcp-tools.md @@ -0,0 +1,1457 @@ +# MCP Tools Reference + +Complete reference for all 30 Model Context Protocol (MCP) tools available in the Code Graph Knowledge System. + +**MCP Server Version**: 2.0.0 +**MCP Protocol Version**: 1.1.0 +**Total Tools**: 30 + +## Overview + +The MCP server provides AI assistants (like Claude Desktop, VS Code with MCP, etc.) with direct access to the Code Graph Knowledge System through the official Model Context Protocol SDK. + +**Key Features**: +- 30 specialized tools across 6 categories +- Session management for tracking context +- Streaming support for long-running operations +- Multi-transport capability (stdio, SSE, WebSocket) +- Standard MCP protocol compliance + +**Architecture**: +- Main server: `mcp_server.py` (310 lines - modular design) +- Tool handlers: `mcp_tools/` package (organized by category) +- Official SDK: `mcp>=1.1.0` + +--- + +## Installation & Setup + +### Start MCP Server + +```bash +# Using start script +python start_mcp.py + +# Using uv (recommended) +uv run mcp_client +``` + +### Claude Desktop Configuration + +Add to `~/Library/Application Support/Claude/claude_desktop_config.json`: + +```json +{ + "mcpServers": { + "code-graph": { + "command": "python", + "args": ["/path/to/codebase-rag/start_mcp.py"], + "env": { + "NEO4J_URI": "bolt://localhost:7687", + "NEO4J_USER": "neo4j", + "NEO4J_PASSWORD": "your-password" + } + } + } +} +``` + +### VS Code MCP Extension + +Configure in `.vscode/mcp.json`: + +```json +{ + "servers": { + "code-graph": { + "command": "python /path/to/codebase-rag/start_mcp.py" + } + } +} +``` + +--- + +## Tool Categories + +| Category | Tools | Description | +|----------|-------|-------------| +| **Knowledge Base** | 5 | Query and manage knowledge graph | +| **Code Graph** | 4 | Repository analysis and context | +| **Memory Store** | 7 | Project knowledge persistence | +| **Memory Extraction** | 5 | Automatic memory extraction (v0.7) | +| **Task Management** | 6 | Async task monitoring | +| **System** | 3 | Schema and statistics | + +--- + +## Knowledge Base Tools (5) + +Tools for querying and managing the knowledge graph. + +### 1. query_knowledge + +Query the knowledge base using Neo4j GraphRAG. + +**Input Parameters**: +```typescript +{ + question: string; // Required: Question to ask + mode?: "hybrid" | "graph_only" | "vector_only"; // Default: "hybrid" +} +``` + +**Query Modes**: +- `hybrid` (recommended): Graph traversal + vector search +- `graph_only`: Use only graph relationships +- `vector_only`: Use only vector similarity + +**Example**: +```json +{ + "question": "How does authentication work in this system?", + "mode": "hybrid" +} +``` + +**Response**: +```json +{ + "success": true, + "answer": "The system uses JWT-based authentication with refresh tokens...", + "source_nodes": [ + { + "text": "JWT implementation details...", + "score": 0.92, + "metadata": {"title": "Auth Guide"} + } + ], + "mode": "hybrid" +} +``` + +--- + +### 2. search_similar_nodes + +Search for similar nodes using vector similarity. + +**Input Parameters**: +```typescript +{ + query: string; // Required: Search query + top_k?: number; // Default: 10, Range: 1-50 +} +``` + +**Example**: +```json +{ + "query": "database configuration", + "top_k": 10 +} +``` + +**Response**: +```json +{ + "success": true, + "results": [ + { + "text": "Database connection settings...", + "score": 0.89, + "metadata": {"title": "Config Guide"} + } + ] +} +``` + +--- + +### 3. add_document + +Add a document to the knowledge base. + +**Input Parameters**: +```typescript +{ + content: string; // Required: Document content + title?: string; // Optional: Document title + metadata?: object; // Optional: Additional metadata +} +``` + +**Size Handling**: +- **Small documents (<10KB)**: Processed synchronously +- **Large documents (>=10KB)**: Processed asynchronously with task_id + +**Example**: +```json +{ + "content": "This is the document content with important information...", + "title": "Architecture Guide", + "metadata": { + "author": "Team", + "tags": ["architecture", "design"] + } +} +``` + +**Response (Small)**: +```json +{ + "success": true, + "message": "Document added successfully", + "document_id": "doc-abc123", + "chunks_created": 5 +} +``` + +**Response (Large)**: +```json +{ + "success": true, + "task_id": "task-xyz789", + "message": "Document processing queued", + "processing_async": true +} +``` + +--- + +### 4. add_file + +Add a file to the knowledge base. + +**Input Parameters**: +```typescript +{ + file_path: string; // Required: Absolute path to file +} +``` + +**Supported file types**: Text files, code files, markdown, etc. + +**Example**: +```json +{ + "file_path": "/absolute/path/to/document.md" +} +``` + +**Response**: +```json +{ + "success": true, + "message": "File added successfully", + "file_path": "/absolute/path/to/document.md", + "chunks_created": 8 +} +``` + +--- + +### 5. add_directory + +Add all files from a directory to the knowledge base. + +**Input Parameters**: +```typescript +{ + directory_path: string; // Required: Absolute directory path + recursive?: boolean; // Default: true +} +``` + +**Example**: +```json +{ + "directory_path": "/absolute/path/to/docs", + "recursive": true +} +``` + +**Response**: +```json +{ + "success": true, + "message": "Directory processed", + "files_processed": 23, + "total_chunks": 156 +} +``` + +--- + +## Code Graph Tools (4) + +Tools for repository analysis and code understanding. + +### 1. code_graph_ingest_repo + +Ingest a code repository into the graph database. + +**Input Parameters**: +```typescript +{ + local_path: string; // Required: Local repository path + repo_url?: string; // Optional: Repository URL + mode?: "full" | "incremental"; // Default: "incremental" +} +``` + +**Ingestion Modes**: +- `full`: Complete re-ingestion (slow but thorough) +- `incremental`: Only changed files (60x faster) + +**Extracts**: +- File nodes +- Symbol nodes (functions, classes) +- IMPORTS relationships +- Code structure + +**Example**: +```json +{ + "local_path": "/path/to/repository", + "mode": "incremental" +} +``` + +**Response**: +```json +{ + "success": true, + "task_id": "ing-20250115-103045-abc12345", + "status": "done", + "message": "Successfully ingested 125 files", + "files_processed": 125, + "mode": "incremental" +} +``` + +--- + +### 2. code_graph_related + +Find files related to a query using fulltext search. + +**Input Parameters**: +```typescript +{ + query: string; // Required: Search query + repo_id: string; // Required: Repository identifier + limit?: number; // Default: 30, Range: 1-100 +} +``` + +**Example**: +```json +{ + "query": "authentication jwt token", + "repo_id": "myproject", + "limit": 30 +} +``` + +**Response**: +```json +{ + "nodes": [ + { + "type": "file", + "ref": "ref://file/src/auth/jwt.py", + "path": "src/auth/jwt.py", + "lang": "python", + "score": 0.92, + "summary": "JWT authentication implementation with token generation" + } + ], + "query": "authentication jwt token", + "repo_id": "myproject" +} +``` + +--- + +### 3. code_graph_impact + +Analyze impact of changes to a file (reverse dependencies). + +**Input Parameters**: +```typescript +{ + repo_id: string; // Required: Repository identifier + file_path: string; // Required: File path to analyze + depth?: number; // Default: 2, Range: 1-5 +} +``` + +**Use Cases**: +- Understanding blast radius of changes +- Finding code that needs updating +- Identifying critical files with many dependents + +**Example**: +```json +{ + "repo_id": "myproject", + "file_path": "src/auth/jwt.py", + "depth": 2 +} +``` + +**Response**: +```json +{ + "nodes": [ + { + "type": "file", + "path": "src/api/auth_routes.py", + "lang": "python", + "repoId": "myproject", + "relationship": "IMPORTS", + "depth": 1, + "score": 0.85, + "ref": "ref://file/src/api/auth_routes.py", + "summary": "Auth API routes (imports jwt.py)" + } + ], + "file": "src/auth/jwt.py", + "repo_id": "myproject", + "depth": 2 +} +``` + +--- + +### 4. context_pack + +Build a context pack for AI agents within token budget. + +**Input Parameters**: +```typescript +{ + repo_id: string; // Required: Repository ID + stage?: "plan" | "review" | "implement"; // Default: "implement" + budget?: number; // Default: 1500, Range: 500-10000 + keywords?: string; // Optional: Focus keywords + focus?: string; // Optional: Focus file paths +} +``` + +**Stages**: +- `plan`: Project overview and high-level architecture +- `review`: Code review focus with detailed analysis +- `implement`: Implementation details and code snippets + +**Example**: +```json +{ + "repo_id": "myproject", + "stage": "implement", + "budget": 2000, + "keywords": "authentication, jwt, middleware" +} +``` + +**Response**: +```json +{ + "items": [ + { + "kind": "file", + "title": "src/auth/jwt.py", + "summary": "JWT authentication with token generation and validation", + "ref": "ref://file/src/auth/jwt.py", + "extra": {"lang": "python", "score": 0.92} + } + ], + "budget_used": 1850, + "budget_limit": 2000, + "stage": "implement", + "repo_id": "myproject" +} +``` + +--- + +## Memory Store Tools (7) + +Tools for project knowledge persistence and management. + +### 1. add_memory + +Add a new memory to project knowledge base. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + memory_type: MemoryType; // Required + title: string; // Required (max 200 chars) + content: string; // Required + reason?: string; // Optional: Rationale + tags?: string[]; // Optional: Tags + importance?: number; // Default: 0.5, Range: 0-1 + related_refs?: string[]; // Optional: ref:// handles +} + +type MemoryType = "decision" | "preference" | "experience" | "convention" | "plan" | "note"; +``` + +**Memory Types**: +- `decision`: Architecture choices, tech stack selection +- `preference`: Coding style, tool preferences +- `experience`: Problems encountered and solutions +- `convention`: Team rules, naming conventions +- `plan`: Future improvements, TODOs +- `note`: Other important information + +**Example**: +```json +{ + "project_id": "myapp", + "memory_type": "decision", + "title": "Use JWT for authentication", + "content": "Decided to use JWT tokens instead of session-based auth", + "reason": "Need stateless authentication for mobile clients", + "tags": ["auth", "architecture"], + "importance": 0.9, + "related_refs": ["ref://file/src/auth/jwt.py"] +} +``` + +**Response**: +```json +{ + "success": true, + "memory_id": "mem-abc123-def456", + "project_id": "myapp", + "message": "Memory added successfully" +} +``` + +--- + +### 2. search_memories + +Search project memories with filters. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + query?: string; // Optional: Search text + memory_type?: MemoryType; // Optional: Filter by type + tags?: string[]; // Optional: Filter by tags + min_importance?: number; // Default: 0.0, Range: 0-1 + limit?: number; // Default: 20, Range: 1-100 +} +``` + +**Example**: +```json +{ + "project_id": "myapp", + "query": "authentication security", + "memory_type": "decision", + "min_importance": 0.7, + "limit": 20 +} +``` + +**Response**: +```json +{ + "success": true, + "memories": [ + { + "memory_id": "mem-abc123", + "memory_type": "decision", + "title": "Use JWT for authentication", + "content": "Decided to use JWT tokens...", + "importance": 0.9, + "tags": ["auth", "architecture"], + "created_at": "2025-01-15T10:30:00Z" + } + ], + "total": 1 +} +``` + +--- + +### 3. get_memory + +Get a specific memory by ID with full details. + +**Input Parameters**: +```typescript +{ + memory_id: string; // Required +} +``` + +**Example**: +```json +{ + "memory_id": "mem-abc123-def456" +} +``` + +**Response**: +```json +{ + "success": true, + "memory": { + "memory_id": "mem-abc123", + "project_id": "myapp", + "memory_type": "decision", + "title": "Use JWT for authentication", + "content": "Decided to use JWT tokens...", + "reason": "Need stateless authentication...", + "tags": ["auth", "architecture"], + "importance": 0.9, + "related_refs": ["ref://file/src/auth/jwt.py"], + "created_at": "2025-01-15T10:30:00Z", + "updated_at": "2025-01-15T10:30:00Z", + "is_superseded": false + } +} +``` + +--- + +### 4. update_memory + +Update an existing memory (partial update supported). + +**Input Parameters**: +```typescript +{ + memory_id: string; // Required + title?: string; // Optional + content?: string; // Optional + reason?: string; // Optional + tags?: string[]; // Optional + importance?: number; // Optional: Range: 0-1 +} +``` + +**Example**: +```json +{ + "memory_id": "mem-abc123", + "importance": 0.95, + "tags": ["auth", "security", "critical"] +} +``` + +**Response**: +```json +{ + "success": true, + "message": "Memory updated successfully", + "memory_id": "mem-abc123" +} +``` + +--- + +### 5. delete_memory + +Delete a memory (soft delete - data retained). + +**Input Parameters**: +```typescript +{ + memory_id: string; // Required +} +``` + +**Example**: +```json +{ + "memory_id": "mem-abc123" +} +``` + +**Response**: +```json +{ + "success": true, + "message": "Memory deleted successfully", + "memory_id": "mem-abc123" +} +``` + +--- + +### 6. supersede_memory + +Create a new memory that supersedes an old one (preserves history). + +**Input Parameters**: +```typescript +{ + old_memory_id: string; // Required + new_memory_type: MemoryType; // Required + new_title: string; // Required + new_content: string; // Required + new_reason?: string; // Optional + new_tags?: string[]; // Optional + new_importance?: number; // Default: 0.5, Range: 0-1 +} +``` + +**Use Case**: When decisions change or better solutions are found. + +**Example**: +```json +{ + "old_memory_id": "mem-abc123", + "new_memory_type": "decision", + "new_title": "Use PostgreSQL instead of MySQL", + "new_content": "Switched to PostgreSQL for better JSON support", + "new_reason": "Need advanced JSON querying capabilities", + "new_importance": 0.8 +} +``` + +**Response**: +```json +{ + "success": true, + "old_memory_id": "mem-abc123", + "new_memory_id": "mem-xyz789", + "message": "Memory superseded successfully" +} +``` + +--- + +### 7. get_project_summary + +Get summary of all memories for a project, organized by type. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required +} +``` + +**Example**: +```json +{ + "project_id": "myapp" +} +``` + +**Response**: +```json +{ + "success": true, + "project_id": "myapp", + "total_memories": 42, + "by_type": { + "decision": { + "count": 12, + "top_memories": [ + { + "memory_id": "mem-abc123", + "title": "Use JWT for authentication", + "importance": 0.9 + } + ] + }, + "preference": {"count": 8}, + "experience": {"count": 15}, + "convention": {"count": 5}, + "plan": {"count": 2} + } +} +``` + +--- + +## Memory Extraction Tools (5) + +Automatic memory extraction from various sources (v0.7). + +### 1. extract_from_conversation + +Extract memories from conversation using LLM analysis. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + conversation: Array<{ // Required + role: string; + content: string; + }>; + auto_save?: boolean; // Default: false +} +``` + +**Auto-save**: If true, automatically saves memories with confidence >= 0.7 + +**Example**: +```json +{ + "project_id": "myapp", + "conversation": [ + {"role": "user", "content": "Should we use Redis or Memcached?"}, + {"role": "assistant", "content": "Let's use Redis because it supports data persistence"} + ], + "auto_save": false +} +``` + +**Response**: +```json +{ + "success": true, + "extracted_memories": [ + { + "memory_type": "decision", + "title": "Use Redis for caching", + "content": "Decided to use Redis instead of Memcached", + "reason": "Redis supports data persistence", + "confidence": 0.85, + "auto_saved": false, + "memory_id": null + } + ], + "total_extracted": 1, + "auto_saved_count": 0 +} +``` + +--- + +### 2. extract_from_git_commit + +Extract memories from git commit using LLM analysis. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + commit_sha: string; // Required + commit_message: string; // Required + changed_files: string[]; // Required + auto_save?: boolean; // Default: false +} +``` + +**Identifies**: +- Feature additions → `decision` +- Bug fixes → `experience` +- Refactoring → `experience`/`convention` +- Breaking changes → high importance `decision` + +**Example**: +```json +{ + "project_id": "myapp", + "commit_sha": "abc123def456", + "commit_message": "feat: add JWT authentication\n\nImplemented JWT-based auth", + "changed_files": ["src/auth/jwt.py", "src/middleware/auth.py"], + "auto_save": true +} +``` + +**Response**: +```json +{ + "success": true, + "extracted_memories": [ + { + "memory_type": "decision", + "title": "Implement JWT authentication", + "content": "Added JWT-based authentication system", + "confidence": 0.92, + "auto_saved": true, + "memory_id": "mem-xyz789" + } + ] +} +``` + +--- + +### 3. extract_from_code_comments + +Extract memories from code comments in source file. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + file_path: string; // Required: Path to source file +} +``` + +**Marker Mappings**: +- `TODO:` → `plan` +- `FIXME:` / `BUG:` → `experience` +- `NOTE:` / `IMPORTANT:` → `convention` +- `DECISION:` → `decision` + +**Example**: +```json +{ + "project_id": "myapp", + "file_path": "/path/to/project/src/service.py" +} +``` + +**Response**: +```json +{ + "success": true, + "extracted_memories": [ + { + "memory_type": "plan", + "title": "TODO: Add rate limiting", + "content": "Need to implement rate limiting for API endpoints", + "line_number": 45, + "auto_saved": true, + "memory_id": "mem-plan123" + } + ], + "total_extracted": 1 +} +``` + +--- + +### 4. suggest_memory_from_query + +Suggest creating memory from knowledge base query. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + query: string; // Required: User query + answer: string; // Required: LLM answer +} +``` + +**Use Cases**: +- Frequently asked questions +- Important architectural information +- Non-obvious solutions or workarounds + +**Example**: +```json +{ + "project_id": "myapp", + "query": "How does the authentication work?", + "answer": "The system uses JWT tokens with refresh token rotation..." +} +``` + +**Response**: +```json +{ + "success": true, + "should_save": true, + "confidence": 0.88, + "suggested_memory": { + "memory_type": "note", + "title": "Authentication mechanism", + "content": "System uses JWT with refresh token rotation", + "importance": 0.7 + } +} +``` + +--- + +### 5. batch_extract_from_repository + +Batch extract memories from entire repository. + +**Input Parameters**: +```typescript +{ + project_id: string; // Required + repo_path: string; // Required: Path to git repo + max_commits?: number; // Default: 50, Range: 1-200 + file_patterns?: string[]; // Optional: e.g., ["*.py", "*.js"] +} +``` + +**Analyzes**: +- Recent git commits (configurable count) +- Code comments in source files +- Documentation files (README, CHANGELOG, etc.) + +**Note**: Long-running operation (may take several minutes). + +**Example**: +```json +{ + "project_id": "myapp", + "repo_path": "/path/to/repository", + "max_commits": 50, + "file_patterns": ["*.py", "*.js"] +} +``` + +**Response**: +```json +{ + "success": true, + "summary": { + "commits_analyzed": 50, + "files_scanned": 125, + "total_extracted": 23, + "by_source": { + "git_commits": 12, + "code_comments": 11 + }, + "by_type": { + "decision": 5, + "experience": 8, + "plan": 10 + } + }, + "execution_time_seconds": 45.2 +} +``` + +--- + +## Task Management Tools (6) + +Tools for monitoring asynchronous task execution. + +### 1. get_task_status + +Get status of a specific task. + +**Input Parameters**: +```typescript +{ + task_id: string; // Required +} +``` + +**Example**: +```json +{ + "task_id": "task-abc123" +} +``` + +**Response**: +```json +{ + "success": true, + "task_id": "task-abc123", + "status": "SUCCESS", + "progress": 100.0, + "message": "Task completed successfully", + "result": { + "chunks_created": 15 + } +} +``` + +**Status Values**: `PENDING`, `PROCESSING`, `SUCCESS`, `FAILED`, `CANCELLED` + +--- + +### 2. watch_task + +Monitor a task in real-time until completion (with timeout). + +**Input Parameters**: +```typescript +{ + task_id: string; // Required + timeout?: number; // Default: 300, Range: 10-600 (seconds) + poll_interval?: number; // Default: 2, Range: 1-10 (seconds) +} +``` + +**Example**: +```json +{ + "task_id": "task-abc123", + "timeout": 300, + "poll_interval": 2 +} +``` + +**Response** (Streaming): +```json +{ + "success": true, + "task_id": "task-abc123", + "final_status": "SUCCESS", + "progress_history": [ + {"timestamp": "2025-01-15T10:30:00Z", "progress": 0.0, "status": "PENDING"}, + {"timestamp": "2025-01-15T10:30:05Z", "progress": 25.0, "status": "PROCESSING"}, + {"timestamp": "2025-01-15T10:30:10Z", "progress": 100.0, "status": "SUCCESS"} + ], + "result": {"chunks_created": 15} +} +``` + +--- + +### 3. watch_tasks + +Monitor multiple tasks until all complete. + +**Input Parameters**: +```typescript +{ + task_ids: string[]; // Required + timeout?: number; // Default: 300, Range: 10-600 + poll_interval?: number; // Default: 2, Range: 1-10 +} +``` + +**Example**: +```json +{ + "task_ids": ["task-abc123", "task-xyz789"], + "timeout": 300, + "poll_interval": 2 +} +``` + +**Response**: +```json +{ + "success": true, + "tasks": { + "task-abc123": { + "status": "SUCCESS", + "progress": 100.0, + "result": {"chunks_created": 15} + }, + "task-xyz789": { + "status": "SUCCESS", + "progress": 100.0, + "result": {"chunks_created": 22} + } + }, + "all_completed": true +} +``` + +--- + +### 4. list_tasks + +List tasks with optional status filter. + +**Input Parameters**: +```typescript +{ + status_filter?: "pending" | "running" | "completed" | "failed"; + limit?: number; // Default: 20, Range: 1-100 +} +``` + +**Example**: +```json +{ + "status_filter": "running", + "limit": 20 +} +``` + +**Response**: +```json +{ + "success": true, + "tasks": [ + { + "task_id": "task-abc123", + "status": "PROCESSING", + "progress": 45.0, + "created_at": "2025-01-15T10:30:00Z" + } + ], + "total": 2 +} +``` + +--- + +### 5. cancel_task + +Cancel a pending or running task. + +**Input Parameters**: +```typescript +{ + task_id: string; // Required +} +``` + +**Example**: +```json +{ + "task_id": "task-abc123" +} +``` + +**Response**: +```json +{ + "success": true, + "message": "Task cancelled successfully", + "task_id": "task-abc123" +} +``` + +--- + +### 6. get_queue_stats + +Get task queue statistics. + +**Input Parameters**: None + +**Example**: +```json +{} +``` + +**Response**: +```json +{ + "success": true, + "pending": 5, + "running": 2, + "completed": 142, + "failed": 6, + "total": 155, + "queue_active": true +} +``` + +--- + +## System Tools (3) + +System information and management tools. + +### 1. get_graph_schema + +Get Neo4j graph schema (node labels, relationship types, statistics). + +**Input Parameters**: None + +**Example**: +```json +{} +``` + +**Response**: +```json +{ + "success": true, + "node_labels": ["Document", "Chunk", "Entity", "Memory", "Project", "File", "Repo"], + "relationship_types": ["HAS_CHUNK", "MENTIONS", "RELATES_TO", "BELONGS_TO"], + "statistics": { + "node_count": 1523, + "relationship_count": 4567 + } +} +``` + +--- + +### 2. get_statistics + +Get knowledge base statistics. + +**Input Parameters**: None + +**Example**: +```json +{} +``` + +**Response**: +```json +{ + "success": true, + "total_nodes": 1523, + "total_relationships": 4567, + "document_count": 45, + "chunk_count": 892, + "entity_count": 586, + "memory_count": 42, + "file_count": 125 +} +``` + +--- + +### 3. clear_knowledge_base + +**⚠️ DANGEROUS**: Clear all data from knowledge base. + +**Input Parameters**: +```typescript +{ + confirmation: string; // Required: Must be "yes" +} +``` + +**Example**: +```json +{ + "confirmation": "yes" +} +``` + +**Response**: +```json +{ + "success": true, + "message": "Knowledge base cleared", + "nodes_deleted": 1523, + "relationships_deleted": 4567 +} +``` + +--- + +## Resources + +MCP resources provide dynamic data access. + +### Available Resources + +1. **knowledge://config** - System configuration and settings +2. **knowledge://status** - Current system status and health + +**Access via MCP**: Resources are accessed through the MCP protocol, not as tools. + +--- + +## Prompts + +MCP prompts provide query suggestions. + +### suggest_queries + +Generate suggested queries for the knowledge graph. + +**Arguments**: +- `domain`: Domain to focus on (general, code, documentation, sql, architecture) + +**Example Domains**: +- `general`: General system questions +- `code`: Code-specific queries +- `documentation`: Documentation queries +- `sql`: Database schema queries +- `architecture`: Architecture questions + +--- + +## Error Handling + +All tools follow consistent error response format. + +### Success Response + +```json +{ + "success": true, + "...": "tool-specific data" +} +``` + +### Error Response + +```json +{ + "success": false, + "error": "Detailed error message", + "error_type": "ValidationError | NotFoundError | ServiceError" +} +``` + +### Common Error Types + +**Validation Error**: +```json +{ + "success": false, + "error": "Invalid memory_type. Must be one of: decision, preference, experience, convention, plan, note" +} +``` + +**Not Found Error**: +```json +{ + "success": false, + "error": "Memory not found: mem-abc123" +} +``` + +**Service Error**: +```json +{ + "success": false, + "error": "Failed to connect to Neo4j database" +} +``` + +--- + +## Best Practices + +### Memory Management + +1. **Importance Scoring**: + - 0.9-1.0: Critical decisions, security findings + - 0.7-0.8: Important architectural choices + - 0.5-0.6: Preferences and conventions + - 0.3-0.4: Plans and future work + +2. **Tagging Strategy**: + - Use domain tags: `auth`, `database`, `api` + - Use type tags: `security`, `performance`, `bug` + - Use status tags: `critical`, `deprecated` + +3. **When to Use Extraction**: + - Use `extract_from_conversation` for Q&A sessions + - Use `extract_from_git_commit` for commit hooks + - Use `extract_from_code_comments` for code reviews + - Use `batch_extract_from_repository` for initial setup + +### Task Monitoring + +1. Use `watch_task` for single long-running operations +2. Use `watch_tasks` for batch operations +3. Set appropriate timeouts based on operation size +4. Use `cancel_task` to stop unnecessary work + +### Code Graph + +1. Use `incremental` mode for regular updates (60x faster) +2. Use `full` mode for initial ingestion or major changes +3. Use `context_pack` to stay within token limits +4. Use `impact` analysis before making changes + +--- + +**Last Updated**: 2025-01-15 +**MCP Server Version**: 2.0.0 +**Total Tools**: 30 diff --git a/docs/api/python-sdk.md b/docs/api/python-sdk.md new file mode 100644 index 0000000..47aaa52 --- /dev/null +++ b/docs/api/python-sdk.md @@ -0,0 +1,1466 @@ +# Python SDK Guide + +Complete guide for using Code Graph Knowledge System services directly in Python applications. + +**Version**: 1.0.0 + +## Table of Contents + +- [Overview](#overview) +- [Installation](#installation) +- [Core Services](#core-services) +- [Neo4jKnowledgeService](#neo4jknowledgeservice) +- [MemoryStore](#memorystore) +- [GraphService](#graphservice) +- [CodeIngestor](#codeingestor) +- [TaskQueue](#taskqueue) +- [Configuration](#configuration) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Best Practices](#best-practices) + +--- + +## Overview + +The Python SDK provides direct access to all system services without going through REST API or MCP. This is ideal for: + +- Building custom integrations +- Embedding knowledge graph capabilities in applications +- Batch processing scripts +- Custom AI agents +- Testing and development + +**Key Services**: +- `Neo4jKnowledgeService`: Knowledge graph and RAG +- `MemoryStore`: Project memory persistence +- `GraphService`: Low-level Neo4j operations +- `CodeIngestor`: Repository ingestion +- `TaskQueue`: Asynchronous task management + +--- + +## Installation + +### Requirements + +```bash +# Python 3.10+ +python --version + +# Install dependencies +pip install -e . + +# Or with uv (recommended) +uv pip install -e . +``` + +### Dependencies + +```python +# Core dependencies +neo4j>=5.0.0 +llama-index-core>=0.10.0 +llama-index-graph-stores-neo4j>=0.2.0 +fastapi>=0.104.0 +pydantic>=2.0.0 +``` + +### Environment Setup + +Create `.env` file: + +```bash +# Neo4j Configuration +NEO4J_URI=bolt://localhost:7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=your-password +NEO4J_DATABASE=neo4j + +# LLM Provider (ollama/openai/gemini/openrouter) +LLM_PROVIDER=ollama +OLLAMA_BASE_URL=http://localhost:11434 +OLLAMA_MODEL=llama2 + +# Embedding Provider +EMBEDDING_PROVIDER=ollama +EMBEDDING_MODEL=nomic-embed-text + +# Optional: OpenAI +OPENAI_API_KEY=sk-... +OPENAI_MODEL=gpt-4 + +# Optional: Google Gemini +GOOGLE_API_KEY=... +GEMINI_MODEL=gemini-pro + +# Optional: OpenRouter +OPENROUTER_API_KEY=... +OPENROUTER_MODEL=anthropic/claude-3-opus +``` + +--- + +## Core Services + +### Import Services + +```python +from services.neo4j_knowledge_service import Neo4jKnowledgeService +from services.memory_store import MemoryStore, memory_store +from services.graph_service import Neo4jGraphService, graph_service +from services.code_ingestor import CodeIngestor, get_code_ingestor +from services.task_queue import TaskQueue, task_queue +from config import settings +``` + +### Service Initialization Pattern + +All services follow async initialization: + +```python +import asyncio + +async def main(): + # Create service instance + service = Neo4jKnowledgeService() + + # Initialize (connect to Neo4j, setup LLM, etc.) + success = await service.initialize() + + if not success: + print("Failed to initialize service") + return + + # Use service + result = await service.query("How does this work?") + print(result) + +asyncio.run(main()) +``` + +--- + +## Neo4jKnowledgeService + +Primary service for knowledge graph operations with LlamaIndex integration. + +### Initialization + +```python +from services.neo4j_knowledge_service import Neo4jKnowledgeService + +# Create instance +knowledge_service = Neo4jKnowledgeService() + +# Initialize (async) +await knowledge_service.initialize() +``` + +### Key Methods + +#### query() + +Query knowledge base using GraphRAG. + +```python +async def query( + question: str, + mode: str = "hybrid" +) -> Dict[str, Any]: + """ + Query knowledge base. + + Args: + question: Question to ask + mode: "hybrid" | "graph_only" | "vector_only" + + Returns: + { + "success": bool, + "answer": str, + "source_nodes": List[Dict], + "mode": str + } + """ +``` + +**Example**: +```python +result = await knowledge_service.query( + question="How does authentication work?", + mode="hybrid" +) + +if result["success"]: + print(f"Answer: {result['answer']}") + print(f"Sources: {len(result['source_nodes'])}") +``` + +#### search_similar_nodes() + +Vector similarity search. + +```python +async def search_similar_nodes( + query: str, + top_k: int = 10 +) -> Dict[str, Any]: + """ + Search similar nodes using vector similarity. + + Args: + query: Search query + top_k: Number of results (1-50) + + Returns: + { + "success": bool, + "results": List[Dict], + "query": str + } + """ +``` + +**Example**: +```python +result = await knowledge_service.search_similar_nodes( + query="database configuration", + top_k=10 +) + +for node in result["results"]: + print(f"Score: {node['score']:.2f} - {node['text'][:100]}") +``` + +#### add_document() + +Add document to knowledge base. + +```python +async def add_document( + content: str, + title: str = "Untitled", + metadata: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + """ + Add document to knowledge base. + + Args: + content: Document content + title: Document title + metadata: Additional metadata + + Returns: + { + "success": bool, + "document_id": str, + "chunks_created": int + } + """ +``` + +**Example**: +```python +result = await knowledge_service.add_document( + content=""" + Authentication System Design + + The system uses JWT tokens for stateless authentication. + Refresh tokens are stored in Redis with 7-day expiration. + """, + title="Auth Design", + metadata={ + "author": "Team", + "tags": ["auth", "design"] + } +) + +print(f"Document ID: {result['document_id']}") +print(f"Chunks created: {result['chunks_created']}") +``` + +#### add_file() + +Add file to knowledge base. + +```python +async def add_file( + file_path: str +) -> Dict[str, Any]: + """ + Add file to knowledge base. + + Args: + file_path: Absolute path to file + + Returns: + { + "success": bool, + "file_path": str, + "chunks_created": int + } + """ +``` + +**Example**: +```python +result = await knowledge_service.add_file( + file_path="/path/to/documentation.md" +) +``` + +#### add_directory() + +Add directory of files to knowledge base. + +```python +async def add_directory( + directory_path: str, + recursive: bool = True, + file_extensions: Optional[List[str]] = None +) -> Dict[str, Any]: + """ + Add directory to knowledge base. + + Args: + directory_path: Absolute directory path + recursive: Process subdirectories + file_extensions: File patterns (e.g., [".md", ".txt"]) + + Returns: + { + "success": bool, + "files_processed": int, + "total_chunks": int + } + """ +``` + +**Example**: +```python +result = await knowledge_service.add_directory( + directory_path="/path/to/docs", + recursive=True, + file_extensions=[".md", ".txt"] +) + +print(f"Processed {result['files_processed']} files") +``` + +#### get_graph_schema() + +Get graph schema information. + +```python +async def get_graph_schema() -> Dict[str, Any]: + """ + Get Neo4j graph schema. + + Returns: + { + "success": bool, + "node_labels": List[str], + "relationship_types": List[str], + "statistics": Dict + } + """ +``` + +#### get_statistics() + +Get knowledge base statistics. + +```python +async def get_statistics() -> Dict[str, Any]: + """ + Get knowledge base statistics. + + Returns: + { + "success": bool, + "total_nodes": int, + "total_relationships": int, + "document_count": int, + "chunk_count": int + } + """ +``` + +#### clear_knowledge_base() + +**⚠️ DANGEROUS**: Clear all knowledge base data. + +```python +async def clear_knowledge_base() -> Dict[str, Any]: + """Clear all data from knowledge base.""" +``` + +--- + +## MemoryStore + +Project memory persistence for AI agents. + +### Initialization + +```python +from services.memory_store import memory_store + +# Initialize (async) +await memory_store.initialize() +``` + +### Key Methods + +#### add_memory() + +Add a new memory. + +```python +async def add_memory( + project_id: str, + memory_type: str, # "decision" | "preference" | "experience" | "convention" | "plan" | "note" + title: str, + content: str, + reason: Optional[str] = None, + tags: Optional[List[str]] = None, + importance: float = 0.5, + related_refs: Optional[List[str]] = None +) -> Dict[str, Any]: + """ + Add memory to project. + + Returns: + { + "success": bool, + "memory_id": str, + "project_id": str + } + """ +``` + +**Example**: +```python +result = await memory_store.add_memory( + project_id="myapp", + memory_type="decision", + title="Use JWT for authentication", + content="Decided to use JWT tokens instead of session-based auth", + reason="Need stateless authentication for mobile clients", + tags=["auth", "architecture"], + importance=0.9, + related_refs=["ref://file/src/auth/jwt.py"] +) + +memory_id = result["memory_id"] +``` + +#### search_memories() + +Search memories with filters. + +```python +async def search_memories( + project_id: str, + query: Optional[str] = None, + memory_type: Optional[str] = None, + tags: Optional[List[str]] = None, + min_importance: float = 0.0, + limit: int = 20 +) -> Dict[str, Any]: + """ + Search project memories. + + Returns: + { + "success": bool, + "memories": List[Dict], + "total": int + } + """ +``` + +**Example**: +```python +result = await memory_store.search_memories( + project_id="myapp", + query="authentication security", + memory_type="decision", + min_importance=0.7, + limit=20 +) + +for memory in result["memories"]: + print(f"{memory['title']} (importance: {memory['importance']})") +``` + +#### get_memory() + +Get specific memory by ID. + +```python +async def get_memory( + memory_id: str +) -> Dict[str, Any]: + """ + Get memory by ID. + + Returns: + { + "success": bool, + "memory": Dict # Full memory details + } + """ +``` + +#### update_memory() + +Update existing memory. + +```python +async def update_memory( + memory_id: str, + title: Optional[str] = None, + content: Optional[str] = None, + reason: Optional[str] = None, + tags: Optional[List[str]] = None, + importance: Optional[float] = None +) -> Dict[str, Any]: + """Update memory (partial update supported).""" +``` + +**Example**: +```python +await memory_store.update_memory( + memory_id=memory_id, + importance=0.95, + tags=["auth", "security", "critical"] +) +``` + +#### delete_memory() + +Delete memory (soft delete). + +```python +async def delete_memory( + memory_id: str +) -> Dict[str, Any]: + """Delete memory (soft delete - data retained).""" +``` + +#### supersede_memory() + +Create new memory that supersedes old one. + +```python +async def supersede_memory( + old_memory_id: str, + new_memory_data: Dict[str, Any] +) -> Dict[str, Any]: + """ + Create memory that supersedes old one. + + Args: + old_memory_id: ID of memory to supersede + new_memory_data: Data for new memory + { + "memory_type": str, + "title": str, + "content": str, + "reason": str, + "tags": List[str], + "importance": float + } + + Returns: + { + "success": bool, + "old_memory_id": str, + "new_memory_id": str + } + """ +``` + +**Example**: +```python +result = await memory_store.supersede_memory( + old_memory_id="mem-abc123", + new_memory_data={ + "memory_type": "decision", + "title": "Use PostgreSQL instead of MySQL", + "content": "Switched to PostgreSQL for better JSON support", + "reason": "Need advanced JSON querying capabilities", + "importance": 0.8 + } +) +``` + +#### get_project_summary() + +Get project memory summary. + +```python +async def get_project_summary( + project_id: str +) -> Dict[str, Any]: + """ + Get project memory summary. + + Returns: + { + "success": bool, + "project_id": str, + "total_memories": int, + "by_type": Dict # Breakdown by memory type + } + """ +``` + +--- + +## GraphService + +Low-level Neo4j graph operations. + +### Initialization + +```python +from services.graph_service import graph_service + +# Connect to Neo4j +await graph_service.connect() +``` + +### Key Methods + +#### execute_cypher() + +Execute Cypher query. + +```python +def execute_cypher( + query: str, + parameters: Optional[Dict[str, Any]] = None +) -> GraphQueryResult: + """ + Execute Cypher query. + + Args: + query: Cypher query string + parameters: Query parameters + + Returns: + GraphQueryResult with nodes, relationships, paths + """ +``` + +**Example**: +```python +result = graph_service.execute_cypher( + query=""" + MATCH (n:Memory {project_id: $project_id}) + WHERE n.importance > $min_importance + RETURN n + LIMIT 10 + """, + parameters={ + "project_id": "myapp", + "min_importance": 0.7 + } +) + +for node in result.nodes: + print(f"Node: {node.properties['title']}") +``` + +#### create_node() + +Create a node. + +```python +def create_node( + labels: List[str], + properties: Dict[str, Any] +) -> str: + """ + Create node. + + Args: + labels: Node labels + properties: Node properties + + Returns: + Node ID + """ +``` + +**Example**: +```python +node_id = graph_service.create_node( + labels=["CustomNode", "Entity"], + properties={ + "name": "Example", + "value": 42, + "created_at": datetime.utcnow().isoformat() + } +) +``` + +#### create_relationship() + +Create a relationship. + +```python +def create_relationship( + start_node_id: str, + end_node_id: str, + relationship_type: str, + properties: Optional[Dict[str, Any]] = None +) -> str: + """Create relationship between nodes.""" +``` + +#### fulltext_search() + +Perform fulltext search on files. + +```python +def fulltext_search( + query_text: str, + repo_id: str, + limit: int = 50 +) -> List[Dict[str, Any]]: + """ + Fulltext search on files. + + Args: + query_text: Search query + repo_id: Repository ID + limit: Max results + + Returns: + List of file matches with paths and languages + """ +``` + +**Example**: +```python +files = graph_service.fulltext_search( + query_text="authentication jwt", + repo_id="myproject", + limit=30 +) + +for file in files: + print(f"{file['path']} ({file['lang']})") +``` + +#### impact_analysis() + +Analyze impact of file changes. + +```python +def impact_analysis( + repo_id: str, + file_path: str, + depth: int = 2, + limit: int = 50 +) -> List[Dict[str, Any]]: + """ + Analyze file impact (reverse dependencies). + + Args: + repo_id: Repository ID + file_path: File path to analyze + depth: Traversal depth + limit: Max results + + Returns: + List of dependent files + """ +``` + +--- + +## CodeIngestor + +Repository code ingestion service. + +### Initialization + +```python +from services.code_ingestor import get_code_ingestor +from services.graph_service import graph_service + +# Initialize graph service first +await graph_service.connect() + +# Get code ingestor +code_ingestor = get_code_ingestor(graph_service) +``` + +### Key Methods + +#### scan_files() + +Scan repository files. + +```python +def scan_files( + repo_path: str, + include_globs: List[str], + exclude_globs: List[str] +) -> List[Dict[str, Any]]: + """ + Scan files in repository. + + Args: + repo_path: Repository path + include_globs: Include patterns (e.g., ["**/*.py"]) + exclude_globs: Exclude patterns (e.g., ["**/node_modules/**"]) + + Returns: + List of file information dictionaries + """ +``` + +**Example**: +```python +files = code_ingestor.scan_files( + repo_path="/path/to/repository", + include_globs=["**/*.py", "**/*.ts", "**/*.tsx"], + exclude_globs=["**/node_modules/**", "**/.git/**", "**/__pycache__/**"] +) + +print(f"Found {len(files)} files") +``` + +#### ingest_files() + +Ingest files into Neo4j graph. + +```python +def ingest_files( + repo_id: str, + files: List[Dict[str, Any]] +) -> Dict[str, Any]: + """ + Ingest files into Neo4j. + + Args: + repo_id: Repository identifier + files: List of file info from scan_files() + + Returns: + { + "success": bool, + "files_processed": int, + "nodes_created": int + } + """ +``` + +**Example**: +```python +result = code_ingestor.ingest_files( + repo_id="myproject", + files=files +) + +print(f"Processed {result['files_processed']} files") +print(f"Created {result['nodes_created']} nodes") +``` + +--- + +## TaskQueue + +Asynchronous task queue management. + +### Initialization + +```python +from services.task_queue import task_queue, TaskStatus + +# Start task queue +await task_queue.start() +``` + +### Key Methods + +#### submit_task() + +Submit a task to the queue. + +```python +async def submit_task( + task_func: Callable, + task_kwargs: Dict[str, Any], + task_name: str, + task_type: str, + metadata: Optional[Dict[str, Any]] = None, + priority: int = 0 +) -> str: + """ + Submit task to queue. + + Args: + task_func: Function to execute + task_kwargs: Function arguments + task_name: Task name + task_type: Task type + metadata: Additional metadata + priority: Priority (higher = more important) + + Returns: + Task ID + """ +``` + +**Example**: +```python +from services.task_processors import process_document_task + +task_id = await task_queue.submit_task( + task_func=process_document_task, + task_kwargs={ + "document_content": "Large document content...", + "title": "Large Doc" + }, + task_name="Process Large Document", + task_type="document_processing", + metadata={"source": "api"}, + priority=5 +) + +print(f"Task submitted: {task_id}") +``` + +#### get_task_status() + +Get task status. + +```python +def get_task_status( + task_id: str +) -> Optional[TaskResult]: + """ + Get task status. + + Returns: + TaskResult or None if not found + """ +``` + +**Example**: +```python +task_result = task_queue.get_task_status(task_id) + +if task_result: + print(f"Status: {task_result.status.value}") + print(f"Progress: {task_result.progress}%") + + if task_result.status == TaskStatus.SUCCESS: + print(f"Result: {task_result.result}") + elif task_result.status == TaskStatus.FAILED: + print(f"Error: {task_result.error}") +``` + +#### cancel_task() + +Cancel a task. + +```python +async def cancel_task( + task_id: str +) -> bool: + """Cancel task. Returns True if cancelled.""" +``` + +#### get_queue_stats() + +Get queue statistics. + +```python +def get_queue_stats() -> Dict[str, int]: + """ + Get queue statistics. + + Returns: + { + "pending": int, + "running": int, + "completed": int, + "failed": int + } + """ +``` + +--- + +## Configuration + +Access configuration settings. + +```python +from config import settings + +# Neo4j settings +print(settings.neo4j_uri) +print(settings.neo4j_database) + +# LLM settings +print(settings.llm_provider) +print(settings.ollama_model) +print(settings.temperature) + +# Embedding settings +print(settings.embedding_provider) +print(settings.embedding_model) + +# Timeouts +print(settings.connection_timeout) +print(settings.operation_timeout) +print(settings.large_document_timeout) + +# Chunk settings +print(settings.chunk_size) +print(settings.chunk_overlap) +print(settings.top_k) +``` + +### Get Current Model Info + +```python +from config import get_current_model_info + +model_info = get_current_model_info() +print(f"LLM: {model_info['llm']}") +print(f"Embedding: {model_info['embedding']}") +``` + +--- + +## Examples + +### Complete Knowledge Base Example + +```python +import asyncio +from services.neo4j_knowledge_service import Neo4jKnowledgeService + +async def main(): + # Initialize service + service = Neo4jKnowledgeService() + await service.initialize() + + # Add documents + await service.add_document( + content="JWT authentication guide...", + title="Auth Guide", + metadata={"tags": ["auth"]} + ) + + # Query + result = await service.query( + question="How does authentication work?", + mode="hybrid" + ) + + print(f"Answer: {result['answer']}") + + # Search + search_results = await service.search_similar_nodes( + query="authentication", + top_k=5 + ) + + for node in search_results["results"]: + print(f"- {node['text'][:100]}") + +asyncio.run(main()) +``` + +### Memory Management Example + +```python +import asyncio +from services.memory_store import memory_store + +async def main(): + # Initialize + await memory_store.initialize() + + # Add decision + result = await memory_store.add_memory( + project_id="myapp", + memory_type="decision", + title="Use Redis for caching", + content="Decided to use Redis instead of Memcached", + reason="Redis supports data persistence", + importance=0.8, + tags=["cache", "architecture"] + ) + + memory_id = result["memory_id"] + + # Search memories + search_result = await memory_store.search_memories( + project_id="myapp", + query="caching", + min_importance=0.5 + ) + + for memory in search_result["memories"]: + print(f"{memory['title']}: {memory['content']}") + + # Get project summary + summary = await memory_store.get_project_summary("myapp") + print(f"Total memories: {summary['total_memories']}") + print(f"By type: {summary['by_type']}") + +asyncio.run(main()) +``` + +### Repository Ingestion Example + +```python +import asyncio +from services.graph_service import graph_service +from services.code_ingestor import get_code_ingestor +from services.git_utils import git_utils + +async def main(): + # Connect to Neo4j + await graph_service.connect() + + # Get code ingestor + code_ingestor = get_code_ingestor(graph_service) + + # Get repository ID + repo_path = "/path/to/repository" + repo_id = git_utils.get_repo_id_from_path(repo_path) + + # Scan files + files = code_ingestor.scan_files( + repo_path=repo_path, + include_globs=["**/*.py", "**/*.ts"], + exclude_globs=["**/node_modules/**", "**/.git/**"] + ) + + print(f"Found {len(files)} files") + + # Ingest into Neo4j + result = code_ingestor.ingest_files( + repo_id=repo_id, + files=files + ) + + print(f"Success: {result['success']}") + print(f"Files processed: {result['files_processed']}") + + # Search code + search_results = graph_service.fulltext_search( + query_text="authentication", + repo_id=repo_id, + limit=10 + ) + + for file in search_results: + print(f"- {file['path']} ({file['lang']})") + +asyncio.run(main()) +``` + +### Task Queue Example + +```python +import asyncio +from services.task_queue import task_queue, TaskStatus +from services.task_processors import process_document_task + +async def main(): + # Start task queue + await task_queue.start() + + # Submit task + task_id = await task_queue.submit_task( + task_func=process_document_task, + task_kwargs={ + "document_content": "Large document content...", + "title": "Large Doc" + }, + task_name="Process Large Document", + task_type="document_processing" + ) + + print(f"Task submitted: {task_id}") + + # Monitor task + while True: + task_result = task_queue.get_task_status(task_id) + + if not task_result: + break + + print(f"Status: {task_result.status.value}, Progress: {task_result.progress}%") + + if task_result.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]: + break + + await asyncio.sleep(2) + + if task_result.status == TaskStatus.SUCCESS: + print(f"Result: {task_result.result}") + else: + print(f"Error: {task_result.error}") + + # Get queue stats + stats = task_queue.get_queue_stats() + print(f"Queue stats: {stats}") + +asyncio.run(main()) +``` + +--- + +## Error Handling + +All services return structured results with error information. + +### Standard Response Format + +```python +# Success +{ + "success": True, + "...": "service-specific data" +} + +# Error +{ + "success": False, + "error": "Error message" +} +``` + +### Handling Errors + +```python +result = await knowledge_service.query("question") + +if not result.get("success"): + error_msg = result.get("error", "Unknown error") + print(f"Error: {error_msg}") + # Handle error +else: + # Process result + answer = result["answer"] +``` + +### Exception Handling + +```python +try: + await knowledge_service.initialize() +except Exception as e: + logger.error(f"Failed to initialize: {e}") + # Handle exception +``` + +--- + +## Best Practices + +### 1. Always Initialize Services + +```python +# Good +service = Neo4jKnowledgeService() +await service.initialize() + +# Bad - will fail +service = Neo4jKnowledgeService() +await service.query("question") # Error: not initialized +``` + +### 2. Check Success Status + +```python +# Good +result = await service.query("question") +if result["success"]: + print(result["answer"]) +else: + print(f"Error: {result['error']}") + +# Bad +result = await service.query("question") +print(result["answer"]) # May crash if error occurred +``` + +### 3. Use Context Managers for Neo4j Sessions + +```python +# Good +async with graph_service.driver.session() as session: + result = await session.run("MATCH (n) RETURN n LIMIT 10") + # Session automatically closed + +# Bad +session = graph_service.driver.session() +result = await session.run("MATCH (n) RETURN n LIMIT 10") +# Session not closed - memory leak +``` + +### 4. Set Appropriate Timeouts + +```python +from config import settings + +# Adjust timeouts for large operations +settings.operation_timeout = 300 # 5 minutes +settings.large_document_timeout = 600 # 10 minutes + +service = Neo4jKnowledgeService() +await service.initialize() +``` + +### 5. Handle Large Documents Asynchronously + +```python +# For large documents, use task queue +if len(document_content) > 10_000: + task_id = await task_queue.submit_task( + task_func=process_document_task, + task_kwargs={"document_content": document_content}, + task_name="Process Large Doc", + task_type="document_processing" + ) + # Monitor task_id +else: + # Process directly + await knowledge_service.add_document(content=document_content) +``` + +### 6. Batch Operations + +```python +# Good - batch insert +files = code_ingestor.scan_files(repo_path, include_globs, exclude_globs) +result = code_ingestor.ingest_files(repo_id, files) + +# Bad - individual inserts +for file in files: + code_ingestor.ingest_files(repo_id, [file]) # Slow! +``` + +### 7. Use Memory Store for Long-term Knowledge + +```python +# Store important decisions +await memory_store.add_memory( + project_id="myapp", + memory_type="decision", + title="Architecture decision", + content="Detailed rationale...", + importance=0.9 # High importance +) + +# Search when needed +memories = await memory_store.search_memories( + project_id="myapp", + memory_type="decision", + min_importance=0.7 +) +``` + +### 8. Clean Up Resources + +```python +# Close connections when done +await graph_service.driver.close() + +# Or use application lifecycle hooks +async def startup(): + await knowledge_service.initialize() + await memory_store.initialize() + await task_queue.start() + +async def shutdown(): + await graph_service.driver.close() + await task_queue.stop() +``` + +--- + +## Performance Tips + +### 1. Connection Pooling + +Neo4j driver handles connection pooling automatically. Reuse service instances: + +```python +# Good - single instance +service = Neo4jKnowledgeService() +await service.initialize() + +for question in questions: + await service.query(question) + +# Bad - multiple instances +for question in questions: + service = Neo4jKnowledgeService() + await service.initialize() # Expensive! + await service.query(question) +``` + +### 2. Batch Queries + +```python +# Good - batch cypher query +query = """ +UNWIND $items as item +CREATE (n:Node {name: item.name, value: item.value}) +""" +graph_service.execute_cypher(query, {"items": items}) + +# Bad - individual queries +for item in items: + graph_service.execute_cypher( + "CREATE (n:Node {name: $name, value: $value})", + item + ) +``` + +### 3. Use Incremental Repository Ingestion + +```python +# 60x faster for updates +from services.git_utils import git_utils + +if git_utils.is_git_repo(repo_path): + changed_files = git_utils.get_changed_files(repo_path) + files_to_process = filter_by_patterns(changed_files) +else: + # Fall back to full scan + files_to_process = code_ingestor.scan_files(repo_path, ...) +``` + +### 4. Limit Result Sets + +```python +# Always use limits for large datasets +result = await knowledge_service.search_similar_nodes( + query="search term", + top_k=10 # Limit results +) +``` + +--- + +**Last Updated**: 2025-01-15 +**SDK Version**: 1.0.0 +**Python Version**: 3.10+ diff --git a/docs/api/rest.md b/docs/api/rest.md new file mode 100644 index 0000000..3184249 --- /dev/null +++ b/docs/api/rest.md @@ -0,0 +1,1283 @@ +# REST API Reference + +Complete reference for Code Graph Knowledge System REST API endpoints. + +**Base URL**: `http://localhost:8000/api/v1` + +**Version**: 1.0.0 + +## Authentication + +Currently, the API does not require authentication. This may be added in future versions. + +--- + +## Health & System + +### Get Health Status + +Get system health and service status. + +**Endpoint**: `GET /health` + +**Response**: +```json +{ + "status": "healthy", + "services": { + "neo4j_knowledge_service": true, + "graph_service": true, + "task_queue": true + }, + "version": "0.1.0" +} +``` + +### Get System Configuration + +Get current system configuration. + +**Endpoint**: `GET /config` + +**Response**: +```json +{ + "app_name": "Code Graph Knowledge System", + "version": "0.1.0", + "debug": false, + "llm_provider": "ollama", + "embedding_provider": "ollama", + "monitoring_enabled": true +} +``` + +### Get Graph Schema + +Get Neo4j graph schema information. + +**Endpoint**: `GET /schema` + +**Response**: +```json +{ + "success": true, + "node_labels": ["Document", "Chunk", "Entity", "Memory", "Project", "File", "Repo"], + "relationship_types": ["HAS_CHUNK", "MENTIONS", "RELATES_TO", "BELONGS_TO"], + "statistics": { + "node_count": 1523, + "relationship_count": 4567 + } +} +``` + +### Get Statistics + +Get knowledge base statistics. + +**Endpoint**: `GET /statistics` + +**Response**: +```json +{ + "success": true, + "total_nodes": 1523, + "total_relationships": 4567, + "document_count": 45, + "chunk_count": 892, + "entity_count": 586 +} +``` + +### Get Prometheus Metrics + +Get system metrics in Prometheus format. + +**Endpoint**: `GET /metrics` + +**Response**: Plain text Prometheus metrics +``` +# HELP http_requests_total Total HTTP requests +# TYPE http_requests_total counter +http_requests_total{method="GET",endpoint="/health"} 1234 + +# HELP neo4j_nodes_total Total nodes in Neo4j +# TYPE neo4j_nodes_total gauge +neo4j_nodes_total 1523 +``` + +### Clear Knowledge Base + +**⚠️ DANGEROUS**: Clear all data from knowledge base. + +**Endpoint**: `DELETE /clear` + +**Response**: +```json +{ + "success": true, + "message": "Knowledge base cleared", + "nodes_deleted": 1523, + "relationships_deleted": 4567 +} +``` + +--- + +## Knowledge Base + +### Query Knowledge Base + +Query the knowledge base using GraphRAG. + +**Endpoint**: `POST /knowledge/query` + +**Request Body**: +```json +{ + "question": "How does authentication work in this system?", + "mode": "hybrid" +} +``` + +**Parameters**: +- `question` (string, required): Question to ask +- `mode` (string, optional): Query mode + - `hybrid` (default): Graph traversal + vector search + - `graph_only`: Only graph relationships + - `vector_only`: Only vector similarity + +**Response**: +```json +{ + "success": true, + "answer": "The system uses JWT-based authentication...", + "source_nodes": [ + { + "text": "JWT implementation details...", + "score": 0.92, + "metadata": { + "title": "Authentication Guide", + "source": "docs/auth.md" + } + } + ], + "mode": "hybrid" +} +``` + +### Search Similar Nodes + +Search for similar nodes using vector similarity. + +**Endpoint**: `POST /knowledge/search` + +**Request Body**: +```json +{ + "query": "database configuration", + "top_k": 10 +} +``` + +**Parameters**: +- `query` (string, required): Search query +- `top_k` (integer, optional): Number of results (default: 10, max: 50) + +**Response**: +```json +{ + "success": true, + "results": [ + { + "text": "Database connection settings...", + "score": 0.89, + "metadata": { + "title": "Configuration Guide", + "type": "document" + } + } + ], + "query": "database configuration", + "top_k": 10 +} +``` + +### Add Document + +Add a document to knowledge base. + +**Endpoint**: `POST /documents` + +**Request Body**: +```json +{ + "content": "This is the document content...", + "title": "My Document", + "metadata": { + "author": "John Doe", + "tags": ["tutorial", "api"] + } +} +``` + +**Parameters**: +- `content` (string, required): Document content +- `title` (string, optional): Document title +- `metadata` (object, optional): Additional metadata + +**Response**: +```json +{ + "success": true, + "message": "Document added successfully", + "document_id": "doc-abc123", + "chunks_created": 5 +} +``` + +**Note**: Large documents (>10KB) are processed asynchronously and return a task_id. + +### Add File + +Add a file to knowledge base. + +**Endpoint**: `POST /documents/file` + +**Request Body**: +```json +{ + "file_path": "/absolute/path/to/file.txt" +} +``` + +**Parameters**: +- `file_path` (string, required): Absolute path to file + +**Response**: +```json +{ + "success": true, + "message": "File added successfully", + "file_path": "/absolute/path/to/file.txt", + "chunks_created": 8 +} +``` + +### Add Directory + +Add all files from directory to knowledge base. + +**Endpoint**: `POST /documents/directory` + +**Request Body**: +```json +{ + "directory_path": "/absolute/path/to/directory", + "recursive": true, + "file_patterns": ["*.md", "*.txt"] +} +``` + +**Parameters**: +- `directory_path` (string, required): Absolute directory path +- `recursive` (boolean, optional): Process subdirectories (default: true) +- `file_patterns` (array, optional): File patterns to include + +**Response**: +```json +{ + "success": true, + "message": "Directory processed", + "files_processed": 23, + "total_chunks": 156 +} +``` + +--- + +## Memory Management + +Memory Store provides project knowledge persistence for AI agents. + +### Add Memory + +Add a new memory to project knowledge base. + +**Endpoint**: `POST /memory/add` + +**Request Body**: +```json +{ + "project_id": "myapp", + "memory_type": "decision", + "title": "Use JWT for authentication", + "content": "Decided to use JWT tokens instead of session-based auth", + "reason": "Need stateless authentication for mobile clients", + "tags": ["auth", "architecture"], + "importance": 0.9, + "related_refs": ["ref://file/src/auth/jwt.py"] +} +``` + +**Parameters**: +- `project_id` (string, required): Project identifier +- `memory_type` (string, required): Type of memory + - `decision`: Architecture choices, tech stack + - `preference`: Coding style, tool preferences + - `experience`: Problems and solutions + - `convention`: Team rules, naming patterns + - `plan`: Future improvements, TODOs + - `note`: Other important information +- `title` (string, required): Short title (max 200 chars) +- `content` (string, required): Detailed content +- `reason` (string, optional): Rationale or explanation +- `tags` (array, optional): Tags for categorization +- `importance` (number, optional): Importance score 0-1 (default: 0.5) +- `related_refs` (array, optional): Related ref:// handles + +**Response**: +```json +{ + "success": true, + "memory_id": "mem-abc123-def456", + "project_id": "myapp", + "message": "Memory added successfully" +} +``` + +### Search Memories + +Search project memories with filters. + +**Endpoint**: `POST /memory/search` + +**Request Body**: +```json +{ + "project_id": "myapp", + "query": "authentication", + "memory_type": "decision", + "tags": ["auth"], + "min_importance": 0.7, + "limit": 20 +} +``` + +**Parameters**: +- `project_id` (string, required): Project identifier +- `query` (string, optional): Search query text +- `memory_type` (string, optional): Filter by memory type +- `tags` (array, optional): Filter by tags +- `min_importance` (number, optional): Minimum importance (default: 0.0) +- `limit` (integer, optional): Max results (default: 20, max: 100) + +**Response**: +```json +{ + "success": true, + "memories": [ + { + "memory_id": "mem-abc123", + "memory_type": "decision", + "title": "Use JWT for authentication", + "content": "Decided to use JWT tokens...", + "reason": "Need stateless authentication...", + "tags": ["auth", "architecture"], + "importance": 0.9, + "created_at": "2025-01-15T10:30:00Z", + "updated_at": "2025-01-15T10:30:00Z" + } + ], + "total": 1, + "query": "authentication" +} +``` + +### Get Memory + +Get a specific memory by ID. + +**Endpoint**: `GET /memory/{memory_id}` + +**Response**: +```json +{ + "success": true, + "memory": { + "memory_id": "mem-abc123", + "project_id": "myapp", + "memory_type": "decision", + "title": "Use JWT for authentication", + "content": "Decided to use JWT tokens...", + "reason": "Need stateless authentication...", + "tags": ["auth", "architecture"], + "importance": 0.9, + "related_refs": ["ref://file/src/auth/jwt.py"], + "created_at": "2025-01-15T10:30:00Z", + "updated_at": "2025-01-15T10:30:00Z", + "is_superseded": false, + "superseded_by": null + } +} +``` + +### Update Memory + +Update an existing memory. + +**Endpoint**: `PUT /memory/{memory_id}` + +**Request Body**: +```json +{ + "title": "Updated title", + "importance": 0.95, + "tags": ["auth", "security", "critical"] +} +``` + +**Parameters**: All fields are optional, only provided fields will be updated +- `title` (string): Update title +- `content` (string): Update content +- `reason` (string): Update reason +- `tags` (array): Update tags +- `importance` (number): Update importance + +**Response**: +```json +{ + "success": true, + "message": "Memory updated successfully", + "memory_id": "mem-abc123" +} +``` + +### Delete Memory + +Delete a memory (soft delete). + +**Endpoint**: `DELETE /memory/{memory_id}` + +**Response**: +```json +{ + "success": true, + "message": "Memory deleted successfully", + "memory_id": "mem-abc123" +} +``` + +### Supersede Memory + +Create a new memory that supersedes an old one. + +**Endpoint**: `POST /memory/supersede` + +**Request Body**: +```json +{ + "old_memory_id": "mem-abc123", + "new_memory_type": "decision", + "new_title": "Use PostgreSQL instead of MySQL", + "new_content": "Switched to PostgreSQL for better JSON support", + "new_reason": "Need advanced JSON querying capabilities", + "new_tags": ["database", "architecture"], + "new_importance": 0.8 +} +``` + +**Response**: +```json +{ + "success": true, + "old_memory_id": "mem-abc123", + "new_memory_id": "mem-xyz789", + "message": "Memory superseded successfully" +} +``` + +### Get Project Summary + +Get summary of all memories for a project. + +**Endpoint**: `GET /memory/project/{project_id}/summary` + +**Response**: +```json +{ + "success": true, + "project_id": "myapp", + "total_memories": 42, + "by_type": { + "decision": { + "count": 12, + "top_memories": [ + { + "memory_id": "mem-abc123", + "title": "Use JWT for authentication", + "importance": 0.9, + "created_at": "2025-01-15T10:30:00Z" + } + ] + }, + "preference": {"count": 8, "top_memories": []}, + "experience": {"count": 15, "top_memories": []}, + "convention": {"count": 5, "top_memories": []}, + "plan": {"count": 2, "top_memories": []} + } +} +``` + +### Memory Health Check + +Check memory store health. + +**Endpoint**: `GET /memory/health` + +**Response**: +```json +{ + "service": "memory_store", + "status": "healthy", + "initialized": true, + "extraction_enabled": true +} +``` + +--- + +## Memory Extraction (v0.7) + +Automatic memory extraction from various sources. + +### Extract from Conversation + +Extract memories from conversation using LLM analysis. + +**Endpoint**: `POST /memory/extract/conversation` + +**Request Body**: +```json +{ + "project_id": "myapp", + "conversation": [ + {"role": "user", "content": "Should we use Redis or Memcached?"}, + {"role": "assistant", "content": "Let's use Redis because it supports data persistence"} + ], + "auto_save": false +} +``` + +**Parameters**: +- `project_id` (string, required): Project identifier +- `conversation` (array, required): Conversation messages +- `auto_save` (boolean, optional): Auto-save high-confidence memories (default: false) + +**Response**: +```json +{ + "success": true, + "extracted_memories": [ + { + "memory_type": "decision", + "title": "Use Redis for caching", + "content": "Decided to use Redis instead of Memcached", + "reason": "Redis supports data persistence", + "confidence": 0.85, + "auto_saved": false, + "memory_id": null + } + ], + "total_extracted": 1, + "auto_saved_count": 0 +} +``` + +### Extract from Git Commit + +Extract memories from git commit using LLM analysis. + +**Endpoint**: `POST /memory/extract/commit` + +**Request Body**: +```json +{ + "project_id": "myapp", + "commit_sha": "abc123def456", + "commit_message": "feat: add JWT authentication\n\nImplemented JWT-based auth for stateless API", + "changed_files": ["src/auth/jwt.py", "src/middleware/auth.py"], + "auto_save": true +} +``` + +**Parameters**: +- `project_id` (string, required): Project identifier +- `commit_sha` (string, required): Git commit SHA +- `commit_message` (string, required): Full commit message +- `changed_files` (array, required): List of changed file paths +- `auto_save` (boolean, optional): Auto-save high-confidence memories + +**Response**: +```json +{ + "success": true, + "extracted_memories": [ + { + "memory_type": "decision", + "title": "Implement JWT authentication", + "content": "Added JWT-based authentication system", + "confidence": 0.92, + "auto_saved": true, + "memory_id": "mem-xyz789" + } + ] +} +``` + +### Extract from Code Comments + +Extract memories from code comments in source file. + +**Endpoint**: `POST /memory/extract/comments` + +**Request Body**: +```json +{ + "project_id": "myapp", + "file_path": "/path/to/project/src/service.py" +} +``` + +**Response**: +```json +{ + "success": true, + "extracted_memories": [ + { + "memory_type": "plan", + "title": "TODO: Add rate limiting", + "content": "Need to implement rate limiting for API endpoints", + "line_number": 45, + "auto_saved": true, + "memory_id": "mem-plan123" + } + ], + "total_extracted": 1 +} +``` + +### Suggest Memory from Query + +Suggest creating memory from knowledge base query. + +**Endpoint**: `POST /memory/suggest` + +**Request Body**: +```json +{ + "project_id": "myapp", + "query": "How does the authentication work?", + "answer": "The system uses JWT tokens with refresh token rotation..." +} +``` + +**Response**: +```json +{ + "success": true, + "should_save": true, + "confidence": 0.88, + "suggested_memory": { + "memory_type": "note", + "title": "Authentication mechanism", + "content": "System uses JWT with refresh token rotation", + "importance": 0.7 + } +} +``` + +### Batch Extract from Repository + +Batch extract memories from entire repository. + +**Endpoint**: `POST /memory/extract/batch` + +**Request Body**: +```json +{ + "project_id": "myapp", + "repo_path": "/path/to/repository", + "max_commits": 50, + "file_patterns": ["*.py", "*.js"] +} +``` + +**Response**: +```json +{ + "success": true, + "summary": { + "commits_analyzed": 50, + "files_scanned": 125, + "total_extracted": 23, + "by_source": { + "git_commits": 12, + "code_comments": 11 + }, + "by_type": { + "decision": 5, + "experience": 8, + "plan": 10 + } + }, + "execution_time_seconds": 45.2 +} +``` + +--- + +## Code Graph + +Code graph analysis endpoints for repository understanding. + +### Ingest Repository + +Ingest a code repository into the graph database. + +**Endpoint**: `POST /ingest/repo` + +**Request Body**: +```json +{ + "local_path": "/path/to/repository", + "repo_url": null, + "branch": "main", + "mode": "incremental", + "include_globs": ["**/*.py", "**/*.ts", "**/*.tsx"], + "exclude_globs": ["**/node_modules/**", "**/.git/**"], + "since_commit": null +} +``` + +**Parameters**: +- `local_path` (string, optional): Local repository path +- `repo_url` (string, optional): Repository URL to clone +- `branch` (string, optional): Branch name (default: "main") +- `mode` (string, optional): Ingestion mode + - `full`: Complete re-ingestion + - `incremental`: Only changed files (60x faster) +- `include_globs` (array, optional): File patterns to include +- `exclude_globs` (array, optional): File patterns to exclude +- `since_commit` (string, optional): For incremental mode + +**Response**: +```json +{ + "task_id": "ing-20250115-103045-abc12345", + "status": "done", + "message": "Successfully ingested 125 files", + "files_processed": 125, + "mode": "incremental", + "changed_files_count": 8 +} +``` + +### Get Related Files + +Find files related to a query using fulltext search. + +**Endpoint**: `GET /graph/related?query={query}&repoId={repoId}&limit={limit}` + +**Parameters**: +- `query` (string, required): Search query +- `repoId` (string, required): Repository identifier +- `limit` (integer, optional): Max results (default: 30, max: 100) + +**Response**: +```json +{ + "nodes": [ + { + "type": "file", + "ref": "ref://file/src/auth/jwt.py", + "path": "src/auth/jwt.py", + "lang": "python", + "score": 0.92, + "summary": "JWT authentication implementation" + } + ], + "query": "authentication", + "repo_id": "myproject" +} +``` + +### Impact Analysis + +Analyze impact of changes to a file (reverse dependencies). + +**Endpoint**: `GET /graph/impact?repoId={repoId}&file={file}&depth={depth}&limit={limit}` + +**Parameters**: +- `repoId` (string, required): Repository identifier +- `file` (string, required): File path to analyze +- `depth` (integer, optional): Traversal depth (default: 2, max: 5) +- `limit` (integer, optional): Max results (default: 50, max: 100) + +**Response**: +```json +{ + "nodes": [ + { + "type": "file", + "path": "src/api/auth_routes.py", + "lang": "python", + "repoId": "myproject", + "relationship": "IMPORTS", + "depth": 1, + "score": 0.85, + "ref": "ref://file/src/api/auth_routes.py", + "summary": "Auth API routes (imports jwt.py)" + } + ], + "file": "src/auth/jwt.py", + "repo_id": "myproject", + "depth": 2 +} +``` + +### Context Pack + +Build a context pack within token budget. + +**Endpoint**: `GET /context/pack?repoId={repoId}&stage={stage}&budget={budget}&keywords={keywords}&focus={focus}` + +**Parameters**: +- `repoId` (string, required): Repository identifier +- `stage` (string, optional): Development stage (default: "plan") + - `plan`: Project overview + - `review`: Code review focus + - `implement`: Implementation details +- `budget` (integer, optional): Token budget (default: 1500, max: 10000) +- `keywords` (string, optional): Comma-separated keywords +- `focus` (string, optional): Comma-separated focus paths + +**Response**: +```json +{ + "items": [ + { + "kind": "file", + "title": "src/auth/jwt.py", + "summary": "JWT authentication implementation with token generation and validation", + "ref": "ref://file/src/auth/jwt.py", + "extra": { + "lang": "python", + "score": 0.92 + } + } + ], + "budget_used": 1450, + "budget_limit": 1500, + "stage": "implement", + "repo_id": "myproject", + "category_counts": { + "file": 8, + "symbol": 12 + } +} +``` + +--- + +## Task Management + +Asynchronous task queue management. + +### Create Task + +Create a new task. + +**Endpoint**: `POST /tasks/` + +**Request Body**: +```json +{ + "task_type": "document_processing", + "task_name": "Process large document", + "payload": { + "document_content": "...", + "title": "Large Doc" + }, + "priority": 0, + "metadata": { + "source": "api" + } +} +``` + +**Valid task types**: +- `document_processing` +- `schema_parsing` +- `knowledge_graph_construction` +- `batch_processing` + +**Response**: +```json +{ + "task_id": "task-abc123", + "status": "created" +} +``` + +### Get Task Status + +Get status of a specific task. + +**Endpoint**: `GET /tasks/{task_id}` + +**Response**: +```json +{ + "task_id": "task-abc123", + "status": "SUCCESS", + "progress": 100.0, + "message": "Task completed successfully", + "result": { + "chunks_created": 15, + "document_id": "doc-xyz789" + }, + "error": null, + "created_at": "2025-01-15T10:30:00Z", + "started_at": "2025-01-15T10:30:05Z", + "completed_at": "2025-01-15T10:30:45Z", + "metadata": { + "source": "api" + } +} +``` + +**Status values**: +- `PENDING`: Waiting in queue +- `PROCESSING`: Currently running +- `SUCCESS`: Completed successfully +- `FAILED`: Failed with error +- `CANCELLED`: Cancelled by user + +### List Tasks + +List tasks with optional filtering. + +**Endpoint**: `GET /tasks/?status={status}&page={page}&page_size={page_size}&task_type={task_type}` + +**Parameters**: +- `status` (string, optional): Filter by status +- `page` (integer, optional): Page number (default: 1) +- `page_size` (integer, optional): Page size (default: 20, max: 100) +- `task_type` (string, optional): Filter by task type + +**Response**: +```json +{ + "tasks": [ + { + "task_id": "task-abc123", + "status": "SUCCESS", + "progress": 100.0, + "message": "Completed", + "created_at": "2025-01-15T10:30:00Z" + } + ], + "total": 42, + "page": 1, + "page_size": 20 +} +``` + +### Cancel Task + +Cancel a pending or running task. + +**Endpoint**: `DELETE /tasks/{task_id}` + +**Response**: +```json +{ + "message": "Task cancelled successfully", + "task_id": "task-abc123" +} +``` + +### Get Task Statistics + +Get task queue statistics. + +**Endpoint**: `GET /tasks/stats/overview` + +**Response**: +```json +{ + "total_tasks": 156, + "pending_tasks": 5, + "processing_tasks": 2, + "completed_tasks": 142, + "failed_tasks": 6, + "cancelled_tasks": 1 +} +``` + +### Retry Task + +Retry a failed or cancelled task. + +**Endpoint**: `POST /tasks/{task_id}/retry` + +**Response**: +```json +{ + "message": "Task retried successfully", + "original_task_id": "task-abc123", + "new_task_id": "task-xyz789" +} +``` + +### Get Queue Status + +Get current queue status. + +**Endpoint**: `GET /tasks/queue/status` + +**Response**: +```json +{ + "running_tasks": 2, + "max_concurrent_tasks": 5, + "available_slots": 3, + "queue_active": true +} +``` + +--- + +## SQL Parsing + +SQL parsing and analysis endpoints. + +### Parse SQL Statement + +Parse and analyze SQL statement. + +**Endpoint**: `POST /sql/parse` + +**Request Body**: +```json +{ + "sql": "SELECT * FROM users WHERE id = 1", + "dialect": "mysql" +} +``` + +**Supported dialects**: `mysql`, `postgresql`, `oracle`, `sqlserver` + +**Response**: +```json +{ + "success": true, + "parsed": { + "statement_type": "SELECT", + "tables": ["users"], + "columns": ["*"], + "where_conditions": ["id = 1"] + } +} +``` + +### Validate SQL Syntax + +Validate SQL syntax. + +**Endpoint**: `POST /sql/validate` + +**Request Body**: +```json +{ + "sql": "SELECT * FROM users", + "dialect": "mysql" +} +``` + +**Response**: +```json +{ + "valid": true, + "errors": [] +} +``` + +### Convert SQL Dialect + +Convert SQL between dialects. + +**Endpoint**: `POST /sql/convert?sql={sql}&from_dialect={from}&to_dialect={to}` + +**Response**: +```json +{ + "success": true, + "original_sql": "SELECT * FROM users LIMIT 10", + "converted_sql": "SELECT TOP 10 * FROM users", + "from_dialect": "mysql", + "to_dialect": "sqlserver" +} +``` + +### Parse SQL Schema + +Parse SQL schema with auto-detection. + +**Endpoint**: `POST /sql/parse-schema` + +**Request Body**: +```json +{ + "schema_content": "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100));", + "file_path": null +} +``` + +**Response**: +```json +{ + "success": true, + "dialect": "mysql", + "tables": [ + { + "name": "users", + "columns": [ + {"name": "id", "type": "INT", "primary_key": true}, + {"name": "name", "type": "VARCHAR(100)", "primary_key": false} + ] + } + ], + "relationships": [] +} +``` + +--- + +## Real-time Monitoring (SSE) + +Server-Sent Events for real-time task monitoring. + +### Monitor Single Task + +Stream updates for a specific task. + +**Endpoint**: `GET /sse/task/{task_id}` + +**Response**: SSE stream +``` +data: {"task_id": "task-abc123", "status": "PROCESSING", "progress": 25.0} + +data: {"task_id": "task-abc123", "status": "PROCESSING", "progress": 50.0} + +data: {"task_id": "task-abc123", "status": "SUCCESS", "progress": 100.0} +``` + +### Monitor All Tasks + +Stream updates for all tasks. + +**Endpoint**: `GET /sse/tasks?status={status}` + +**Parameters**: +- `status` (string, optional): Filter by status + +**Response**: SSE stream +``` +data: {"event": "task_update", "task_id": "task-1", "status": "PROCESSING"} + +data: {"event": "task_update", "task_id": "task-2", "status": "SUCCESS"} +``` + +### Get SSE Statistics + +Get active SSE connection statistics. + +**Endpoint**: `GET /sse/stats` + +**Response**: +```json +{ + "active_connections": 5, + "task_streams": 3, + "global_streams": 2 +} +``` + +--- + +## Error Handling + +All endpoints follow consistent error response format. + +### Error Response Format + +```json +{ + "detail": "Error message describing what went wrong" +} +``` + +### HTTP Status Codes + +- `200 OK`: Success +- `201 Created`: Resource created +- `400 Bad Request`: Invalid request parameters +- `404 Not Found`: Resource not found +- `500 Internal Server Error`: Server error + +### Common Errors + +**Invalid Parameters**: +```json +{ + "detail": "Invalid task type. Must be one of: document_processing, schema_parsing, knowledge_graph_construction, batch_processing" +} +``` + +**Resource Not Found**: +```json +{ + "detail": "Task not found" +} +``` + +**Service Error**: +```json +{ + "detail": "Failed to initialize Neo4j connection" +} +``` + +--- + +## Rate Limits + +Currently no rate limits are enforced. This may change in future versions. + +## Pagination + +Endpoints that return lists support pagination: +- `page`: Page number (default: 1) +- `page_size`: Items per page (default: 20, max: 100) + +Response includes: +- `total`: Total item count +- `page`: Current page +- `page_size`: Items per page + +--- + +**Last Updated**: 2025-01-15 +**API Version**: 1.0.0 +**Documentation Version**: 1.0 diff --git a/docs/architecture/components.md b/docs/architecture/components.md new file mode 100644 index 0000000..3babc32 --- /dev/null +++ b/docs/architecture/components.md @@ -0,0 +1,1492 @@ +# System Components + +## Table of Contents + +- [Overview](#overview) +- [API Layer Components](#api-layer-components) +- [Service Layer Components](#service-layer-components) +- [Storage Components](#storage-components) +- [Utility Components](#utility-components) +- [MCP Server Components](#mcp-server-components) +- [Component Dependencies](#component-dependencies) + +## Overview + +The Code Graph Knowledge System consists of multiple specialized components organized in layers. This document provides detailed descriptions of each component, their responsibilities, interfaces, and interactions. + +```mermaid +graph TB + subgraph "API Layer" + FastAPI[FastAPI Application] + MCP[MCP Server] + Routes[API Routes] + Middleware[Middleware Stack] + end + + subgraph "Service Layer" + KnowServ[Knowledge Service] + MemServ[Memory Store] + GraphServ[Graph Service] + TaskQ[Task Queue] + CodeIng[Code Ingestor] + MemExt[Memory Extractor] + end + + subgraph "Supporting Services" + SQLParse[SQL Parser] + Ranker[Result Ranker] + PackBuild[Context Pack Builder] + GitUtil[Git Utilities] + Metrics[Metrics Service] + end + + subgraph "Storage" + Neo4j[(Neo4j)] + SQLite[(SQLite)] + FileSystem[File System] + end + + FastAPI --> Routes + FastAPI --> Middleware + Routes --> KnowServ + Routes --> MemServ + Routes --> GraphServ + Routes --> TaskQ + + MCP --> KnowServ + MCP --> MemServ + MCP --> GraphServ + + KnowServ --> Neo4j + MemServ --> Neo4j + GraphServ --> Neo4j + TaskQ --> SQLite + CodeIng --> GraphServ + MemExt --> MemServ + + style FastAPI fill:#4CAF50 + style MCP fill:#2196F3 + style Neo4j fill:#f9a825 +``` + +## API Layer Components + +### FastAPI Application + +**File**: `main.py`, `core/app.py` + +**Purpose**: Main web server providing RESTful API endpoints + +**Key Responsibilities**: +- HTTP request handling +- Route management +- Middleware processing +- Static file serving +- API documentation (OpenAPI/Swagger) + +**Configuration**: +```python +app = FastAPI( + title="Code Graph Knowledge Service", + version="1.0.0", + lifespan=lifespan, # Startup/shutdown hooks + docs_url="/docs", + redoc_url="/redoc" +) +``` + +**Dependencies**: +- All service layer components +- Configuration settings +- Middleware stack +- Exception handlers + +**Startup Sequence**: +1. Load configuration from environment +2. Initialize logging system +3. Initialize all services via lifespan manager +4. Setup middleware +5. Register routes +6. Mount static files +7. Integrate monitoring UI (if enabled) + +**Shutdown Sequence**: +1. Stop accepting new requests +2. Stop task queue +3. Close Memory Store +4. Close Knowledge Service +5. Close database connections + +### API Routes + +**File**: `core/routes.py`, `api/*.py` + +**Purpose**: Organize and register all API endpoints + +**Route Modules**: + +#### 1. Main Routes (`api/routes.py`) +```python +# Health check +GET /api/v1/health + +# Knowledge base operations +POST /api/v1/knowledge/query +POST /api/v1/knowledge/search +POST /api/v1/documents/add +POST /api/v1/documents/file +POST /api/v1/documents/directory + +# SQL parsing +POST /api/v1/sql/parse +POST /api/v1/sql/schema/upload + +# Code graph +POST /api/v1/code/ingest +POST /api/v1/code/search +POST /api/v1/code/related +POST /api/v1/code/impact +POST /api/v1/code/context-pack +``` + +#### 2. Memory Routes (`api/memory_routes.py`) +```python +# Memory management +POST /api/v1/memory/add +POST /api/v1/memory/search +GET /api/v1/memory/{memory_id} +PUT /api/v1/memory/{memory_id} +DELETE /api/v1/memory/{memory_id} +POST /api/v1/memory/supersede +GET /api/v1/memory/project/{project_id}/summary + +# Memory extraction (v0.7) +POST /api/v1/memory/extract/conversation +POST /api/v1/memory/extract/commit +POST /api/v1/memory/extract/comments +POST /api/v1/memory/suggest +POST /api/v1/memory/extract/batch +``` + +#### 3. Task Routes (`api/task_routes.py`) +```python +# Task management +GET /api/v1/tasks/{task_id} +GET /api/v1/tasks +POST /api/v1/tasks/{task_id}/cancel +GET /api/v1/queue/stats +``` + +#### 4. SSE Routes (`api/sse_routes.py`) +```python +# Server-Sent Events for real-time updates +GET /api/v1/sse/task/{task_id} +GET /api/v1/sse/tasks +GET /api/v1/sse/stats +``` + +#### 5. WebSocket Routes (`api/websocket_routes.py`) +```python +# WebSocket connections +WS /api/v1/ws/task/{task_id} +``` + +**Request/Response Models**: +```python +# Example: Document addition +class DocumentAddRequest(BaseModel): + content: str + title: str = "Untitled" + metadata: Optional[Dict[str, Any]] = None + +class DocumentAddResponse(BaseModel): + success: bool + document_id: Optional[str] = None + message: str + error: Optional[str] = None +``` + +### Middleware Stack + +**File**: `core/middleware.py` + +**Purpose**: Process all requests/responses with cross-cutting concerns + +**Middleware Components**: + +#### 1. CORS Middleware +```python +CORSMiddleware( + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"] +) +``` + +**Purpose**: Handle cross-origin requests for web clients + +#### 2. GZip Middleware +```python +GZipMiddleware(minimum_size=1000) +``` + +**Purpose**: Compress responses for bandwidth optimization + +#### 3. Request Logging Middleware +```python +@app.middleware("http") +async def log_requests(request: Request, call_next): + start_time = time.time() + response = await call_next(request) + duration = time.time() - start_time + logger.info(f"{request.method} {request.url.path} {response.status_code} {duration:.3f}s") + return response +``` + +**Purpose**: Log all HTTP requests with timing information + +#### 4. Error Handling Middleware +```python +@app.exception_handler(Exception) +async def global_exception_handler(request: Request, exc: Exception): + logger.error(f"Unhandled exception: {exc}") + return JSONResponse( + status_code=500, + content={"detail": "Internal server error"} + ) +``` + +**Purpose**: Catch and handle all uncaught exceptions + +### Lifespan Manager + +**File**: `core/lifespan.py` + +**Purpose**: Manage application startup and shutdown lifecycle + +**Initialization Sequence**: +```python +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup + logger.info("Starting services...") + + # 1. Initialize Neo4j Knowledge Service + await neo4j_knowledge_service.initialize() + + # 2. Initialize Memory Store + await memory_store.initialize() + + # 3. Initialize Task Processors + processor_registry.initialize_default_processors(neo4j_knowledge_service) + + # 4. Start Task Queue + await task_queue.start() + + yield + + # Shutdown + await task_queue.stop() + await memory_store.close() + await neo4j_knowledge_service.close() +``` + +**Design Pattern**: Context manager ensures proper cleanup even on errors + +## Service Layer Components + +### Knowledge Service + +**File**: `services/neo4j_knowledge_service.py` + +**Purpose**: Primary service for knowledge graph operations using LlamaIndex + +**Key Capabilities**: +- Document processing and chunking +- Vector embedding generation +- Knowledge graph construction +- RAG-based query answering +- Semantic similarity search + +**Architecture**: +```python +class Neo4jKnowledgeService: + def __init__(self): + self.graph_store = None # Neo4j graph store + self.knowledge_index = None # LlamaIndex KnowledgeGraphIndex + self.query_engine = None # RAG query engine + self._initialized = False +``` + +**Initialization Flow**: +```mermaid +sequenceDiagram + participant Client + participant KnowServ as Knowledge Service + participant LlamaIndex + participant Neo4j + + Client->>KnowServ: initialize() + KnowServ->>KnowServ: _create_llm() + KnowServ->>KnowServ: _create_embed_model() + KnowServ->>Neo4j: Connect via Neo4jGraphStore + Neo4j-->>KnowServ: Connection established + KnowServ->>LlamaIndex: Configure Settings + KnowServ->>LlamaIndex: Create KnowledgeGraphIndex + LlamaIndex-->>KnowServ: Index ready + KnowServ->>KnowServ: Create query engine + KnowServ-->>Client: Initialized +``` + +**Core Methods**: + +#### 1. Document Addition +```python +async def add_document( + self, + content: str, + title: str = "Untitled", + metadata: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + """Add document to knowledge graph""" + # 1. Create LlamaIndex Document + document = Document(text=content, metadata={...}) + + # 2. Insert into index (creates nodes, embeddings, relationships) + await asyncio.to_thread(self.knowledge_index.insert, document) + + # 3. Return result with document ID + return {"success": True, "document_id": doc_id} +``` + +#### 2. Query Processing (RAG) +```python +async def query( + self, + question: str, + top_k: int = 5 +) -> Dict[str, Any]: + """Query knowledge base with RAG""" + # 1. Use query engine to retrieve relevant context + # 2. Generate answer using LLM with context + response = await asyncio.to_thread( + self.query_engine.query, + question + ) + + # 3. Return answer with source nodes + return { + "success": True, + "answer": str(response), + "sources": [node.metadata for node in response.source_nodes] + } +``` + +#### 3. Semantic Search +```python +async def search_similar( + self, + query: str, + top_k: int = 5 +) -> Dict[str, Any]: + """Find similar documents using vector search""" + # 1. Generate query embedding + # 2. Search Neo4j vector index + # 3. Return ranked results +``` + +**LLM Provider Support**: +```python +def _create_llm(self): + provider = settings.llm_provider + + if provider == "ollama": + return Ollama(model=settings.ollama_model, ...) + elif provider == "openai": + return OpenAI(model=settings.openai_model, ...) + elif provider == "gemini": + return Gemini(model=settings.gemini_model, ...) + elif provider == "openrouter": + return OpenRouter(model=settings.openrouter_model, ...) +``` + +**Embedding Model Support**: +```python +def _create_embed_model(self): + provider = settings.embedding_provider + + if provider == "ollama": + return OllamaEmbedding(model_name=settings.ollama_embedding_model) + elif provider == "openai": + return OpenAIEmbedding(model=settings.openai_embedding_model) + elif provider == "gemini": + return GeminiEmbedding(model_name=settings.gemini_embedding_model) + elif provider == "huggingface": + return HuggingFaceEmbedding(model_name=settings.huggingface_embedding_model) +``` + +**Configuration**: +```python +# Global LlamaIndex settings +Settings.llm = self._create_llm() +Settings.embed_model = self._create_embed_model() +Settings.chunk_size = settings.chunk_size +Settings.chunk_overlap = settings.chunk_overlap +Settings.node_parser = SimpleNodeParser.from_defaults() +``` + +### Memory Store + +**File**: `services/memory_store.py` + +**Purpose**: Persistent project knowledge management for AI agents + +**Memory Types**: +```python +MemoryType = Literal[ + "decision", # Architecture choices, tech decisions + "preference", # Coding styles, tool preferences + "experience", # Problems and solutions + "convention", # Team rules, naming conventions + "plan", # Future improvements, TODOs + "note" # Other important information +] +``` + +**Data Model**: +```python +class Memory: + id: str # Unique identifier + project_id: str # Project namespace + memory_type: MemoryType # Type of memory + title: str # Short description + content: str # Main content + reason: Optional[str] # Rationale/context + importance: float # 0.0-1.0 score + tags: List[str] # Categorization tags + created_at: datetime # Creation timestamp + updated_at: datetime # Last update + is_active: bool # Soft delete flag + superseded_by: Optional[str] # Replacement memory ID +``` + +**Graph Schema**: +```cypher +// Nodes +(:Memory { + id: string, + project_id: string, + memory_type: string, + title: string, + content: string, + reason: string, + importance: float, + tags: [string], + created_at: datetime, + updated_at: datetime, + is_active: boolean, + superseded_by: string +}) + +(:Project { + id: string, + name: string, + created_at: datetime +}) + +// Relationships +(Memory)-[:BELONGS_TO]->(Project) +(Memory)-[:RELATES_TO]->(Memory) +(Memory)-[:SUPERSEDES]->(Memory) +``` + +**Core Operations**: + +#### 1. Add Memory +```python +async def add_memory( + self, + project_id: str, + memory_type: MemoryType, + title: str, + content: str, + reason: Optional[str] = None, + importance: float = 0.5, + tags: Optional[List[str]] = None +) -> Dict[str, Any]: + """Add new project memory""" + # 1. Generate unique ID + # 2. Create Memory node in Neo4j + # 3. Link to Project + # 4. Create fulltext indexes + # 5. Return memory details +``` + +#### 2. Search Memories +```python +async def search_memories( + self, + project_id: str, + query: Optional[str] = None, + memory_type: Optional[MemoryType] = None, + tags: Optional[List[str]] = None, + min_importance: float = 0.0, + limit: int = 10 +) -> List[Dict[str, Any]]: + """Search project memories with filters""" + # 1. Build Cypher query with filters + # 2. Use fulltext search if query provided + # 3. Filter by type, tags, importance + # 4. Order by relevance and importance + # 5. Return ranked results +``` + +#### 3. Supersede Memory +```python +async def supersede_memory( + self, + old_memory_id: str, + new_title: str, + new_content: str, + ... +) -> Dict[str, Any]: + """Replace old memory with new version""" + # 1. Create new memory + # 2. Mark old memory as superseded + # 3. Create SUPERSEDES relationship + # 4. Maintain history chain +``` + +**Indexes**: +```cypher +// Constraints +CREATE CONSTRAINT memory_id_unique IF NOT EXISTS +FOR (m:Memory) REQUIRE m.id IS UNIQUE; + +// Fulltext search +CREATE FULLTEXT INDEX memory_search IF NOT EXISTS +FOR (m:Memory) ON EACH [m.title, m.content, m.reason, m.tags]; +``` + +### Graph Service + +**File**: `services/graph_service.py` + +**Purpose**: Direct Neo4j graph database operations + +**Key Capabilities**: +- Raw Cypher query execution +- Code graph management +- Schema operations +- Batch operations +- Transaction management + +**Architecture**: +```python +class Neo4jGraphService: + def __init__(self): + self.driver = None # Neo4j driver + self._connected = False +``` + +**Core Methods**: + +#### 1. Execute Query +```python +async def execute_query( + self, + cypher: str, + parameters: Optional[Dict[str, Any]] = None +) -> GraphQueryResult: + """Execute Cypher query and return results""" + with self.driver.session(database=settings.neo4j_database) as session: + result = session.run(cypher, parameters) + return self._process_result(result) +``` + +#### 2. Create Nodes +```python +def create_node( + self, + labels: List[str], + properties: Dict[str, Any] +) -> GraphNode: + """Create graph node""" + cypher = f""" + CREATE (n:{':'.join(labels)}) + SET n = $properties + RETURN n + """ + # Execute and return node +``` + +#### 3. Create Relationships +```python +def create_relationship( + self, + start_node_id: str, + end_node_id: str, + relationship_type: str, + properties: Optional[Dict[str, Any]] = None +) -> GraphRelationship: + """Create relationship between nodes""" + cypher = """ + MATCH (a), (b) + WHERE a.id = $start_id AND b.id = $end_id + CREATE (a)-[r:$rel_type]->(b) + SET r = $properties + RETURN r + """ + # Execute and return relationship +``` + +**Code Graph Schema**: +```cypher +// Repository structure +(:Repo {id: string, name: string, path: string}) +(:File {repoId: string, path: string, lang: string, content: string}) +(:Symbol {id: string, name: string, type: string, line: int}) + +// Code entities +(:Function {id: string, name: string, params: [string], returns: string}) +(:Class {id: string, name: string, methods: [string]}) +(:Table {id: string, name: string, columns: [string]}) + +// Relationships +(File)-[:BELONGS_TO]->(Repo) +(Symbol)-[:DEFINED_IN]->(File) +(Symbol)-[:CALLS]->(Symbol) +(Symbol)-[:INHERITS]->(Symbol) +(Symbol)-[:USES]->(Symbol) +``` + +### Task Queue + +**File**: `services/task_queue.py` + +**Purpose**: Asynchronous background task processing with persistence + +**Design Pattern**: Producer-Consumer with SQLite persistence + +**Architecture**: +```python +class TaskQueue: + def __init__(self, max_concurrent_tasks: int = 3): + self.max_concurrent_tasks = max_concurrent_tasks + self.tasks: Dict[str, TaskResult] = {} # In-memory cache + self.running_tasks: Dict[str, asyncio.Task] = {} # Active tasks + self.task_semaphore = asyncio.Semaphore(max_concurrent_tasks) + self._storage = None # SQLite storage + self._worker_id = str(uuid.uuid4()) # Worker identity +``` + +**Task Lifecycle**: +```mermaid +stateDiagram-v2 + [*] --> PENDING: Task created + PENDING --> PROCESSING: Worker picks up + PROCESSING --> SUCCESS: Completed + PROCESSING --> FAILED: Error occurred + PROCESSING --> CANCELLED: User cancelled + SUCCESS --> [*] + FAILED --> [*] + CANCELLED --> [*] + + note right of PROCESSING + Progress updates + sent via SSE/WebSocket + end note +``` + +**Task Status**: +```python +class TaskStatus(Enum): + PENDING = "pending" # Queued, not started + PROCESSING = "processing" # Currently running + SUCCESS = "success" # Completed successfully + FAILED = "failed" # Error occurred + CANCELLED = "cancelled" # User cancelled +``` + +**Task Result**: +```python +@dataclass +class TaskResult: + task_id: str + status: TaskStatus + progress: float = 0.0 # 0.0 to 1.0 + message: str = "" # Status message + result: Optional[Dict[str, Any]] = None # Final result + error: Optional[str] = None # Error details + created_at: datetime + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + metadata: Dict[str, Any] # Task-specific data +``` + +**Core Operations**: + +#### 1. Submit Task +```python +async def submit_task( + self, + task_func: Callable, + *args, + task_type: str = "generic", + **kwargs +) -> str: + """Submit new task for processing""" + # 1. Generate task ID + task_id = str(uuid.uuid4()) + + # 2. Create TaskResult + task_result = TaskResult( + task_id=task_id, + status=TaskStatus.PENDING, + metadata={"type": task_type} + ) + + # 3. Store in SQLite + await self._storage.store_task(task_result) + + # 4. Cache in memory + self.tasks[task_id] = task_result + + # 5. Worker will pick up automatically + return task_id +``` + +#### 2. Process Tasks (Worker) +```python +async def _process_pending_tasks(self): + """Background worker to process pending tasks""" + while True: + try: + # 1. Get pending tasks from SQLite + pending = await self._storage.get_pending_tasks(limit=10) + + # 2. Process each task + for task in pending: + if len(self.running_tasks) < self.max_concurrent_tasks: + await self._execute_task(task) + + # 3. Wait before next poll + await asyncio.sleep(1) + + except asyncio.CancelledError: + break +``` + +#### 3. Execute Task +```python +async def _execute_task(self, task: TaskResult): + """Execute single task with error handling""" + async with self.task_semaphore: + try: + # 1. Update status to PROCESSING + task.status = TaskStatus.PROCESSING + task.started_at = datetime.now() + await self._storage.update_task_status(task.task_id, task.status) + + # 2. Get processor for task type + processor = processor_registry.get_processor(task.metadata["type"]) + + # 3. Execute processor + result = await processor.process(task) + + # 4. Update status to SUCCESS + task.status = TaskStatus.SUCCESS + task.result = result + task.completed_at = datetime.now() + + except Exception as e: + # Update status to FAILED + task.status = TaskStatus.FAILED + task.error = str(e) + + finally: + # Save to storage + await self._storage.update_task(task) +``` + +**Progress Tracking**: +```python +async def update_progress( + self, + task_id: str, + progress: float, + message: str +): + """Update task progress""" + task = self.tasks.get(task_id) + if task: + task.progress = progress + task.message = message + await self._storage.update_task(task) + + # Notify SSE/WebSocket listeners + await self._notify_listeners(task_id, task) +``` + +### Code Ingestor + +**File**: `services/code_ingestor.py` + +**Purpose**: Parse and ingest code repositories into graph structure + +**Supported Languages**: +- Python +- JavaScript/TypeScript +- Java +- Go +- C/C++ +- SQL + +**Ingestion Process**: +```mermaid +sequenceDiagram + participant Client + participant Ingestor + participant Parser + participant GraphService + participant Neo4j + + Client->>Ingestor: ingest_repository(path) + Ingestor->>Ingestor: Scan directory + loop For each file + Ingestor->>Parser: parse_file(file, language) + Parser-->>Ingestor: AST + Symbols + Ingestor->>GraphService: create_file_node() + Ingestor->>GraphService: create_symbol_nodes() + Ingestor->>GraphService: create_relationships() + GraphService->>Neo4j: Cypher queries + end + Ingestor-->>Client: Ingestion complete +``` + +**Core Methods**: + +#### 1. Ingest Repository +```python +async def ingest_repository( + self, + repo_path: str, + repo_name: Optional[str] = None +) -> Dict[str, Any]: + """Ingest entire code repository""" + # 1. Create Repo node + # 2. Walk directory tree + # 3. Parse each file + # 4. Create graph structure + # 5. Return statistics +``` + +#### 2. Parse File +```python +def parse_file(self, file_path: str, language: str) -> ParseResult: + """Parse code file and extract symbols""" + if language == "python": + return self._parse_python(file_path) + elif language == "javascript": + return self._parse_javascript(file_path) + # ... other languages +``` + +#### 3. Extract Symbols +```python +def _parse_python(self, file_path: str) -> ParseResult: + """Parse Python file using AST""" + import ast + + with open(file_path) as f: + tree = ast.parse(f.read()) + + symbols = [] + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + symbols.append({ + "type": "function", + "name": node.name, + "line": node.lineno + }) + elif isinstance(node, ast.ClassDef): + symbols.append({ + "type": "class", + "name": node.name, + "line": node.lineno + }) + + return ParseResult(symbols=symbols, relationships=[]) +``` + +### Memory Extractor + +**File**: `services/memory_extractor.py` + +**Purpose**: Automatically extract memories from various sources (v0.7) + +**Extraction Sources**: +1. Conversation analysis +2. Git commit mining +3. Code comment extraction +4. Query/answer analysis +5. Batch repository analysis + +**Core Methods**: + +#### 1. Extract from Conversation +```python +async def extract_from_conversation( + self, + project_id: str, + conversation: List[Dict[str, str]], + auto_save: bool = False +) -> List[Dict[str, Any]]: + """Extract memories from AI conversation""" + # 1. Format conversation for LLM + # 2. Use LLM to identify decisions, learnings + # 3. Generate memory objects + # 4. Optionally auto-save high-confidence memories +``` + +#### 2. Extract from Git Commit +```python +async def extract_from_git_commit( + self, + project_id: str, + commit_sha: str, + commit_message: str, + changed_files: List[str], + auto_save: bool = False +) -> List[Dict[str, Any]]: + """Extract memories from git commit""" + # 1. Analyze commit message + # 2. Analyze changed files + # 3. Use LLM to extract decisions/experiences + # 4. Generate memories with context +``` + +#### 3. Extract from Code Comments +```python +async def extract_from_code_comments( + self, + project_id: str, + file_path: str +) -> List[Dict[str, Any]]: + """Mine TODO, FIXME, NOTE markers""" + # 1. Parse file for comment markers + # 2. Extract context around markers + # 3. Classify as plan/note/experience + # 4. Generate memory objects +``` + +### Task Processors + +**File**: `services/task_processors.py` + +**Purpose**: Implement specific task processing logic + +**Processor Registry**: +```python +class ProcessorRegistry: + def __init__(self): + self.processors: Dict[str, TaskProcessor] = {} + + def register(self, task_type: str, processor: TaskProcessor): + """Register processor for task type""" + self.processors[task_type] = processor + + def get_processor(self, task_type: str) -> TaskProcessor: + """Get processor for task type""" + return self.processors.get(task_type) +``` + +**Built-in Processors**: + +#### 1. Document Processor +```python +class DocumentProcessor(TaskProcessor): + async def process(self, task: TaskResult) -> Dict[str, Any]: + """Process document ingestion task""" + # 1. Read document from file/content + # 2. Call knowledge service + # 3. Update progress + # 4. Return result +``` + +#### 2. Directory Processor +```python +class DirectoryProcessor(TaskProcessor): + async def process(self, task: TaskResult) -> Dict[str, Any]: + """Process batch directory ingestion""" + # 1. List files in directory + # 2. Filter by patterns + # 3. Process each file + # 4. Update progress incrementally + # 5. Return summary +``` + +#### 3. Code Ingestion Processor +```python +class CodeIngestionProcessor(TaskProcessor): + async def process(self, task: TaskResult) -> Dict[str, Any]: + """Process code repository ingestion""" + # 1. Call code ingestor + # 2. Track progress per file + # 3. Return ingestion statistics +``` + +## Storage Components + +### Neo4j Graph Database + +**Purpose**: Primary storage for all graph data + +**Node Types**: +```cypher +// Knowledge graph +:Document, :Entity, :Chunk + +// Memory store +:Memory, :Project + +// Code graph +:Repo, :File, :Symbol, :Function, :Class, :Table + +// SQL schema +:Database, :Table, :Column +``` + +**Indexes**: +```cypher +// Constraints +CREATE CONSTRAINT FOR (d:Document) REQUIRE d.id IS UNIQUE; +CREATE CONSTRAINT FOR (m:Memory) REQUIRE m.id IS UNIQUE; +CREATE CONSTRAINT FOR (r:Repo) REQUIRE r.id IS UNIQUE; + +// Fulltext indexes +CREATE FULLTEXT INDEX memory_search FOR (m:Memory) + ON EACH [m.title, m.content, m.reason, m.tags]; + +CREATE FULLTEXT INDEX file_text FOR (f:File) + ON EACH [f.path, f.lang]; + +// Vector index +CALL db.index.vector.createNodeIndex( + 'knowledge_vectors', + 'Document', + 'embedding', + 1536, + 'cosine' +); +``` + +### SQLite Task Storage + +**File**: `services/task_storage.py` + +**Purpose**: Persistent storage for task queue + +**Schema**: +```sql +CREATE TABLE tasks ( + task_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + task_type TEXT NOT NULL, + progress REAL DEFAULT 0.0, + message TEXT, + result TEXT, -- JSON + error TEXT, + metadata TEXT, -- JSON + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + worker_id TEXT, + locked_at TEXT +); + +CREATE INDEX idx_status ON tasks(status); +CREATE INDEX idx_created ON tasks(created_at); +CREATE INDEX idx_worker ON tasks(worker_id); +``` + +**Concurrency Control**: +```python +async def get_pending_tasks(self, limit: int = 10) -> List[TaskResult]: + """Get and lock pending tasks""" + # Use SELECT ... FOR UPDATE to prevent race conditions + query = """ + UPDATE tasks + SET worker_id = ?, locked_at = ? + WHERE task_id IN ( + SELECT task_id FROM tasks + WHERE status = 'pending' + AND (locked_at IS NULL OR locked_at < datetime('now', '-5 minutes')) + ORDER BY created_at + LIMIT ? + ) + RETURNING * + """ +``` + +## Utility Components + +### SQL Parser + +**File**: `services/sql_parser.py` + +**Purpose**: Parse SQL queries and extract metadata + +**Capabilities**: +- SQL syntax parsing +- Table/column extraction +- Query type detection +- Dependency analysis + +### Result Ranker + +**File**: `services/ranker.py` + +**Purpose**: Rank search results by relevance + +**Ranking Factors**: +- Vector similarity score +- Graph distance +- Metadata match +- Recency + +### Context Pack Builder + +**File**: `services/pack_builder.py` + +**Purpose**: Generate context packages for AI tools + +**Output Format**: +```python +{ + "files": [ + {"path": "src/main.py", "content": "...", "relevance": 0.95}, + {"path": "src/utils.py", "content": "...", "relevance": 0.87} + ], + "symbols": [ + {"name": "process_data", "type": "function", "file": "src/main.py"} + ], + "relationships": [ + {"from": "main.py", "to": "utils.py", "type": "imports"} + ], + "metadata": { + "total_files": 2, + "total_lines": 450, + "languages": ["python"] + } +} +``` + +### Git Utilities + +**File**: `services/git_utils.py` + +**Purpose**: Git repository operations + +**Capabilities**: +- Commit history retrieval +- Diff extraction +- Branch operations +- File change tracking + +## MCP Server Components + +### MCP Server Main + +**File**: `mcp_server.py` + +**Purpose**: Model Context Protocol server using official SDK + +**Architecture**: +```python +# Official MCP SDK +from mcp.server import Server +from mcp.server.models import InitializationOptions + +app = Server("code-graph-knowledge") + +# Tool registration +@app.list_tools() +async def list_tools() -> list[Tool]: + return get_tool_definitions() + +# Tool execution +@app.call_tool() +async def call_tool(name: str, arguments: dict) -> Sequence[TextContent]: + # Route to appropriate handler + handler = tool_handlers.get(name) + result = await handler(arguments) + return [TextContent(type="text", text=format_result(result))] +``` + +**Tool Categories** (30 tools total): + +#### 1. Knowledge Base Tools (5) +- `query_knowledge`: RAG-based Q&A +- `search_similar_nodes`: Vector similarity search +- `add_document`: Add document from content +- `add_file`: Add document from file +- `add_directory`: Batch directory processing + +#### 2. Code Graph Tools (4) +- `code_graph_ingest_repo`: Ingest repository +- `code_graph_related`: Find related code +- `code_graph_impact`: Impact analysis +- `context_pack`: Generate AI context + +#### 3. Memory Tools (7) +- `add_memory`: Create memory +- `search_memories`: Search with filters +- `get_memory`: Get by ID +- `update_memory`: Modify memory +- `delete_memory`: Soft delete +- `supersede_memory`: Replace with new version +- `get_project_summary`: Project overview + +#### 4. Memory Extraction Tools (5) +- `extract_from_conversation`: Analyze conversations +- `extract_from_git_commit`: Mine commits +- `extract_from_code_comments`: Extract from code +- `suggest_memory_from_query`: Suggest from Q&A +- `batch_extract_from_repository`: Batch analysis + +#### 5. Task Tools (6) +- `get_task_status`: Check task status +- `watch_task`: Monitor single task +- `watch_tasks`: Monitor multiple tasks +- `list_tasks`: List all tasks +- `cancel_task`: Cancel task +- `get_queue_stats`: Queue statistics + +#### 6. System Tools (3) +- `get_graph_schema`: Neo4j schema +- `get_statistics`: System stats +- `clear_knowledge_base`: Clear data + +### MCP Tool Handlers + +**File**: `mcp_tools/*.py` + +**Modular Organization**: +``` +mcp_tools/ +├── __init__.py # Exports +├── tool_definitions.py # Tool schemas +├── knowledge_handlers.py # Knowledge operations +├── code_handlers.py # Code graph operations +├── memory_handlers.py # Memory operations +├── task_handlers.py # Task operations +├── system_handlers.py # System operations +├── resources.py # MCP resources +├── prompts.py # MCP prompts +└── utils.py # Shared utilities +``` + +**Handler Pattern**: +```python +async def handle_query_knowledge(arguments: dict) -> dict: + """Handle knowledge query request""" + # 1. Validate arguments + question = arguments.get("question") + if not question: + return {"success": False, "error": "Question required"} + + # 2. Call service + result = await neo4j_knowledge_service.query( + question=question, + top_k=arguments.get("top_k", 5) + ) + + # 3. Return result + return result +``` + +## Component Dependencies + +### Dependency Graph + +```mermaid +graph TB + subgraph "API Layer" + FastAPI + MCPServer[MCP Server] + end + + subgraph "Service Layer" + KnowServ[Knowledge Service] + MemServ[Memory Store] + GraphServ[Graph Service] + TaskQ[Task Queue] + CodeIng[Code Ingestor] + end + + subgraph "External" + Neo4j + LLM[LLM Providers] + end + + FastAPI --> KnowServ + FastAPI --> MemServ + FastAPI --> GraphServ + FastAPI --> TaskQ + + MCPServer --> KnowServ + MCPServer --> MemServ + MCPServer --> GraphServ + MCPServer --> TaskQ + + KnowServ --> Neo4j + KnowServ --> LLM + MemServ --> Neo4j + GraphServ --> Neo4j + CodeIng --> GraphServ + + TaskQ --> KnowServ + TaskQ --> CodeIng +``` + +### Initialization Order + +Critical for avoiding circular dependencies: + +```python +# 1. Configuration (no dependencies) +from config import settings + +# 2. Storage layer (no app dependencies) +neo4j_connection = Neo4jGraphStore(...) + +# 3. Service layer (depends on storage) +knowledge_service = Neo4jKnowledgeService() +memory_store = MemoryStore() +graph_service = Neo4jGraphService() + +# 4. Processors (depend on services) +processor_registry.initialize_default_processors(knowledge_service) + +# 5. Task queue (depends on processors) +await task_queue.start() + +# 6. API layer (depends on all services) +app = create_app() +``` + +### Service Communication Patterns + +**1. Direct Method Calls** (within same process): +```python +# FastAPI route calls service +result = await knowledge_service.query(question) +``` + +**2. Task Queue** (async operations): +```python +# Submit task for background processing +task_id = await task_queue.submit_task( + task_func=process_large_document, + document_path=path +) +``` + +**3. Event Streaming** (real-time updates): +```python +# SSE for task progress +async def task_progress_stream(task_id: str): + while True: + task = task_queue.get_task(task_id) + yield f"data: {json.dumps(task.to_dict())}\n\n" + await asyncio.sleep(1) +``` + +## Component Configuration + +All components are configured via environment variables: + +```python +# config.py +class Settings(BaseSettings): + # Database + neo4j_uri: str = "bolt://localhost:7687" + neo4j_username: str = "neo4j" + neo4j_password: str = "password" + + # LLM + llm_provider: str = "ollama" + ollama_model: str = "llama2" + + # Timeouts + connection_timeout: int = 30 + operation_timeout: int = 120 + + class Config: + env_file = ".env" +``` + +Components access configuration: +```python +from config import settings + +# Use in service +self.timeout = settings.operation_timeout +``` + +## Testing Components + +Each component has corresponding tests: + +``` +tests/ +├── test_neo4j_knowledge_service.py +├── test_memory_store.py +├── test_graph_service.py +├── test_task_queue.py +├── test_code_ingestor.py +└── test_mcp_handlers.py +``` + +**Test Patterns**: +```python +@pytest.mark.asyncio +async def test_add_memory(): + # Setup + memory_store = MemoryStore() + await memory_store.initialize() + + # Execute + result = await memory_store.add_memory( + project_id="test", + memory_type="decision", + title="Test decision", + content="Test content" + ) + + # Assert + assert result["success"] == True + assert "memory_id" in result + + # Cleanup + await memory_store.close() +``` + +## Conclusion + +The component architecture follows these principles: + +1. **Single Responsibility**: Each component has one clear purpose +2. **Loose Coupling**: Components communicate via interfaces +3. **High Cohesion**: Related functionality grouped together +4. **Dependency Injection**: Services injected rather than created +5. **Async-First**: All I/O operations are asynchronous +6. **Testability**: Components designed for easy testing + +This modular design enables: +- Independent development and testing +- Easy component replacement +- Clear debugging and troubleshooting +- Scalable architecture evolution diff --git a/docs/architecture/dataflow.md b/docs/architecture/dataflow.md new file mode 100644 index 0000000..e4ce465 --- /dev/null +++ b/docs/architecture/dataflow.md @@ -0,0 +1,1770 @@ +# Data Flow and Processing Pipelines + +## Table of Contents + +- [Overview](#overview) +- [Document Processing Pipeline](#document-processing-pipeline) +- [Knowledge Query Pipeline (RAG)](#knowledge-query-pipeline-rag) +- [Memory Management Pipeline](#memory-management-pipeline) +- [Code Graph Ingestion Pipeline](#code-graph-ingestion-pipeline) +- [Task Processing Pipeline](#task-processing-pipeline) +- [Real-time Update Pipeline](#real-time-update-pipeline) +- [MCP Request Pipeline](#mcp-request-pipeline) +- [Data Flow Patterns](#data-flow-patterns) + +## Overview + +This document describes how data flows through the Code Graph Knowledge System, detailing the processing pipelines for different operations. Understanding these flows is crucial for: + +- Debugging issues +- Optimizing performance +- Extending functionality +- Understanding system behavior + +### Core Data Types + +```python +# Document content +Document = { + "id": str, + "content": str, + "title": str, + "metadata": dict, + "embedding": List[float], # Vector representation + "chunks": List[str] # Processed chunks +} + +# Memory object +Memory = { + "id": str, + "project_id": str, + "memory_type": str, # decision/preference/experience/etc. + "title": str, + "content": str, + "importance": float, + "tags": List[str] +} + +# Code symbol +Symbol = { + "id": str, + "name": str, + "type": str, # function/class/variable + "file_path": str, + "line": int, + "relationships": List[dict] +} + +# Task result +Task = { + "task_id": str, + "status": str, # pending/processing/success/failed + "progress": float, + "result": dict, + "error": Optional[str] +} +``` + +## Document Processing Pipeline + +### Overview + +Documents go through multiple stages from upload to queryable knowledge: + +```mermaid +graph TB + Start([User uploads document]) --> Validate[Validate & Parse] + Validate --> Size{Size check} + + Size -->|Small < 10KB| DirectProc[Direct Processing] + Size -->|Medium 10-50KB| TaskQueue[Queue as Task] + Size -->|Large > 50KB| Reject[Return size limit error] + + DirectProc --> Chunk[Text Chunking] + TaskQueue --> Worker[Task Worker] + Worker --> Chunk + + Chunk --> Embed[Generate Embeddings] + Embed --> Extract[Extract Entities] + Extract --> BuildGraph[Build Knowledge Graph] + BuildGraph --> StoreNeo4j[Store in Neo4j] + StoreNeo4j --> Index[Create Vector Index] + Index --> Complete([Document ready for query]) + + Reject --> Error([Error response]) + + style DirectProc fill:#90EE90 + style TaskQueue fill:#FFD700 + style Reject fill:#FFB6C1 +``` + +### Step-by-Step Flow + +#### 1. Document Upload + +**API Endpoint**: `POST /api/v1/documents/add` + +```python +# Request +{ + "content": "This is the document content...", + "title": "My Document", + "metadata": {"author": "John", "topic": "AI"} +} +``` + +**Validation**: +```python +# Size check +content_size = len(content.encode('utf-8')) +if content_size > settings.max_document_size: + raise HTTPException(413, "Document too large") + +# Content check +if not content.strip(): + raise HTTPException(400, "Empty content") +``` + +#### 2. Routing Decision + +Based on document size: + +```python +if content_size < 10 * 1024: # < 10KB + # Direct synchronous processing + result = await knowledge_service.add_document(content, title, metadata) + return {"success": True, "document_id": result["document_id"]} + +elif content_size < 50 * 1024: # 10-50KB + # Background task processing + task_id = await submit_document_processing_task(content, title, metadata) + return {"task_id": task_id, "message": "Processing in background"} + +else: # > 50KB + # Reject - too large for direct processing + return {"error": "Document too large, use file upload or directory processing"} +``` + +#### 3. Text Chunking + +**LlamaIndex SimpleNodeParser**: + +```python +from llama_index.core.node_parser import SimpleNodeParser + +parser = SimpleNodeParser.from_defaults( + chunk_size=512, # Characters per chunk + chunk_overlap=50 # Overlap between chunks +) + +# Create chunks +chunks = parser.get_nodes_from_documents([document]) +``` + +**Chunking Strategy**: +``` +Original document (2000 chars): +"Text here... [chunk 1: 512 chars] + ... [overlap: 50 chars] + ... [chunk 2: 512 chars] + ... [overlap: 50 chars] + ... [chunk 3: remainder]" + +Result: 3-4 chunks with overlapping context +``` + +**Why Chunking?** +- LLM context window limits +- Better semantic granularity +- Improved retrieval accuracy +- Parallel processing capability + +#### 4. Embedding Generation + +**Process**: +```python +# For each chunk +embedding = await embed_model.get_text_embedding(chunk.text) +# embedding: List[float] with length = vector_dimension (e.g., 1536) +``` + +**Provider-Specific**: + +**Ollama** (local): +```python +from llama_index.embeddings.ollama import OllamaEmbedding + +embed_model = OllamaEmbedding( + model_name="nomic-embed-text", + base_url="http://localhost:11434" +) +# Dimension: 768 +``` + +**OpenAI**: +```python +from llama_index.embeddings.openai import OpenAIEmbedding + +embed_model = OpenAIEmbedding( + model="text-embedding-ada-002" +) +# Dimension: 1536 +``` + +**Performance Considerations**: +- Batch embeddings when possible +- Cache embeddings for identical text +- Monitor API rate limits +- Use local models for privacy/cost + +#### 5. Entity Extraction + +**LlamaIndex Knowledge Graph Extraction**: + +```python +# Configured in Knowledge Service +from llama_index.core import KnowledgeGraphIndex + +# Extract entities and relationships +index = KnowledgeGraphIndex.from_documents( + documents=[document], + max_triplets_per_chunk=10 +) +``` + +**Extracted Entities**: +```python +# Example triplets (subject, predicate, object) +[ + ("Python", "is_a", "Programming Language"), + ("Python", "supports", "Object-Oriented Programming"), + ("Django", "is_built_with", "Python") +] +``` + +**Entity Types**: +- Concepts (abstract ideas) +- Technologies (tools, frameworks) +- People (authors, contributors) +- Organizations (companies, projects) +- Events (releases, changes) + +#### 6. Graph Construction + +**Neo4j Storage**: + +```cypher +// Create document node +CREATE (d:Document { + id: $doc_id, + title: $title, + created_at: datetime(), + metadata: $metadata +}) + +// Create chunk nodes with embeddings +CREATE (c:Chunk { + id: $chunk_id, + text: $chunk_text, + embedding: $embedding_vector, + chunk_index: $index +}) +CREATE (d)-[:HAS_CHUNK]->(c) + +// Create entity nodes +CREATE (e:Entity { + id: $entity_id, + name: $entity_name, + type: $entity_type +}) + +// Create relationships +CREATE (c)-[:MENTIONS]->(e) +CREATE (e1)-[:RELATES_TO {type: $relation_type}]->(e2) +``` + +**Graph Structure**: +```mermaid +graph TB + Doc[Document Node] + Doc --> Chunk1[Chunk 1 + embedding] + Doc --> Chunk2[Chunk 2 + embedding] + Doc --> Chunk3[Chunk 3 + embedding] + + Chunk1 --> Entity1[Entity: Python] + Chunk1 --> Entity2[Entity: Django] + Chunk2 --> Entity2 + Chunk2 --> Entity3[Entity: FastAPI] + + Entity1 -.->|RELATES_TO| Entity2 + Entity2 -.->|SIMILAR_TO| Entity3 +``` + +#### 7. Vector Index Creation + +**Neo4j Vector Index**: + +```cypher +// Create vector index (if not exists) +CALL db.index.vector.createNodeIndex( + 'knowledge_vectors', // Index name + 'Chunk', // Node label + 'embedding', // Property name + 1536, // Vector dimension + 'cosine' // Similarity metric +) +``` + +**Index Operations**: +- Automatically indexes new chunks +- Enables fast similarity search +- Supports approximate nearest neighbor (ANN) + +#### 8. Completion + +**Success Response**: +```python +{ + "success": true, + "document_id": "doc_abc123", + "chunks_created": 4, + "entities_extracted": 12, + "processing_time": 2.5 # seconds +} +``` + +### Error Handling + +**Common Errors**: + +```python +try: + result = await knowledge_service.add_document(content, title) +except EmbeddingError as e: + # LLM/embedding provider unavailable + return {"error": "Embedding service unavailable", "retry": True} + +except Neo4jError as e: + # Database connection issue + return {"error": "Database error", "retry": True} + +except ValidationError as e: + # Invalid input + return {"error": str(e), "retry": False} + +except TimeoutError as e: + # Operation timeout + return {"error": "Processing timeout", "retry": True} +``` + +## Knowledge Query Pipeline (RAG) + +### Overview + +Query processing combines vector search, graph traversal, and LLM generation: + +```mermaid +graph TB + Query([User asks question]) --> Embed[Generate Query Embedding] + Embed --> VectorSearch[Vector Similarity Search] + VectorSearch --> TopK[Retrieve Top-K Chunks] + TopK --> GraphExpand[Expand via Graph Relationships] + GraphExpand --> Rerank[Rerank by Relevance] + Rerank --> BuildContext[Build Context Window] + BuildContext --> LLMPrompt[Create LLM Prompt] + LLMPrompt --> LLMGenerate[LLM Generate Answer] + LLMGenerate --> PostProcess[Post-process Response] + PostProcess --> Response([Return Answer + Sources]) + + style VectorSearch fill:#E6F3FF + style GraphExpand fill:#FFF4E6 + style LLMGenerate fill:#F0E6FF +``` + +### Step-by-Step Flow + +#### 1. Query Embedding + +**API Endpoint**: `POST /api/v1/knowledge/query` + +```python +# Request +{ + "question": "How does authentication work in the system?", + "top_k": 5 +} +``` + +**Generate embedding**: +```python +query_embedding = await embed_model.get_query_embedding(question) +# Same embedding model as documents for consistency +``` + +#### 2. Vector Similarity Search + +**Neo4j Vector Query**: + +```cypher +CALL db.index.vector.queryNodes( + 'knowledge_vectors', // Index name + $top_k, // Number of results + $query_embedding // Query vector +) +YIELD node, score + +MATCH (node)-[:BELONGS_TO]->(doc:Document) +RETURN node.text as text, + doc.title as source, + score as similarity +ORDER BY score DESC +``` + +**Result**: +```python +[ + { + "text": "The system uses JWT tokens for authentication...", + "source": "Authentication Guide", + "similarity": 0.89 + }, + { + "text": "Users authenticate via POST /api/auth/login...", + "source": "API Documentation", + "similarity": 0.84 + }, + # ... more results +] +``` + +#### 3. Graph-Based Expansion + +**Expand context via relationships**: + +```cypher +// Get related chunks via entity relationships +MATCH (chunk:Chunk)-[:MENTIONS]->(e:Entity)<-[:MENTIONS]-(related:Chunk) +WHERE chunk.id IN $initial_chunk_ids + AND related.id NOT IN $initial_chunk_ids +RETURN related.text as text, + COUNT(*) as connection_strength +ORDER BY connection_strength DESC +LIMIT 3 +``` + +**Why expand?** +- Capture related context +- Find transitively related information +- Improve answer completeness + +#### 4. Result Reranking + +**Combine scores**: + +```python +def rerank_results(results: List[dict]) -> List[dict]: + """Rerank by multiple factors""" + for result in results: + score = ( + result['similarity'] * 0.6 + # Vector similarity + result['graph_score'] * 0.2 + # Graph connectivity + result['recency_score'] * 0.1 + # Document age + result['metadata_match'] * 0.1 # Metadata relevance + ) + result['final_score'] = score + + return sorted(results, key=lambda x: x['final_score'], reverse=True) +``` + +#### 5. Context Window Building + +**Create prompt context**: + +```python +def build_context(chunks: List[dict], max_tokens: int = 2000) -> str: + """Build context staying within token limit""" + context_parts = [] + total_tokens = 0 + + for chunk in chunks: + chunk_tokens = estimate_tokens(chunk['text']) + if total_tokens + chunk_tokens > max_tokens: + break + + context_parts.append(f"[Source: {chunk['source']}]\n{chunk['text']}") + total_tokens += chunk_tokens + + return "\n\n".join(context_parts) +``` + +#### 6. LLM Prompt Construction + +**Prompt Template**: + +```python +prompt = f"""You are a helpful assistant. Answer the question based on the provided context. + +Context: +{context} + +Question: {question} + +Answer: Provide a comprehensive answer based on the context above. If the context doesn't contain enough information to fully answer the question, say so. +""" +``` + +**Advanced Prompting**: +```python +# With instructions +prompt = f"""You are a technical documentation expert. + +Context from knowledge base: +{context} + +User question: {question} + +Instructions: +1. Answer based on the provided context +2. Cite sources when possible +3. If information is incomplete, state what's missing +4. Use technical accuracy + +Answer:""" +``` + +#### 7. LLM Generation + +**Call LLM**: + +```python +# OpenAI example +response = await llm.acomplete(prompt) +answer = response.text + +# Ollama example +response = await llm.acomplete(prompt) +answer = response.text +``` + +**Streaming Support**: +```python +async for token in llm.astream_complete(prompt): + yield token.text # Stream to client +``` + +#### 8. Response Assembly + +**Final Response**: + +```python +{ + "success": true, + "answer": "The system uses JWT token-based authentication...", + "sources": [ + { + "title": "Authentication Guide", + "relevance": 0.89, + "excerpt": "JWT tokens are used..." + }, + { + "title": "API Documentation", + "relevance": 0.84, + "excerpt": "Login endpoint returns..." + } + ], + "metadata": { + "chunks_retrieved": 5, + "chunks_used": 3, + "processing_time": 1.2, + "model": "gpt-3.5-turbo" + } +} +``` + +### Performance Optimization + +**Caching**: +```python +# Cache query embeddings +@lru_cache(maxsize=1000) +async def get_cached_embedding(query: str) -> List[float]: + return await embed_model.get_query_embedding(query) + +# Cache common queries +query_cache = {} +cache_key = f"{question}:{top_k}" +if cache_key in query_cache: + return query_cache[cache_key] +``` + +**Parallel Processing**: +```python +# Parallel embedding and metadata lookup +embedding_task = asyncio.create_task(generate_embedding(question)) +metadata_task = asyncio.create_task(get_metadata_filters(question)) + +embedding = await embedding_task +metadata = await metadata_task +``` + +## Memory Management Pipeline + +### Overview + +Memory lifecycle from creation to retrieval: + +```mermaid +graph TB + Create([Create Memory]) --> Validate[Validate Input] + Validate --> Generate[Generate Memory ID] + Generate --> Store[Store in Neo4j] + Store --> Index[Index for Search] + Index --> Link[Link to Project] + Link --> Complete([Memory Ready]) + + Search([Search Request]) --> Parse[Parse Filters] + Parse --> Fulltext{Fulltext Search?} + Fulltext -->|Yes| FTS[Fulltext Query] + Fulltext -->|No| Filter[Filter Query] + FTS --> Merge[Merge Results] + Filter --> Merge + Merge --> Rank[Rank by Importance] + Rank --> Return([Return Results]) + + style Create fill:#90EE90 + style Search fill:#FFD700 +``` + +### Memory Creation Flow + +#### 1. Manual Memory Creation + +**API Endpoint**: `POST /api/v1/memory/add` + +```python +{ + "project_id": "myapp", + "memory_type": "decision", + "title": "Use PostgreSQL for main database", + "content": "Decided to use PostgreSQL instead of MySQL", + "reason": "Need advanced JSON support and better performance", + "importance": 0.9, + "tags": ["database", "architecture", "backend"] +} +``` + +**Processing**: +```python +# 1. Validate +if not 0 <= importance <= 1: + raise ValueError("Importance must be 0-1") + +# 2. Generate ID +memory_id = f"mem_{uuid.uuid4().hex[:12]}" + +# 3. Store in Neo4j +await memory_store.add_memory( + project_id=project_id, + memory_type=memory_type, + title=title, + content=content, + reason=reason, + importance=importance, + tags=tags +) +``` + +**Neo4j Storage**: +```cypher +// Ensure project exists +MERGE (p:Project {id: $project_id}) + +// Create memory +CREATE (m:Memory { + id: $memory_id, + project_id: $project_id, + memory_type: $memory_type, + title: $title, + content: $content, + reason: $reason, + importance: $importance, + tags: $tags, + created_at: datetime(), + updated_at: datetime(), + is_active: true +}) + +// Link to project +CREATE (m)-[:BELONGS_TO]->(p) + +RETURN m +``` + +#### 2. Automatic Memory Extraction + +**Conversation Analysis Flow**: + +```mermaid +sequenceDiagram + participant User + participant API + participant Extractor + participant LLM + participant MemStore + + User->>API: Extract from conversation + API->>Extractor: analyze_conversation() + Extractor->>LLM: Identify decisions/learnings + LLM-->>Extractor: Extracted memories + Extractor->>Extractor: Score confidence + alt Auto-save enabled & high confidence + Extractor->>MemStore: Save memories + else + Extractor->>API: Return suggestions + end + API-->>User: Memories/suggestions +``` + +**LLM Prompt for Extraction**: +```python +prompt = f"""Analyze this conversation and extract important project knowledge. + +Conversation: +{format_conversation(conversation)} + +Extract: +1. Technical decisions and rationale +2. Preferences and conventions +3. Problems encountered and solutions +4. Important insights or learnings + +For each item, provide: +- Type (decision/preference/experience/note) +- Title (short summary) +- Content (detailed description) +- Confidence score (0-1) + +Return as JSON list. +""" +``` + +**Example Extraction Result**: +```python +[ + { + "type": "decision", + "title": "Use Redis for caching", + "content": "Decided to add Redis caching layer for API responses", + "reason": "Reduce database load and improve response times", + "confidence": 0.85, + "importance": 0.8, + "tags": ["caching", "performance", "redis"] + } +] +``` + +**Auto-save Logic**: +```python +if auto_save: + for memory in extracted_memories: + if memory['confidence'] > 0.7: # High confidence threshold + await memory_store.add_memory(**memory) +``` + +### Memory Search Flow + +**API Endpoint**: `POST /api/v1/memory/search` + +```python +{ + "project_id": "myapp", + "query": "database decisions", + "memory_type": "decision", + "tags": ["database"], + "min_importance": 0.7, + "limit": 10 +} +``` + +**Search Strategy**: + +```cypher +// Fulltext search with filters +CALL db.index.fulltext.queryNodes('memory_search', $query) +YIELD node as m, score + +WHERE m.project_id = $project_id + AND m.is_active = true + AND ($memory_type IS NULL OR m.memory_type = $memory_type) + AND m.importance >= $min_importance + AND ($tags IS NULL OR ANY(tag IN $tags WHERE tag IN m.tags)) + +RETURN m, score +ORDER BY score DESC, m.importance DESC +LIMIT $limit +``` + +**Result Ranking**: +```python +def rank_memories(results: List[dict]) -> List[dict]: + """Rank by search score and importance""" + for result in results: + result['rank_score'] = ( + result['search_score'] * 0.6 + + result['importance'] * 0.4 + ) + return sorted(results, key=lambda x: x['rank_score'], reverse=True) +``` + +### Memory Evolution (Supersede) + +**Supersede Flow**: + +```mermaid +graph TB + Old[Old Memory: Use MySQL] --> Decision{Decision Changed} + Decision --> New[New Memory: Use PostgreSQL] + New --> Link[Create SUPERSEDES Relationship] + Link --> Deactivate[Mark Old as Superseded] + Deactivate --> History[Maintain History Chain] + + style Old fill:#FFE6E6 + style New fill:#E6FFE6 +``` + +**Implementation**: +```cypher +// 1. Create new memory +CREATE (new:Memory { + id: $new_id, + title: $new_title, + content: $new_content, + ... +}) + +// 2. Link to old memory +MATCH (old:Memory {id: $old_id}) +CREATE (new)-[:SUPERSEDES]->(old) + +// 3. Update old memory +SET old.superseded_by = $new_id, + old.is_active = false + +RETURN new, old +``` + +**History Traversal**: +```cypher +// Get full evolution history +MATCH path = (latest:Memory)-[:SUPERSEDES*]->(oldest:Memory) +WHERE latest.id = $memory_id +RETURN nodes(path) as history +``` + +## Code Graph Ingestion Pipeline + +### Overview + +Converting code repositories into queryable graph structure: + +```mermaid +graph TB + Start([Repository Path]) --> Scan[Scan Directory Tree] + Scan --> Filter[Filter by Language] + Filter --> Parse[Parse Each File] + + Parse --> AST[Generate AST] + AST --> Extract[Extract Symbols] + Extract --> Analyze[Analyze Relationships] + + Analyze --> CreateNodes[Create Graph Nodes] + CreateNodes --> CreateRels[Create Relationships] + CreateRels --> Index[Create Indexes] + Index --> Complete([Repository Indexed]) + + style Parse fill:#E6F3FF + style Extract fill:#FFF4E6 + style CreateNodes fill:#F0E6FF +``` + +### Step-by-Step Flow + +#### 1. Repository Scanning + +**API Endpoint**: `POST /api/v1/code/ingest` + +```python +{ + "repo_path": "/path/to/repository", + "repo_name": "myapp", + "file_patterns": ["*.py", "*.js"] # Optional filters +} +``` + +**Directory Walk**: +```python +def scan_repository(repo_path: str) -> List[str]: + """Scan directory and collect files""" + files = [] + + for root, dirs, filenames in os.walk(repo_path): + # Skip common ignored directories + dirs[:] = [d for d in dirs if d not in { + '.git', 'node_modules', '__pycache__', '.venv' + }] + + for filename in filenames: + file_path = os.path.join(root, filename) + if should_process_file(file_path): + files.append(file_path) + + return files +``` + +**Language Detection**: +```python +LANGUAGE_EXTENSIONS = { + '.py': 'python', + '.js': 'javascript', + '.ts': 'typescript', + '.java': 'java', + '.go': 'go', + '.cpp': 'cpp', + '.sql': 'sql' +} + +language = LANGUAGE_EXTENSIONS.get(file_extension) +``` + +#### 2. File Parsing + +**Python Parsing Example**: + +```python +import ast + +def parse_python_file(file_path: str) -> ParseResult: + """Parse Python file using AST""" + with open(file_path, 'r') as f: + source = f.read() + + tree = ast.parse(source, filename=file_path) + + symbols = [] + relationships = [] + + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + # Extract function + symbols.append({ + 'type': 'function', + 'name': node.name, + 'line': node.lineno, + 'params': [arg.arg for arg in node.args.args], + 'decorators': [d.id for d in node.decorator_list if isinstance(d, ast.Name)] + }) + + # Extract function calls + for child in ast.walk(node): + if isinstance(child, ast.Call): + if isinstance(child.func, ast.Name): + relationships.append({ + 'from': node.name, + 'to': child.func.id, + 'type': 'CALLS' + }) + + elif isinstance(node, ast.ClassDef): + # Extract class + symbols.append({ + 'type': 'class', + 'name': node.name, + 'line': node.lineno, + 'bases': [base.id for base in node.bases if isinstance(base, ast.Name)], + 'methods': [m.name for m in node.body if isinstance(m, ast.FunctionDef)] + }) + + # Extract inheritance + for base in node.bases: + if isinstance(base, ast.Name): + relationships.append({ + 'from': node.name, + 'to': base.id, + 'type': 'INHERITS' + }) + + return ParseResult(symbols=symbols, relationships=relationships) +``` + +**JavaScript Parsing Example**: + +```python +# Using esprima or similar parser +import esprima + +def parse_javascript_file(file_path: str) -> ParseResult: + """Parse JavaScript file""" + with open(file_path, 'r') as f: + source = f.read() + + tree = esprima.parseScript(source, {'loc': True}) + + # Extract functions, classes, imports + # Similar to Python parsing +``` + +#### 3. Graph Construction + +**Create Repository Node**: +```cypher +CREATE (r:Repo { + id: $repo_id, + name: $repo_name, + path: $repo_path, + created_at: datetime(), + file_count: $file_count +}) +``` + +**Create File Nodes**: +```cypher +CREATE (f:File { + repoId: $repo_id, + path: $file_path, + lang: $language, + content: $source_code, + lines: $line_count, + last_modified: datetime() +}) +CREATE (f)-[:BELONGS_TO]->(r:Repo {id: $repo_id}) +``` + +**Create Symbol Nodes**: +```cypher +// Function +CREATE (s:Symbol:Function { + id: $symbol_id, + name: $function_name, + type: 'function', + line: $line_number, + params: $parameters, + file_path: $file_path +}) +CREATE (s)-[:DEFINED_IN]->(f:File {path: $file_path}) + +// Class +CREATE (s:Symbol:Class { + id: $symbol_id, + name: $class_name, + type: 'class', + line: $line_number, + methods: $method_list, + file_path: $file_path +}) +CREATE (s)-[:DEFINED_IN]->(f:File {path: $file_path}) +``` + +**Create Relationships**: +```cypher +// Function calls +MATCH (caller:Function {name: $caller_name}) +MATCH (callee:Function {name: $callee_name}) +CREATE (caller)-[:CALLS]->(callee) + +// Class inheritance +MATCH (child:Class {name: $child_name}) +MATCH (parent:Class {name: $parent_name}) +CREATE (child)-[:INHERITS]->(parent) + +// Module imports +MATCH (file:File {path: $importer_path}) +MATCH (imported:File {path: $imported_path}) +CREATE (file)-[:IMPORTS]->(imported) +``` + +#### 4. Progress Tracking + +**Task Progress Updates**: +```python +async def ingest_repository_with_progress(repo_path: str, task_id: str): + """Ingest with progress updates""" + files = scan_repository(repo_path) + total_files = len(files) + + for i, file_path in enumerate(files): + # Parse file + result = parse_file(file_path) + + # Store in graph + await store_file_in_graph(result) + + # Update progress + progress = (i + 1) / total_files + await task_queue.update_progress( + task_id, + progress, + f"Processed {i+1}/{total_files} files" + ) +``` + +#### 5. Code Graph Queries + +**Find Related Code**: +```cypher +// Find all functions that call a specific function +MATCH (caller:Function)-[:CALLS]->(target:Function {name: $function_name}) +RETURN caller.name, caller.file_path + +// Find class hierarchy +MATCH path = (child:Class)-[:INHERITS*]->(parent:Class {name: $class_name}) +RETURN nodes(path) + +// Find all files that import a module +MATCH (file:File)-[:IMPORTS]->(module:File {path: $module_path}) +RETURN file.path +``` + +**Impact Analysis**: +```cypher +// Find all code affected by changing a function +MATCH (target:Function {name: $function_name}) +MATCH path = (dependent:Function)-[:CALLS*1..3]->(target) +RETURN DISTINCT dependent.name, dependent.file_path, length(path) as depth +ORDER BY depth +``` + +## Task Processing Pipeline + +### Overview + +Asynchronous task lifecycle: + +```mermaid +stateDiagram-v2 + [*] --> Submitted: User submits task + Submitted --> Stored: Store in SQLite + Stored --> Pending: Queue for processing + + Pending --> Locked: Worker locks task + Locked --> Processing: Worker starts processing + + Processing --> Success: Completed + Processing --> Failed: Error + Processing --> Cancelled: User cancels + + Success --> [*] + Failed --> [*] + Cancelled --> [*] + + Processing --> Processing: Progress updates +``` + +### Task Submission Flow + +**Submit Document Task**: + +```python +async def submit_document_processing_task( + content: str, + title: str, + metadata: dict +) -> str: + """Submit document processing as background task""" + # 1. Create task data + task_data = { + 'type': 'document_processing', + 'content': content, + 'title': title, + 'metadata': metadata + } + + # 2. Submit to queue + task_id = await task_queue.submit_task( + task_func=process_document_task, + task_type='document_processing', + task_data=task_data + ) + + return task_id +``` + +**Task Storage**: + +```python +# Store in SQLite +await task_storage.store_task( + task_id=task_id, + status=TaskStatus.PENDING, + task_type='document_processing', + metadata=task_data +) +``` + +### Worker Processing Flow + +**Worker Loop**: + +```python +async def _process_pending_tasks(self): + """Background worker continuously processes tasks""" + while True: + try: + # 1. Get pending tasks (with lock) + pending = await self._storage.get_pending_tasks(limit=10) + + # 2. Process each task (respecting concurrency limit) + for task in pending: + if len(self.running_tasks) < self.max_concurrent_tasks: + # Start task processing + asyncio.create_task(self._execute_task(task)) + + # 3. Wait before next poll + await asyncio.sleep(1) + + except asyncio.CancelledError: + logger.info("Worker shutting down") + break + except Exception as e: + logger.error(f"Worker error: {e}") + await asyncio.sleep(5) # Back off on error +``` + +**Task Execution**: + +```python +async def _execute_task(self, task: TaskResult): + """Execute single task""" + async with self.task_semaphore: # Limit concurrency + try: + # 1. Update status + task.status = TaskStatus.PROCESSING + task.started_at = datetime.now() + await self._storage.update_task_status(task.task_id, task.status) + + # 2. Get appropriate processor + processor = processor_registry.get_processor(task.metadata['type']) + + # 3. Execute with progress updates + result = await processor.process(task) + + # 4. Mark success + task.status = TaskStatus.SUCCESS + task.result = result + task.completed_at = datetime.now() + task.progress = 1.0 + + except Exception as e: + # Mark failed + task.status = TaskStatus.FAILED + task.error = str(e) + task.completed_at = datetime.now() + logger.error(f"Task {task.task_id} failed: {e}") + + finally: + # Save final state + await self._storage.update_task(task) + self.running_tasks.pop(task.task_id, None) +``` + +### Real-time Monitoring + +**SSE Stream**: + +```python +@app.get("/api/v1/sse/task/{task_id}") +async def stream_task_progress(task_id: str): + """Stream task progress via Server-Sent Events""" + async def event_generator(): + while True: + # Get current task status + task = task_queue.get_task(task_id) + + if task: + # Send update + yield { + "event": "progress", + "data": json.dumps({ + "task_id": task.task_id, + "status": task.status.value, + "progress": task.progress, + "message": task.message + }) + } + + # Stop if terminal state + if task.status in {TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.CANCELLED}: + break + + await asyncio.sleep(0.5) # Poll interval + + return EventSourceResponse(event_generator()) +``` + +**WebSocket Updates**: + +```python +@app.websocket("/api/v1/ws/task/{task_id}") +async def task_websocket(websocket: WebSocket, task_id: str): + """WebSocket for real-time task updates""" + await websocket.accept() + + try: + while True: + task = task_queue.get_task(task_id) + + if task: + # Send update + await websocket.send_json({ + "task_id": task.task_id, + "status": task.status.value, + "progress": task.progress, + "message": task.message + }) + + # Stop if done + if task.status in {TaskStatus.SUCCESS, TaskStatus.FAILED}: + break + + await asyncio.sleep(0.5) + + except WebSocketDisconnect: + logger.info(f"Client disconnected from task {task_id}") +``` + +## Real-time Update Pipeline + +### Architecture + +```mermaid +graph TB + Task[Task Processing] --> Update[Progress Update] + Update --> Queue[Update Queue] + + Queue --> SSE[SSE Connections] + Queue --> WS[WebSocket Connections] + Queue --> MCP[MCP Watch Tools] + + SSE --> Client1[Browser Client] + WS --> Client2[Web UI] + MCP --> Client3[Claude Desktop] + + style Task fill:#E6F3FF + style Queue fill:#FFF4E6 +``` + +### Update Flow + +**1. Task Progress Update**: + +```python +# Inside task processor +await task_queue.update_progress( + task_id=task_id, + progress=0.5, + message="Processed 50% of documents" +) +``` + +**2. Broadcast to Listeners**: + +```python +class TaskQueue: + async def update_progress(self, task_id: str, progress: float, message: str): + """Update progress and notify listeners""" + # Update task + task = self.tasks[task_id] + task.progress = progress + task.message = message + + # Store in database + await self._storage.update_task(task) + + # Notify SSE listeners + await self._notify_sse_listeners(task_id, task) + + # Notify WebSocket listeners + await self._notify_ws_listeners(task_id, task) +``` + +**3. SSE Delivery**: + +```python +async def _notify_sse_listeners(self, task_id: str, task: TaskResult): + """Send update to SSE clients""" + if task_id in self.sse_listeners: + event_data = { + "task_id": task_id, + "status": task.status.value, + "progress": task.progress, + "message": task.message + } + + for queue in self.sse_listeners[task_id]: + await queue.put(event_data) +``` + +## MCP Request Pipeline + +### Overview + +Model Context Protocol request handling: + +```mermaid +sequenceDiagram + participant Client as Claude Desktop + participant MCP as MCP Server + participant Handler as Tool Handler + participant Service as Service Layer + participant Neo4j + + Client->>MCP: Call tool (query_knowledge) + MCP->>MCP: Validate request + MCP->>Handler: Route to handler + Handler->>Service: Call knowledge service + Service->>Neo4j: Execute query + Neo4j-->>Service: Results + Service-->>Handler: Processed result + Handler->>Handler: Format response + Handler-->>MCP: Tool result + MCP-->>Client: Return response +``` + +### Request Flow + +**1. Tool Invocation**: + +```python +# Client (Claude Desktop) calls tool +{ + "method": "tools/call", + "params": { + "name": "query_knowledge", + "arguments": { + "question": "How does authentication work?", + "top_k": 5 + } + } +} +``` + +**2. Server Routing**: + +```python +@app.call_tool() +async def call_tool(name: str, arguments: dict) -> Sequence[TextContent]: + """Route tool call to appropriate handler""" + # Get handler for tool + handler = TOOL_HANDLERS.get(name) + + if not handler: + raise ValueError(f"Unknown tool: {name}") + + # Execute handler + result = await handler(arguments) + + # Format response + return [TextContent( + type="text", + text=format_result(result) + )] +``` + +**3. Handler Execution**: + +```python +async def handle_query_knowledge(arguments: dict) -> dict: + """Handle knowledge query""" + # Validate arguments + question = arguments.get("question") + if not question: + return {"success": False, "error": "Question required"} + + top_k = arguments.get("top_k", 5) + + # Call service + result = await neo4j_knowledge_service.query( + question=question, + top_k=top_k + ) + + return result +``` + +**4. Response Formatting**: + +```python +def format_result(result: dict) -> str: + """Format result for MCP client""" + if not result.get("success"): + return f"Error: {result.get('error', 'Unknown error')}" + + # Format answer with sources + answer = result.get("answer", "") + sources = result.get("sources", []) + + formatted = f"Answer:\n{answer}\n\n" + + if sources: + formatted += "Sources:\n" + for i, source in enumerate(sources, 1): + formatted += f"{i}. {source.get('title')} (relevance: {source.get('relevance', 0):.2f})\n" + + return formatted +``` + +## Data Flow Patterns + +### Pattern 1: Synchronous Request-Response + +**Use Case**: Quick operations (< 1 second) + +```mermaid +sequenceDiagram + Client->>API: POST /api/v1/memory/search + API->>MemoryStore: search_memories() + MemoryStore->>Neo4j: Cypher query + Neo4j-->>MemoryStore: Results + MemoryStore-->>API: Formatted results + API-->>Client: JSON response +``` + +**Example**: Memory search, graph queries, statistics + +### Pattern 2: Asynchronous Task Processing + +**Use Case**: Long operations (> 1 second) + +```mermaid +sequenceDiagram + Client->>API: POST /api/v1/documents/directory + API->>TaskQueue: submit_task() + TaskQueue->>SQLite: Store task + TaskQueue-->>API: task_id + API-->>Client: {"task_id": "..."} + + Note over TaskQueue,Worker: Background Processing + + Worker->>SQLite: Get pending tasks + Worker->>Processor: process_directory() + Processor->>KnowServ: Add documents + Processor->>TaskQueue: Update progress + + Client->>API: GET /api/v1/tasks/{task_id} + API->>TaskQueue: get_task() + TaskQueue-->>API: Task status + API-->>Client: {"status": "processing", "progress": 0.7} +``` + +**Example**: Directory processing, large document ingestion, repository ingestion + +### Pattern 3: Streaming Updates + +**Use Case**: Real-time progress monitoring + +```mermaid +sequenceDiagram + Client->>API: GET /api/v1/sse/task/{task_id} + API-->>Client: SSE connection established + + loop Every 500ms + Worker->>TaskQueue: Update progress + TaskQueue->>API: Broadcast update + API-->>Client: event: progress\ndata: {...} + end + + Worker->>TaskQueue: Complete task + TaskQueue->>API: Final update + API-->>Client: event: complete\ndata: {...} + API->>API: Close connection +``` + +**Example**: Task monitoring, batch operations, long-running queries + +### Pattern 4: Batch Processing + +**Use Case**: Multiple operations with coordination + +```mermaid +graph TB + Start([Batch Request]) --> Split[Split into Tasks] + Split --> T1[Task 1] + Split --> T2[Task 2] + Split --> T3[Task 3] + + T1 --> Process1[Process] + T2 --> Process2[Process] + T3 --> Process3[Process] + + Process1 --> Collect[Collect Results] + Process2 --> Collect + Process3 --> Collect + + Collect --> Aggregate[Aggregate] + Aggregate --> Complete([Return Combined Result]) +``` + +**Implementation**: +```python +async def batch_process_files(file_paths: List[str]) -> dict: + """Process multiple files in parallel""" + # Create tasks + tasks = [ + asyncio.create_task(process_file(path)) + for path in file_paths + ] + + # Wait for all tasks + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Aggregate results + successful = [r for r in results if not isinstance(r, Exception)] + failed = [r for r in results if isinstance(r, Exception)] + + return { + "total": len(file_paths), + "successful": len(successful), + "failed": len(failed), + "results": successful + } +``` + +## Performance Considerations + +### Database Query Optimization + +**Use Indexes**: +```cypher +// Always use indexed properties in WHERE clauses +MATCH (m:Memory) +WHERE m.id = $id // Uses unique constraint index +RETURN m + +// vs. + +MATCH (m:Memory) +WHERE m.content CONTAINS $text // Full scan - slow! +RETURN m +``` + +**Limit Result Sets**: +```cypher +// Always use LIMIT +MATCH (d:Document) +RETURN d +LIMIT 100 // Prevent returning entire database +``` + +**Use Query Hints**: +```cypher +// Force index usage +MATCH (m:Memory) +USING INDEX m:Memory(id) +WHERE m.id = $id +RETURN m +``` + +### Embedding Optimization + +**Batch Embeddings**: +```python +# Instead of one at a time +for text in texts: + embedding = await embed_model.get_text_embedding(text) + +# Batch process +embeddings = await embed_model.get_text_embeddings(texts) +``` + +**Cache Embeddings**: +```python +# Cache by content hash +content_hash = hashlib.sha256(text.encode()).hexdigest() +if content_hash in embedding_cache: + return embedding_cache[content_hash] +``` + +### Task Queue Optimization + +**Concurrency Tuning**: +```python +# Adjust based on resources +task_queue = TaskQueue(max_concurrent_tasks=5) # CPU-bound +task_queue = TaskQueue(max_concurrent_tasks=20) # I/O-bound +``` + +**Task Prioritization**: +```python +# High priority tasks first +async def get_pending_tasks(self, priority_order: bool = True): + if priority_order: + query = "ORDER BY priority DESC, created_at ASC" + else: + query = "ORDER BY created_at ASC" +``` + +## Error Handling Patterns + +### Retry Pattern + +```python +async def retry_on_error(func, max_retries=3, backoff=2): + """Retry with exponential backoff""" + for attempt in range(max_retries): + try: + return await func() + except TransientError as e: + if attempt == max_retries - 1: + raise + wait_time = backoff ** attempt + await asyncio.sleep(wait_time) +``` + +### Circuit Breaker + +```python +class CircuitBreaker: + """Prevent cascading failures""" + def __init__(self, failure_threshold=5, timeout=60): + self.failure_count = 0 + self.failure_threshold = failure_threshold + self.timeout = timeout + self.last_failure_time = None + self.state = "CLOSED" # CLOSED, OPEN, HALF_OPEN + + async def call(self, func): + if self.state == "OPEN": + if time.time() - self.last_failure_time > self.timeout: + self.state = "HALF_OPEN" + else: + raise CircuitBreakerOpen() + + try: + result = await func() + if self.state == "HALF_OPEN": + self.state = "CLOSED" + self.failure_count = 0 + return result + except Exception as e: + self.failure_count += 1 + self.last_failure_time = time.time() + + if self.failure_count >= self.failure_threshold: + self.state = "OPEN" + raise +``` + +## Conclusion + +The data flow architecture is designed for: + +1. **Efficiency**: Minimize processing time and resource usage +2. **Reliability**: Handle errors gracefully with retries and fallbacks +3. **Scalability**: Support both sync and async patterns +4. **Observability**: Track all data transformations and state changes +5. **Flexibility**: Support multiple ingestion and query patterns + +Understanding these flows enables: +- Effective debugging +- Performance optimization +- System extension +- Operational monitoring diff --git a/docs/architecture/design.md b/docs/architecture/design.md new file mode 100644 index 0000000..0ccf50b --- /dev/null +++ b/docs/architecture/design.md @@ -0,0 +1,959 @@ +# System Design and Architecture + +## Table of Contents + +- [Overview](#overview) +- [Architecture Tiers](#architecture-tiers) +- [Design Philosophy](#design-philosophy) +- [System Architecture](#system-architecture) +- [Technology Stack](#technology-stack) +- [Design Decisions](#design-decisions) +- [Scalability Considerations](#scalability-considerations) +- [Security Architecture](#security-architecture) + +## Overview + +Code Graph Knowledge System is a Neo4j-based intelligent knowledge management system that combines: + +- **Vector Search**: Semantic similarity search using embeddings +- **Graph Database**: Relationship-based knowledge representation +- **LLM Integration**: Multiple provider support for AI-powered features +- **RAG (Retrieval Augmented Generation)**: Context-aware question answering +- **Code Graph Analysis**: Repository structure and dependency analysis +- **Memory Management**: Persistent project knowledge for AI agents + +The system is designed as a **multi-tier architecture** where each tier builds upon the previous one, allowing users to adopt capabilities incrementally based on their needs. + +## Architecture Tiers + +The system implements a three-tier architecture, each providing distinct capabilities: + +```mermaid +graph TB + subgraph "Tier 1: Minimal - Code Graph" + T1[Code Graph Service] + T1A[Repository Ingestion] + T1B[Code Search] + T1C[Impact Analysis] + T1D[Context Pack Generation] + end + + subgraph "Tier 2: Standard - Memory" + T2[Memory Store Service] + T2A[Decision Tracking] + T2B[Preference Management] + T2C[Experience Recording] + T2D[Memory Extraction] + end + + subgraph "Tier 3: Full - Knowledge RAG" + T3[Knowledge Service] + T3A[Document Processing] + T3B[Vector Search] + T3C[RAG Query Engine] + T3D[Graph Relationships] + end + + T1 --> T2 + T2 --> T3 + + style T1 fill:#e1f5e1 + style T2 fill:#e3f2fd + style T3 fill:#fff9e6 +``` + +### Tier 1: Minimal (Code Graph) + +**Purpose**: Static code analysis and repository understanding + +**Components**: +- Code ingestor with multi-language support +- Graph-based code structure representation +- Symbol relationship tracking +- Impact analysis engine + +**Use Cases**: +- Understanding codebase structure +- Finding related code components +- Analyzing change impact +- Generating context for AI tools + +**Resource Requirements**: Low (minimal LLM usage) + +### Tier 2: Standard (+ Memory) + +**Purpose**: Project knowledge persistence for AI agents + +**Components**: +- Memory Store service with typed memories +- Search and retrieval system +- Automatic extraction from commits/comments +- Memory evolution tracking (supersede mechanism) + +**Use Cases**: +- Recording architectural decisions +- Tracking team preferences +- Learning from past problems +- Maintaining consistency across sessions + +**Resource Requirements**: Medium (LLM for extraction features) + +### Tier 3: Full (+ Knowledge RAG) + +**Purpose**: Intelligent document processing and question answering + +**Components**: +- LlamaIndex-based knowledge graph +- Vector embedding generation +- Multi-source document ingestion +- RAG query engine with graph traversal + +**Use Cases**: +- Natural language querying +- Document-based question answering +- Cross-document knowledge synthesis +- Semantic search across knowledge base + +**Resource Requirements**: High (intensive LLM and embedding usage) + +## Design Philosophy + +### 1. Progressive Complexity + +The tier-based architecture allows users to: +- Start with minimal features (Code Graph only) +- Add memory capabilities when needed +- Enable full RAG when ready for advanced features + +**Trade-off**: Increased system complexity vs. flexibility + +### 2. Multi-Provider Support + +Support for multiple LLM and embedding providers: +- **Ollama**: Local deployment, privacy-focused +- **OpenAI**: High quality, cloud-based +- **Google Gemini**: Competitive performance +- **OpenRouter**: Access to multiple models +- **HuggingFace**: Open-source embeddings + +**Trade-off**: More configuration complexity vs. vendor flexibility + +### 3. Async-First Design + +All I/O operations are asynchronous: +- Non-blocking request handling +- Background task processing +- Concurrent operation support + +**Trade-off**: Programming complexity vs. performance + +### 4. Service-Oriented Architecture + +Clear separation of concerns: +- Each service has a single responsibility +- Services communicate through well-defined interfaces +- Easy to test and maintain + +**Trade-off**: More files/modules vs. maintainability + +## System Architecture + +### High-Level Architecture + +```mermaid +graph TB + subgraph "Client Layer" + HTTP[HTTP/REST Clients] + MCP[MCP Clients
Claude Desktop, VSCode] + UI[Web UI
Monitoring Interface] + end + + subgraph "API Layer" + FastAPI[FastAPI Server] + MCPS[MCP Server
Official SDK] + SSE[Server-Sent Events] + WS[WebSocket] + end + + subgraph "Service Layer" + KS[Knowledge Service
LlamaIndex + Neo4j] + MS[Memory Store
Project Knowledge] + GS[Graph Service
Code Analysis] + TQ[Task Queue
Async Processing] + ME[Memory Extractor
Auto-extraction] + end + + subgraph "Storage Layer" + Neo4j[(Neo4j Graph DB
Vector Index)] + SQLite[(SQLite
Task Persistence)] + FS[File System
Temp Files] + end + + subgraph "External Services" + LLM[LLM Providers
Ollama/OpenAI/Gemini] + Embed[Embedding Models
Vector Generation] + end + + HTTP --> FastAPI + MCP --> MCPS + UI --> FastAPI + + FastAPI --> KS + FastAPI --> MS + FastAPI --> GS + FastAPI --> TQ + FastAPI --> SSE + FastAPI --> WS + + MCPS --> KS + MCPS --> MS + MCPS --> GS + MCPS --> TQ + + KS --> Neo4j + MS --> Neo4j + GS --> Neo4j + TQ --> SQLite + TQ --> FS + ME --> MS + + KS --> LLM + KS --> Embed + MS --> LLM + ME --> LLM + + style FastAPI fill:#4CAF50 + style MCPS fill:#2196F3 + style Neo4j fill:#f9a825 + style LLM fill:#9C27B0 +``` + +### Component Layers + +#### 1. Client Layer + +**HTTP/REST Clients**: +- Standard HTTP requests +- JSON-based communication +- OpenAPI/Swagger documentation + +**MCP Clients**: +- Claude Desktop integration +- VSCode with MCP extension +- Custom MCP client implementations +- Uses official MCP SDK protocol + +**Web UI**: +- Real-time monitoring interface (NiceGUI) +- Task status visualization +- File upload and processing +- WebSocket-based updates + +#### 2. API Layer + +**FastAPI Server** (`main.py`, `core/app.py`): +- RESTful API endpoints +- Async request handling +- CORS middleware +- GZip compression +- Exception handling + +**MCP Server** (`mcp_server.py`, `start_mcp.py`): +- 30 tools across 6 categories +- Official MCP SDK implementation +- Session management +- Streaming support +- Multi-transport (stdio, SSE, WebSocket) + +**Real-time Communication**: +- Server-Sent Events for task monitoring +- WebSocket for UI updates +- Streaming responses for long operations + +#### 3. Service Layer + +**Knowledge Service** (`services/neo4j_knowledge_service.py`): +- LlamaIndex KnowledgeGraphIndex integration +- Vector embedding generation +- Document processing and chunking +- RAG query engine + +**Memory Store** (`services/memory_store.py`): +- Project knowledge persistence +- Typed memory system (decision/preference/experience/convention/plan/note) +- Search with filters and importance scoring +- Memory evolution (supersede mechanism) + +**Graph Service** (`services/graph_service.py`): +- Code graph management +- Cypher query execution +- Schema management +- Relationship traversal + +**Task Queue** (`services/task_queue.py`): +- Async background processing +- SQLite-based persistence +- Concurrent task limiting +- Status tracking and updates + +**Memory Extractor** (`services/memory_extractor.py`): +- Conversation analysis +- Git commit mining +- Code comment extraction +- Batch repository analysis + +#### 4. Storage Layer + +**Neo4j Graph Database**: +- Knowledge graph storage +- Native vector indexing +- Relationship management +- Fulltext search indexes + +**SQLite Database**: +- Task queue persistence +- Task status tracking +- Worker coordination + +**File System**: +- Temporary file storage +- Large document handling +- Upload processing + +## Technology Stack + +### Core Framework + +```yaml +Web Framework: + - FastAPI: Async web framework + - Uvicorn: ASGI server + - Pydantic: Data validation + +MCP Integration: + - mcp>=1.1.0: Official Model Context Protocol SDK + - Custom handlers: Modular tool organization +``` + +### Database & Storage + +```yaml +Graph Database: + - Neo4j 5.0+: Graph and vector storage + - APOC plugin: Advanced procedures + - Native vector index: Semantic search + +Task Persistence: + - SQLite: Lightweight task storage + - Async driver: Non-blocking operations +``` + +### AI & ML + +```yaml +LLM Integration: + - LlamaIndex: RAG framework + - Ollama: Local LLM hosting + - OpenAI: GPT models + - Google Gemini: Gemini models + - OpenRouter: Multi-provider access + +Embedding Models: + - Ollama: nomic-embed-text + - OpenAI: text-embedding-ada-002 + - Gemini: models/embedding-001 + - HuggingFace: BAAI/bge-small-en-v1.5 +``` + +### Developer Tools + +```yaml +Code Quality: + - Black: Code formatting + - isort: Import sorting + - Ruff: Fast linting + - pytest: Testing framework + +Monitoring: + - Loguru: Structured logging + - NiceGUI: Web monitoring UI + - SSE: Real-time updates +``` + +## Design Decisions + +### 1. Neo4j as Primary Database + +**Decision**: Use Neo4j for all persistent storage (knowledge, memory, code graph) + +**Rationale**: +- Native graph queries for relationships +- Built-in vector indexing (v5.0+) +- Fulltext search capabilities +- ACID compliance +- Scales well for graph traversal + +**Trade-offs**: +- More complex than traditional SQL +- Requires Neo4j infrastructure +- Learning curve for Cypher queries +- Higher memory usage + +**Alternatives Considered**: +- PostgreSQL + pgvector: Good but weaker graph queries +- Separate vector DB (Pinecone/Weaviate): Additional infrastructure +- MongoDB: Poor relationship handling + +### 2. LlamaIndex for RAG + +**Decision**: Use LlamaIndex's KnowledgeGraphIndex + +**Rationale**: +- Production-ready RAG framework +- Neo4j integration out-of-the-box +- Flexible node parser system +- Active development and community + +**Trade-offs**: +- Additional abstraction layer +- Some LlamaIndex-specific patterns +- Updates may require code changes + +**Alternatives Considered**: +- LangChain: More complex, heavier +- Custom RAG: More control but more work +- Haystack: Less graph-oriented + +### 3. Async Task Queue + +**Decision**: Custom async task queue with SQLite persistence + +**Rationale**: +- Simple deployment (no external queue) +- Sufficient for single-server deployment +- Task persistence across restarts +- Direct integration with FastAPI + +**Trade-offs**: +- Not distributed (single server only) +- Limited throughput vs. Redis/RabbitMQ +- SQLite lock contention possible + +**Alternatives Considered**: +- Celery + Redis: Overkill for single server +- RQ: Still requires Redis +- Dramatiq: More dependencies + +### 4. Multi-Provider LLM Support + +**Decision**: Support multiple LLM and embedding providers + +**Rationale**: +- Vendor independence +- Local deployment option (Ollama) +- Cost optimization +- Feature comparison capability + +**Trade-offs**: +- More configuration complexity +- Testing burden across providers +- Inconsistent behavior possible + +**Alternatives Considered**: +- Single provider (OpenAI): Simple but vendor lock-in +- LiteLLM proxy: Additional component + +### 5. MCP Server with Official SDK + +**Decision**: Migrate from FastMCP to official MCP SDK + +**Rationale**: +- Official protocol compliance +- Better long-term support +- Advanced features (streaming, sessions) +- Industry standard + +**Trade-offs**: +- More verbose code +- Lower-level API +- Migration effort required + +**Alternatives Considered**: +- Keep FastMCP: Simpler but less standard +- Direct HTTP API only: Miss Claude Desktop integration + +### 6. Tier-Based Architecture + +**Decision**: Three-tier progressive architecture + +**Rationale**: +- Gradual adoption curve +- Cost optimization (use only what's needed) +- Clear feature boundaries +- Independent scaling + +**Trade-offs**: +- More complex initialization +- Feature interdependencies +- Documentation overhead + +**Alternatives Considered**: +- All-or-nothing: Simpler but less flexible +- Plugin system: More complex + +## Scalability Considerations + +### Current Architecture (Single Server) + +**Designed for**: +- Small to medium teams (1-50 users) +- Moderate query volume (<1000 req/hour) +- Single deployment instance +- Shared Neo4j database + +**Bottlenecks**: +1. Neo4j connection pool +2. Task queue concurrency limit +3. LLM API rate limits +4. Memory constraints for large documents + +### Horizontal Scaling Path + +```mermaid +graph TB + subgraph "Load Balancer" + LB[Nginx / HAProxy] + end + + subgraph "API Servers" + API1[FastAPI Instance 1] + API2[FastAPI Instance 2] + API3[FastAPI Instance N] + end + + subgraph "MCP Servers" + MCP1[MCP Instance 1] + MCP2[MCP Instance 2] + end + + subgraph "Shared Services" + Neo4j[(Neo4j Cluster)] + Redis[(Redis
Task Queue)] + S3[Object Storage
Documents] + end + + LB --> API1 + LB --> API2 + LB --> API3 + + API1 --> Neo4j + API2 --> Neo4j + API3 --> Neo4j + + API1 --> Redis + API2 --> Redis + API3 --> Redis + + API1 --> S3 + API2 --> S3 + API3 --> S3 + + MCP1 --> Neo4j + MCP2 --> Neo4j +``` + +**Required Changes**: +1. Replace SQLite task queue with Redis/RabbitMQ +2. Use object storage (S3/MinIO) for file uploads +3. Session management with Redis +4. Neo4j clustering for HA +5. Shared cache layer + +### Vertical Scaling + +**Immediate Improvements**: +- Increase Neo4j memory (`dbms.memory.heap.max_size`) +- Tune vector index parameters +- Optimize chunk sizes +- Add Redis caching layer +- Use faster embedding models + +### Performance Optimization + +**Database Level**: +```cypher +// Ensure proper indexes exist +CREATE INDEX IF NOT EXISTS FOR (n:Document) ON (n.id); +CREATE INDEX IF NOT EXISTS FOR (m:Memory) ON (m.project_id, m.importance); +CREATE FULLTEXT INDEX IF NOT EXISTS FOR (m:Memory) ON EACH [m.title, m.content]; + +// Vector index configuration +CALL db.index.vector.createNodeIndex( + 'knowledge_vectors', + 'Document', + 'embedding', + 1536, // Dimension + 'cosine' +); +``` + +**Application Level**: +- Connection pooling +- Query result caching +- Batch operations +- Async I/O everywhere +- Background task offloading + +## Security Architecture + +### Authentication & Authorization + +**Current Implementation**: +- Optional API key authentication +- Environment-based configuration +- No user management (designed for internal use) + +**Production Recommendations**: +```yaml +Authentication: + - API key per user/service + - JWT tokens for session management + - OAuth2 for third-party integration + +Authorization: + - Role-based access control (RBAC) + - Project-level permissions + - Rate limiting per API key +``` + +### Data Security + +**At Rest**: +- Neo4j encryption (`dbms.security.encryption.enabled=true`) +- Environment variable encryption +- Secrets management (AWS Secrets Manager, Vault) + +**In Transit**: +- TLS/HTTPS for all HTTP traffic +- Neo4j Bolt encryption +- Secure WebSocket (WSS) + +**Code Security**: +```python +# Input validation with Pydantic +class DocumentAddRequest(BaseModel): + content: str = Field(..., max_length=10_000_000) + title: str = Field(..., max_length=200) + +# SQL injection prevention (parameterized queries) +await session.run( + "CREATE (d:Document {id: $id, title: $title})", + id=doc_id, title=title +) + +# XSS prevention (automatic escaping in FastAPI) +# CSRF protection for web UI +``` + +### Network Security + +**Recommended Deployment**: +```yaml +VPC Configuration: + - Private subnet for Neo4j + - Public subnet for API (behind ALB) + - Security groups for port control + +Firewall Rules: + - 8123: API access (restricted IPs) + - 7687: Neo4j Bolt (internal only) + - 7474: Neo4j Browser (VPN only) + +TLS Configuration: + - Minimum TLS 1.2 + - Strong cipher suites + - Certificate pinning for MCP +``` + +### Secrets Management + +**Environment Variables**: +```bash +# Required secrets +NEO4J_PASSWORD= +OPENAI_API_KEY= +GOOGLE_API_KEY= +API_KEY= + +# Use secrets manager +AWS_SECRETS_MANAGER_SECRET_ID=code-graph-prod +VAULT_ADDR=https://vault.company.com +``` + +**Best Practices**: +- Never commit secrets to version control +- Rotate API keys regularly +- Use managed secrets services in production +- Separate secrets per environment +- Audit secret access + +### Threat Model + +**Potential Threats**: +1. **Unauthorized Access**: API key leakage + - Mitigation: Strong keys, rotation, IP whitelisting + +2. **Data Injection**: Malicious document content + - Mitigation: Input validation, content sanitization + +3. **Resource Exhaustion**: Large document uploads + - Mitigation: Size limits, rate limiting, timeouts + +4. **Prompt Injection**: Malicious queries to LLM + - Mitigation: Input sanitization, output filtering + +5. **Data Leakage**: Sensitive information in graph + - Mitigation: Access controls, data classification + +**Security Checklist**: +- [ ] Enable Neo4j authentication +- [ ] Use HTTPS/TLS in production +- [ ] Implement API key authentication +- [ ] Set up rate limiting +- [ ] Enable CORS restrictions +- [ ] Configure file size limits +- [ ] Set up logging and monitoring +- [ ] Regular security updates +- [ ] Backup encryption +- [ ] Secrets rotation schedule + +## Monitoring & Observability + +### Logging Strategy + +**Structured Logging with Loguru**: +```python +logger.info("Document processed", + doc_id=doc_id, + size=len(content), + duration=elapsed_time +) +``` + +**Log Levels**: +- DEBUG: Detailed troubleshooting +- INFO: General operational events +- WARNING: Potential issues +- ERROR: Error conditions +- CRITICAL: System failures + +### Metrics Collection + +**Key Metrics**: +```yaml +Application Metrics: + - Request rate (req/sec) + - Response time (p50, p95, p99) + - Error rate (%) + - Task queue depth + - Active tasks count + +Database Metrics: + - Query execution time + - Connection pool usage + - Vector search latency + - Graph traversal depth + +LLM Metrics: + - API call duration + - Token usage + - Error rate per provider + - Cost tracking +``` + +### Health Checks + +**Endpoint**: `/api/v1/health` + +**Checks**: +- Neo4j connectivity +- LLM provider availability +- Task queue status +- Memory Store status + +**Example Response**: +```json +{ + "status": "healthy", + "timestamp": "2025-11-06T12:00:00Z", + "services": { + "neo4j": true, + "knowledge_service": true, + "memory_store": true, + "task_queue": true, + "ollama": true + }, + "version": "1.0.0" +} +``` + +### Alerting + +**Critical Alerts**: +- Service down (Neo4j, LLM provider) +- High error rate (>5%) +- Task queue backup (>100 pending) +- Disk space low (<10%) +- Memory usage high (>90%) + +**Warning Alerts**: +- Slow queries (>5s) +- High response time (>1s p95) +- LLM API errors +- Connection pool exhaustion + +## Disaster Recovery + +### Backup Strategy + +**Neo4j Backups**: +```bash +# Daily full backup +neo4j-admin database dump neo4j --to-path=/backups/$(date +%Y%m%d) + +# Incremental backup (Enterprise) +neo4j-admin database backup --backup-dir=/backups neo4j +``` + +**Task Queue Backups**: +```bash +# SQLite database backup +cp tasks.db /backups/tasks_$(date +%Y%m%d_%H%M%S).db +``` + +**Configuration Backups**: +- `.env` file (encrypted) +- Neo4j configuration +- Application configuration + +### Recovery Procedures + +**Full System Recovery**: +1. Restore Neo4j from backup +2. Restore SQLite database +3. Restore configuration files +4. Verify service connectivity +5. Resume task processing + +**Partial Recovery**: +- Knowledge graph: Restore from Neo4j backup +- Memory Store: Restore from Neo4j backup +- Tasks: Re-queue failed tasks + +**RTO/RPO Targets**: +- RTO (Recovery Time Objective): 4 hours +- RPO (Recovery Point Objective): 24 hours (daily backups) + +### High Availability + +**Single Point of Failure**: +- Neo4j database (can cluster in Enterprise) +- Application server (can load balance) +- LLM provider (multi-provider fallback) + +**Mitigation**: +```yaml +Neo4j Clustering: + - 3-node cluster minimum + - Automatic failover + - Read replicas for scaling + +Application: + - Multiple instances behind load balancer + - Stateless design for easy scaling + - Health check-based routing + +LLM Providers: + - Primary + fallback provider + - Automatic retry with exponential backoff + - Circuit breaker pattern +``` + +## Future Architecture Considerations + +### Potential Enhancements + +**1. Distributed Task Queue**: +```python +# Replace SQLite with Redis/RabbitMQ +from celery import Celery +app = Celery('tasks', broker='redis://localhost:6379') +``` + +**2. Caching Layer**: +```python +# Add Redis caching +from redis import asyncio as aioredis +cache = await aioredis.from_url("redis://localhost") +``` + +**3. API Gateway**: +```yaml +Kong/Tyk Configuration: + - Rate limiting + - Authentication + - Request transformation + - Analytics +``` + +**4. Microservices Split**: +``` +Current: Monolith +Future: + - knowledge-service + - memory-service + - code-graph-service + - task-worker-service +``` + +**5. Event-Driven Architecture**: +```python +# Event bus for service communication +from aiokafka import AIOKafkaProducer + +producer = AIOKafkaProducer(bootstrap_servers='localhost:9092') +await producer.send('document.processed', value=event_data) +``` + +### Technology Evolution + +**Short-term (3-6 months)**: +- Add Redis caching +- Implement comprehensive metrics +- Enhanced error handling +- Performance optimization + +**Mid-term (6-12 months)**: +- Kubernetes deployment +- Neo4j clustering +- Distributed tracing (Jaeger) +- Advanced monitoring (Prometheus + Grafana) + +**Long-term (12+ months)**: +- Microservices architecture +- Multi-region deployment +- GraphQL API option +- ML model serving infrastructure + +## Conclusion + +The Code Graph Knowledge System architecture is designed with these core principles: + +1. **Progressive Adoption**: Three-tier architecture allows gradual capability adoption +2. **Flexibility**: Multi-provider support for LLM and embeddings +3. **Scalability**: Clear path from single-server to distributed deployment +4. **Maintainability**: Service-oriented design with clear boundaries +5. **Performance**: Async-first design for optimal throughput +6. **Security**: Built-in security considerations for production use + +The architecture balances simplicity for initial deployment with clear paths for scaling and enhancement as needs grow. diff --git a/docs/assets/favicon.svg b/docs/assets/favicon.svg new file mode 100644 index 0000000..19e8d96 --- /dev/null +++ b/docs/assets/favicon.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + </> + diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 0000000..985b2e2 --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + </> + diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..5611071 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,475 @@ +# Changelog + +All notable changes to the Code Graph Knowledge System will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Planned Features +- API authentication with JWT +- Web-based configuration UI +- Multi-provider LLM support (simultaneous) +- Advanced code refactoring suggestions +- Rust, C++, C# language support +- Real-time collaboration features +- Plugin system for custom extensions + +## [0.7.0] - 2025-01-15 + +### Added - Automatic Memory Extraction +- **Conversation Analysis**: Extract memories from AI conversation history + - LLM-powered decision and experience detection + - Confidence scoring for automatic saving + - Configurable auto-save threshold +- **Git Commit Analysis**: Analyze git commits for architectural decisions + - Parse commit messages and changed files + - Extract decisions, experiences, and conventions + - Link memories to specific commits +- **Code Comment Mining**: Extract TODO, FIXME, NOTE, DECISION markers + - Automatic scanning of code comments + - Convert markers to structured memories + - Track technical debt and action items +- **Query-based Memory Suggestions**: Suggest important memories from Q&A + - Analyze knowledge base queries and answers + - Identify information worth remembering + - Suggest memory creation with auto-populated fields +- **Batch Repository Extraction**: Comprehensive codebase analysis + - Extract from git history (configurable commit limit) + - Mine code comments across file patterns + - Bulk memory creation from repository insights + +### Added - New MCP Tools (5 tools) +- `extract_from_conversation`: Extract memories from conversation history +- `extract_from_git_commit`: Analyze git commits for memories +- `extract_from_code_comments`: Mine code comments for action items +- `suggest_memory_from_query`: Suggest memories from Q&A sessions +- `batch_extract_from_repository`: Full repository analysis + +### Added - New API Endpoints (5 endpoints) +- `POST /api/v1/memory/extract/conversation`: Extract from conversations +- `POST /api/v1/memory/extract/commit`: Extract from git commits +- `POST /api/v1/memory/extract/comments`: Extract from code comments +- `POST /api/v1/memory/suggest`: Suggest memory from query/answer +- `POST /api/v1/memory/extract/batch`: Batch repository extraction + +### Changed +- Enhanced memory extraction service with LLM-powered analysis +- Improved error messages for memory operations +- Updated MCP handler architecture documentation +- Enhanced memory search relevance scoring + +### Fixed +- Neo4j connection timeout in Docker environments +- Memory search not finding recently added memories +- Environment variable handling in Docker deployment +- Race condition in concurrent memory additions + +### Documentation +- Added comprehensive memory extraction guide +- Updated API documentation with extraction endpoints +- New examples for automatic memory extraction +- Enhanced troubleshooting guide + +## [0.6.0] - 2024-12-20 + +### Added - Memory Store for AI Agents +- **Memory Management System**: Long-term project knowledge persistence + - Decision memory type: Architecture and technical choices + - Preference memory type: Coding styles and conventions + - Experience memory type: Problems encountered and solutions + - Convention memory type: Team rules and standards + - Plan memory type: Future improvements and TODOs + - Note memory type: General project information +- **Memory Operations**: Full CRUD operations for memories + - Add memory with importance scoring + - Search memories with semantic search + - Update existing memories + - Delete memories (soft delete) + - Supersede memories (version history) + - Project memory summaries +- **Memory Relationships**: Graph-based memory connections + - `BELONGS_TO`: Memory to project relationships + - `RELATES_TO`: Inter-memory relationships + - `SUPERSEDES`: Memory version history + +### Added - Multi-Language Support (3 languages) +- **Java Support**: Complete Java code analysis + - Import statement parsing (standard and static) + - Class inheritance and interface tracking + - Method visibility detection (public/protected/private) + - Package dependency mapping +- **PHP Support**: PHP code analysis + - Use statement parsing (class, function, const) + - Require/include dependency tracking + - Class extends and implements relationships + - Function type hint extraction +- **Go Support**: Golang code analysis + - Package import parsing (single and blocks) + - Struct and interface detection + - Function and method extraction (with receivers) + - Package alias tracking + +### Added - Docker Multi-Mode Deployment +- **Three deployment modes**: + - Minimal: Code Graph only (~800MB) + - Standard: Code Graph + Memory (~1.2GB) + - Full: All features (~1.5GB) +- **Docker Compose configurations**: + - `docker-compose.minimal.yml` + - `docker-compose.standard.yml` + - `docker-compose.full.yml` +- **Multi-platform support**: amd64, arm64 +- **Helper scripts**: Simplified deployment commands + +### Added - MCP Tools (7 memory tools) +- `add_memory`: Save new project knowledge +- `search_memories`: Find relevant memories +- `get_memory`: Retrieve specific memory +- `update_memory`: Modify existing memory +- `delete_memory`: Remove memory +- `supersede_memory`: Create new memory that replaces old one +- `get_project_summary`: Get project memory overview + +### Added - API Endpoints (7 memory endpoints) +- `POST /api/v1/memory/add`: Add new memory +- `POST /api/v1/memory/search`: Search memories +- `GET /api/v1/memory/{memory_id}`: Get specific memory +- `PUT /api/v1/memory/{memory_id}`: Update memory +- `DELETE /api/v1/memory/{memory_id}`: Delete memory +- `POST /api/v1/memory/supersede`: Supersede old memory +- `GET /api/v1/memory/project/{project_id}/summary`: Project summary + +### Changed +- Updated file patterns to include Java, PHP, Go files +- Enhanced code graph to support new language relationships +- Improved transformer architecture for multi-language support + +### Documentation +- Added Memory Store user guide +- Added memory API documentation +- Updated examples with memory usage +- Enhanced CLAUDE.md with memory workflows + +## [0.5.0] - 2024-11-15 + +### Added - MCP Protocol Support +- **Official MCP SDK Integration**: Model Context Protocol v1.1.0+ +- **Modular Architecture**: Handler-based design (310-line main server) + - Knowledge handlers: Query, search, document management + - Code graph handlers: Ingestion, analysis, statistics + - System handlers: Health checks, configuration + - Task handlers: Background processing, monitoring +- **30 MCP Tools**: Comprehensive AI assistant integration + - 8 knowledge tools + - 10 code graph tools + - 4 system tools + - 8 task monitoring tools +- **Advanced Features**: + - Session management framework + - Streaming support (SSE) + - Multi-transport capability (stdio, SSE, WebSocket) + +### Added - Prometheus Metrics +- **15+ metrics** for monitoring: + - Request counters (total, by endpoint, by status) + - Request duration histograms + - Active request gauges + - Neo4j operation metrics + - Document processing metrics + - Error rate tracking +- **Metrics endpoint**: `GET /api/v1/metrics` +- **Grafana dashboard** configuration (optional) + +### Added - Neo4j Health Monitoring +- Connection status tracking +- Query performance metrics +- Database size monitoring +- Index usage statistics + +### Changed +- Refactored MCP server from 1400 lines to 310 lines (78% reduction) +- Extracted handlers into `mcp_tools/` package +- Improved error handling and logging +- Enhanced code organization and maintainability + +### Documentation +- Added MCP v2 modularization guide +- Updated MCP integration documentation +- Added Prometheus metrics documentation +- Enhanced deployment guides + +## [0.4.0] - 2024-10-20 + +### Added - Real-time Task Monitoring +- **Web UI Monitoring**: NiceGUI-based monitoring interface + - Real-time task status updates via WebSocket + - File upload functionality (50KB size limit) + - Directory batch processing + - Task progress visualization + - Accessible at `/ui/monitor` when `ENABLE_MONITORING=true` +- **Server-Sent Events (SSE)**: Streaming APIs for real-time updates + - `/api/v1/sse/task/{task_id}`: Monitor single task + - `/api/v1/sse/tasks`: Monitor all tasks with filtering + - `/api/v1/sse/stats`: SSE connection statistics +- **Task Queue System**: Background processing with monitoring + - Async task execution + - Progress tracking + - Error handling and retry logic + - Task history and logs + +### Added - Large File Handling +- **Multi-strategy approach**: + - Direct processing: Files < 10KB + - Temporary file strategy: Files 10-50KB + - Directory processing prompt: Files > 50KB + - MCP automatic temp files: All sizes +- **Configurable limits**: Size thresholds via environment variables + +### Added - Client Examples +- `examples/pure_mcp_client.py`: Pure MCP client with watch tools +- `examples/hybrid_http_sse_client.py`: HTTP + SSE hybrid approach +- Real-time monitoring demonstrations + +### Changed +- Enhanced file upload handling with size validation +- Improved error messages for large file uploads +- Better timeout handling for large documents + +### Fixed +- Memory leaks in long-running tasks +- SSE connection stability issues +- File upload timeout for large files + +## [0.3.0] - 2024-09-15 + +### Added - Universal SQL Schema Parser +- **Multi-dialect support**: Oracle, MySQL, PostgreSQL, SQL Server +- **Configurable business domain classification**: YAML/JSON configuration +- **Pre-built industry templates**: + - Insurance: Policies, claims, underwriting + - E-commerce: Products, orders, customers + - Banking: Accounts, transactions, loans + - Healthcare: Patients, diagnoses, treatments +- **Comprehensive parsing**: + - Table and column extraction + - Foreign key relationships + - Index definitions + - Business domain classification +- **Professional documentation generation**: Markdown output +- **Real-world tested**: 356-table Oracle database (4,511 columns) + +### Added - SQL API Endpoints +- `POST /api/v1/sql/parse`: Parse SQL schema files +- `POST /api/v1/sql/analyze`: Analyze database structure +- `GET /api/v1/sql/templates`: List available templates + +### Changed +- Enhanced schema parsing with configurable templates +- Improved relationship detection +- Better error handling for malformed SQL + +### Documentation +- Added SQL parsing user guide +- Industry template documentation +- Configuration examples + +## [0.2.0] - 2024-08-01 + +### Added - Multi-Provider LLM Support +- **Ollama integration**: Local LLM hosting (default) +- **OpenAI integration**: GPT models and embeddings +- **Google Gemini integration**: Gemini models and embeddings +- **OpenRouter integration**: Multi-provider access +- **HuggingFace embeddings**: Local embedding models +- **Provider configuration**: Via `.env` file with flexible switching + +### Added - Enhanced Configuration +- Environment-based configuration system +- Support for multiple embedding providers +- Configurable timeouts and limits +- Feature flags (monitoring, Prometheus) + +### Changed +- Refactored service initialization for multi-provider support +- Improved LLM provider abstraction layer +- Enhanced error messages for provider issues + +### Fixed +- OpenAI API compatibility issues +- Gemini embedding dimension mismatches +- Provider-specific timeout handling + +## [0.1.0] - 2024-07-01 + +### Added - Initial Release +- **Core Features**: + - Neo4j GraphRAG integration + - Vector search with LlamaIndex + - Document processing (text, markdown, code) + - Knowledge graph construction + - Intelligent query engine + - RESTful API +- **Code Analysis**: + - Python code parsing + - TypeScript/JavaScript parsing + - Import relationship mapping + - Basic code graph visualization +- **Document Management**: + - Multi-format support + - Asynchronous processing + - Chunk-based indexing + - Vector similarity search +- **API Endpoints**: + - `/api/v1/health`: Health check + - `/api/v1/knowledge/query`: Query knowledge base + - `/api/v1/knowledge/search`: Vector search + - `/api/v1/documents/upload`: Upload documents + - `/api/v1/documents/list`: List documents +- **Infrastructure**: + - FastAPI backend + - Neo4j database + - Docker support + - Basic logging and error handling + +### Documentation +- Initial README +- API documentation +- Basic deployment guide +- Example scripts + +--- + +## Version History Summary + +| Version | Release Date | Key Features | +|---------|--------------|--------------| +| 0.7.0 | 2025-01-15 | Automatic memory extraction (5 tools) | +| 0.6.0 | 2024-12-20 | Memory Store, Multi-language (Java/PHP/Go), Docker modes | +| 0.5.0 | 2024-11-15 | MCP protocol, Prometheus metrics, Modular architecture | +| 0.4.0 | 2024-10-20 | Real-time monitoring, SSE, Large file handling | +| 0.3.0 | 2024-09-15 | Universal SQL parser, Business domain templates | +| 0.2.0 | 2024-08-01 | Multi-provider LLM support (Ollama/OpenAI/Gemini) | +| 0.1.0 | 2024-07-01 | Initial release with core features | + +## Upgrade Guides + +### Upgrading from 0.6.x to 0.7.0 + +**No breaking changes**. Simply pull new Docker image: + +```bash +docker pull royisme/codebase-rag:0.7.0-full +docker-compose restart +``` + +**New Features Available:** +- Memory extraction endpoints and MCP tools +- Automatic memory mining from git and code + +### Upgrading from 0.5.x to 0.6.0 + +**Breaking Changes:** +- None, fully backward compatible + +**New Configuration Options:** +```env +# Optional: Enable memory features (included in standard/full modes) +ENABLE_MEMORY_STORE=true +``` + +**Data Migration:** +- No migration needed +- Memory Store creates new nodes in existing Neo4j database + +### Upgrading from 0.4.x to 0.5.0 + +**Breaking Changes:** +- MCP server entry point changed from `mcp_server.py` to `start_mcp.py` + +**Configuration Update:** +```json +// claude_desktop_config.json +{ + "mcpServers": { + "code-graph": { + "command": "python", + "args": ["/path/to/start_mcp.py"] // Changed from mcp_server.py + } + } +} +``` + +**Data Migration:** +- No database changes +- MCP protocol fully backward compatible + +### Upgrading from 0.3.x to 0.4.0 + +**No breaking changes**. New features are opt-in: + +```env +# Enable monitoring UI +ENABLE_MONITORING=true + +# Enable Prometheus metrics +ENABLE_PROMETHEUS=true +``` + +## Migration Notes + +### Python Version Upgrade +As of v0.6.0, Python 3.13+ is required. If upgrading from older versions: + +```bash +# Update Python +python3.13 -m venv .venv +source .venv/bin/activate + +# Reinstall dependencies +pip install --upgrade pip +pip install -e . +``` + +### Neo4j Version Compatibility +All versions support Neo4j 5.0+. No database migration needed between versions. + +### Environment Variables +Check `.env.example` for new configuration options in each version. + +## Deprecation Notices + +### Deprecated in 0.7.0 +- None + +### Deprecated in 0.6.0 +- None + +### Deprecated in 0.5.0 +- **Old MCP server entry point** (`mcp_server.py`): Use `start_mcp.py` instead +- Will be removed in: v1.0.0 + +### Removed in 0.5.0 +- None + +## Contributing + +See [CONTRIBUTING.md](./development/contributing.md) for guidelines on contributing to this project. + +## Support + +- **Documentation**: https://code-graph.vantagecraft.dev +- **Issues**: https://github.com/royisme/codebase-rag/issues +- **Discussions**: https://github.com/royisme/codebase-rag/discussions + +## Links + +- [Homepage](https://code-graph.vantagecraft.dev) +- [GitHub Repository](https://github.com/royisme/codebase-rag) +- [Docker Hub](https://hub.docker.com/r/royisme/codebase-rag) +- [Issue Tracker](https://github.com/royisme/codebase-rag/issues) + +--- + +**Note**: Dates in this changelog are illustrative. Check [GitHub Releases](https://github.com/royisme/codebase-rag/releases) for actual release dates. diff --git a/docs/deployment/docker.md b/docs/deployment/docker.md new file mode 100644 index 0000000..adff713 --- /dev/null +++ b/docs/deployment/docker.md @@ -0,0 +1,504 @@ +# Docker Deployment Guide + +Comprehensive guide for deploying Code Graph Knowledge System using Docker and Docker Compose. + +## Overview + +The system provides three Docker images: +- `royisme/codebase-rag:minimal` - Code Graph only (smallest) +- `royisme/codebase-rag:standard` - Code Graph + Memory +- `royisme/codebase-rag:full` - All features (largest) + +## Docker Compose Files + +### Location +- `docker-compose.yml` - Default (points to minimal) +- `docker/docker-compose.minimal.yml` - Minimal mode +- `docker/docker-compose.standard.yml` - Standard mode +- `docker/docker-compose.full.yml` - Full mode with optional Ollama + +### Common Structure + +All compose files include: +```yaml +services: + neo4j: + image: neo4j:5-enterprise # or neo4j:5-community + environment: + - NEO4J_AUTH=neo4j/password + - NEO4J_PLUGINS=["apoc"] + volumes: + - neo4j-data:/data + ports: + - "7474:7474" # HTTP + - "7687:7687" # Bolt + + mcp: + image: royisme/codebase-rag:MODE + environment: + - NEO4J_URI=bolt://neo4j:7687 + - DEPLOYMENT_MODE=MODE + volumes: + - ./repos:/repos + - ./data:/data + depends_on: + - neo4j +``` + +## Building Custom Images + +### Build from Source + +```bash +# Clone repository +git clone https://github.com/royisme/codebase-rag.git +cd codebase-rag + +# Build minimal +docker build -f docker/Dockerfile.minimal -t my-codebase-rag:minimal . + +# Build standard +docker build -f docker/Dockerfile.standard -t my-codebase-rag:standard . + +# Build full +docker build -f docker/Dockerfile.full -t my-codebase-rag:full . +``` + +### Build with Buildx (Multi-Platform) + +```bash +# Create builder +docker buildx create --name mybuilder --use + +# Build for multiple platforms +docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -f docker/Dockerfile.minimal \ + -t my-codebase-rag:minimal \ + --push \ + . +``` + +## Volume Management + +### Important Volumes + +**1. Neo4j Data** (`neo4j-data`) +```yaml +volumes: + neo4j-data: + driver: local +``` + +Contains all graph database data. **Must be backed up regularly.** + +**2. Repository Mount** (`./repos:/repos`) +```yaml +volumes: + - ./repos:/repos:ro # Read-only recommended +``` + +Mount local repositories for ingestion. + +**3. Application Data** (`./data:/data`) +```yaml +volumes: + - ./data:/data +``` + +Temporary files, logs, and processing data. + +### Backup Volumes + +```bash +# Backup Neo4j data +docker run --rm \ + -v codebase-rag_neo4j-data:/data \ + -v $(pwd)/backup:/backup \ + alpine \ + tar czf /backup/neo4j-backup-$(date +%Y%m%d).tar.gz /data + +# Restore from backup +docker run --rm \ + -v codebase-rag_neo4j-data:/data \ + -v $(pwd)/backup:/backup \ + alpine \ + tar xzf /backup/neo4j-backup-20241106.tar.gz -C / +``` + +## Network Configuration + +### Default Network + +```yaml +networks: + default: + name: codebase-rag-network +``` + +### Custom Network + +```yaml +networks: + codebase-rag: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/16 + +services: + neo4j: + networks: + codebase-rag: + ipv4_address: 172.28.0.10 +``` + +### External Services + +Connect to external Ollama: + +```yaml +services: + mcp: + environment: + - OLLAMA_BASE_URL=http://host.docker.internal:11434 + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +## Environment Variables + +### Core Variables + +```bash +# Neo4j Connection +NEO4J_URI=bolt://neo4j:7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD= +NEO4J_DATABASE=neo4j + +# Deployment Mode +DEPLOYMENT_MODE=minimal|standard|full +ENABLE_KNOWLEDGE_RAG=true|false +ENABLE_AUTO_EXTRACTION=true|false +``` + +### LLM Configuration + +```bash +# Provider Selection +LLM_PROVIDER=ollama|openai|gemini|openrouter +EMBEDDING_PROVIDER=ollama|openai|gemini|huggingface + +# Ollama +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_MODEL=llama3.2 +OLLAMA_EMBEDDING_MODEL=nomic-embed-text + +# OpenAI +OPENAI_API_KEY=sk-... +OPENAI_MODEL=gpt-4o +OPENAI_EMBEDDING_MODEL=text-embedding-3-small + +# Gemini +GOOGLE_API_KEY=AIza... +GEMINI_MODEL=gemini-1.5-flash +GEMINI_EMBEDDING_MODEL=models/embedding-001 +``` + +### Performance Tuning + +```bash +# Timeouts (seconds) +CONNECTION_TIMEOUT=30 +OPERATION_TIMEOUT=300 +LARGE_DOCUMENT_TIMEOUT=600 + +# Neo4j Memory +NEO4J_server_memory_heap_initial__size=2G +NEO4J_server_memory_heap_max__size=4G +NEO4J_server_memory_pagecache_size=2G +``` + +## Docker Profiles + +Use profiles to optionally include services: + +```yaml +services: + ollama: + profiles: + - with-ollama + image: ollama/ollama:latest +``` + +```bash +# Start without Ollama +docker-compose up -d + +# Start with Ollama +docker-compose --profile with-ollama up -d +``` + +## Health Checks + +All images include health checks: + +```yaml +services: + mcp: + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/health"] + interval: 30s + timeout: 10s + start_period: 40s + retries: 3 +``` + +Check health: + +```bash +# View health status +docker ps + +# Check specific container +docker inspect --format='{{.State.Health.Status}}' codebase-rag-mcp + +# View health logs +docker inspect --format='{{range .State.Health.Log}}{{.Output}}{{end}}' codebase-rag-mcp +``` + +## Resource Limits + +### Memory Limits + +```yaml +services: + mcp: + deploy: + resources: + limits: + memory: 4G + reservations: + memory: 2G +``` + +### CPU Limits + +```yaml +services: + mcp: + deploy: + resources: + limits: + cpus: '2.0' + reservations: + cpus: '1.0' +``` + +## Logging + +### Configure Logging Driver + +```yaml +services: + mcp: + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" +``` + +### View Logs + +```bash +# Follow logs +docker-compose logs -f mcp + +# Last 100 lines +docker-compose logs --tail=100 mcp + +# Since timestamp +docker-compose logs --since 2024-11-06T10:00:00 mcp +``` + +## Multi-Stage Deployment + +### Development + +```yaml +# docker-compose.dev.yml +services: + mcp: + build: + context: . + dockerfile: docker/Dockerfile.minimal + volumes: + - .:/app # Mount source code + environment: + - DEBUG=true +``` + +### Production + +```yaml +# docker-compose.prod.yml +services: + mcp: + image: royisme/codebase-rag:minimal + restart: unless-stopped + logging: + driver: "syslog" + deploy: + resources: + limits: + memory: 4G +``` + +## Security Best Practices + +### 1. Use Secrets + +```yaml +services: + mcp: + secrets: + - neo4j_password + - openai_api_key + +secrets: + neo4j_password: + file: ./secrets/neo4j_password.txt + openai_api_key: + file: ./secrets/openai_api_key.txt +``` + +### 2. Non-Root User + +All images run as non-root user `appuser` (UID 1000). + +### 3. Read-Only Filesystem + +```yaml +services: + mcp: + read_only: true + tmpfs: + - /tmp + - /app/temp +``` + +### 4. Network Isolation + +```yaml +networks: + frontend: + driver: bridge + backend: + driver: bridge + internal: true # No external access + +services: + mcp: + networks: + - frontend + neo4j: + networks: + - backend +``` + +## Updating Images + +### Pull Latest + +```bash +# Pull latest image +docker pull royisme/codebase-rag:minimal + +# Recreate containers +docker-compose up -d --force-recreate mcp +``` + +### Zero-Downtime Update + +```bash +# Scale up new version +docker-compose up -d --scale mcp=2 --no-recreate + +# Remove old container +docker stop codebase-rag-mcp-1 +docker rm codebase-rag-mcp-1 + +# Scale back to 1 +docker-compose up -d --scale mcp=1 +``` + +## Troubleshooting + +### Container Won't Start + +```bash +# Check logs +docker logs codebase-rag-mcp + +# Check health +docker inspect codebase-rag-mcp + +# Try recreating +docker-compose down +docker-compose up -d +``` + +### Network Issues + +```bash +# Test connectivity +docker exec -it codebase-rag-mcp ping neo4j + +# Check network +docker network inspect codebase-rag-network + +# Recreate network +docker-compose down +docker network prune +docker-compose up -d +``` + +### Performance Issues + +```bash +# Check resource usage +docker stats + +# Check Neo4j performance +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p password +# Run: CALL dbms.listQueries(); + +# Increase resources in docker-compose.yml +``` + +## Advanced Patterns + +### Using Docker Swarm + +```bash +# Initialize swarm +docker swarm init + +# Deploy stack +docker stack deploy -c docker-compose.yml codebase-rag + +# Scale service +docker service scale codebase-rag_mcp=3 +``` + +### Using Kubernetes + +See separate Kubernetes deployment guide (coming soon). + +## Next Steps + +- [Minimal Mode Guide](minimal.md) - Deploy minimal mode +- [Production Setup](production.md) - Production best practices +- [Troubleshooting](../troubleshooting.md) - Common issues diff --git a/docs/deployment/full.md b/docs/deployment/full.md new file mode 100644 index 0000000..80c8034 --- /dev/null +++ b/docs/deployment/full.md @@ -0,0 +1,466 @@ +# Full Mode Deployment + +Full Mode provides **all features** including Code Graph, Memory Store, Knowledge RAG, and LLM-powered auto-extraction. + +## Complete Feature Set + +###All Features Enabled +- ✅ **Code Graph**: Repository indexing, search, impact analysis +- ✅ **Memory Store**: Project knowledge with vector search +- ✅ **Knowledge RAG**: Document processing and intelligent Q&A +- ✅ **Auto-Extraction**: LLM-powered memory extraction from: + - Git commits + - Code comments (TODO, FIXME, NOTE) + - AI conversations + - Knowledge base queries + +### Use Cases +- Full-featured AI coding assistant +- Intelligent documentation systems +- Automated knowledge capture +- Enterprise code intelligence platform + +## System Requirements + +### With Local LLM (Ollama) +- **CPU**: 8+ cores (16+ recommended) +- **RAM**: 16GB minimum (32GB recommended) +- **GPU**: Optional but highly recommended (8GB+ VRAM) +- **Disk**: 100GB SSD + +### With Cloud LLM +- **CPU**: 4 cores +- **RAM**: 8GB +- **Disk**: 50GB SSD +- **API Access**: OpenAI, Gemini, or OpenRouter + +## Quick Start + +### 1. Choose LLM Provider + +=== "Ollama (Local, Private)" + + ```bash + # Install Ollama + curl -fsSL https://ollama.com/install.sh | sh + + # Pull models + ollama pull llama3.2 # 8B parameter model + ollama pull nomic-embed-text # Embedding model + + # For better quality (requires more RAM) + # ollama pull mistral:7b + # ollama pull qwen2.5:14b + ``` + +=== "OpenAI (Cloud, Best Quality)" + + ```bash + # Get API key + # Visit: https://platform.openai.com/api-keys + export OPENAI_API_KEY=sk-proj-... + ``` + +=== "Google Gemini (Cloud, Cost-Effective)" + + ```bash + # Get API key + # Visit: https://makersuite.google.com/app/apikey + export GOOGLE_API_KEY=AIza... + ``` + +=== "OpenRouter (Multi-Provider)" + + ```bash + # Get API key + # Visit: https://openrouter.ai/keys + export OPENROUTER_API_KEY=sk-or-v1-... + ``` + +### 2. Configure Environment + +```bash +# Copy full template +cp docker/.env.template/.env.full .env + +# Edit configuration +nano .env +``` + +Example with Ollama: + +```bash +# Neo4j +NEO4J_URI=bolt://neo4j:7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=your_secure_password +NEO4J_DATABASE=neo4j + +# Deployment Mode - Enable all features +DEPLOYMENT_MODE=full +ENABLE_KNOWLEDGE_RAG=true +ENABLE_AUTO_EXTRACTION=true + +# LLM Configuration +LLM_PROVIDER=ollama +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_MODEL=llama3.2 + +# Embedding Configuration +EMBEDDING_PROVIDER=ollama +OLLAMA_EMBEDDING_MODEL=nomic-embed-text +``` + +### 3. Start Services + +=== "With Bundled Ollama" + + ```bash + # Start with Ollama container included + make docker-full-with-ollama + + # Or + docker-compose -f docker/docker-compose.full.yml --profile with-ollama up -d + ``` + +=== "With External Ollama" + + ```bash + # Start without Ollama (use system Ollama) + make docker-full + + # Or + docker-compose -f docker/docker-compose.full.yml up -d + ``` + +### 4. Verify Deployment + +```bash +# Check all containers +docker ps +# Should see: mcp, neo4j, (optionally ollama) + +# Test LLM +curl http://localhost:11434/api/generate -d '{ + "model": "llama3.2", + "prompt": "Hello, how are you?", + "stream": false +}' + +# Test embedding +curl http://localhost:11434/api/embeddings -d '{ + "model": "nomic-embed-text", + "prompt": "test embedding" +}' + +# Check service health (if using FastAPI) +curl http://localhost:8000/api/v1/health +``` + +## Available MCP Tools + +Full mode provides **30 tools** across 6 categories: + +### Code Graph Tools (4) +- `code_graph_ingest_repo` +- `code_graph_fulltext_search` +- `code_graph_impact_analysis` +- `code_graph_pack_context` + +### Memory Management Tools (7) +- `add_memory` +- `search_memories` +- `get_memory` +- `update_memory` +- `delete_memory` +- `supersede_memory` +- `get_project_summary` + +### Auto-Extraction Tools (5) - New! +- `extract_from_conversation` +- `extract_from_git_commit` +- `extract_from_code_comments` +- `suggest_memory_from_query` +- `batch_extract_from_repository` + +### Knowledge RAG Tools (8) - New! +- `knowledge_add_document` +- `knowledge_add_directory` +- `knowledge_query` +- `knowledge_search` +- `knowledge_list_documents` +- `knowledge_delete_document` +- `knowledge_update_document` +- `knowledge_get_stats` + +### Task Queue Tools (4) +- `task_submit` +- `task_status` +- `task_cancel` +- `list_tasks` + +### System Tools (2) +- `health_check` +- `system_info` + +## Advanced Features + +### Auto-Extraction from Git Commits + +Automatically extract decisions and learnings: + +```json +{ + "tool": "extract_from_git_commit", + "input": { + "project_id": "myapp", + "commit_sha": "abc123...", + "commit_message": "feat: implement JWT authentication\n\nAdded JWT middleware for API auth", + "changed_files": ["src/auth/jwt.py", "src/middleware/auth.py"], + "auto_save": true + } +} +``` + +### Mine Code Comments + +Extract TODOs and decisions from code: + +```json +{ + "tool": "extract_from_code_comments", + "input": { + "project_id": "myapp", + "file_path": "src/api/routes.py" + } +} +``` + +### Conversation Analysis + +Extract memories from AI conversations: + +```json +{ + "tool": "extract_from_conversation", + "input": { + "project_id": "myapp", + "conversation": [ + {"role": "user", "content": "Should we use Redis or Memcached?"}, + {"role": "assistant", "content": "Redis is better because..."} + ], + "auto_save": false + } +} +``` + +### Knowledge RAG + +Process and query documents: + +```json +{ + "tool": "knowledge_add_document", + "input": { + "file_path": "/docs/architecture.md", + "metadata": {"type": "architecture", "version": "1.0"} + } +} + +{ + "tool": "knowledge_query", + "input": { + "query": "How does the authentication system work?", + "max_results": 5 + } +} +``` + +### Batch Repository Extraction + +Comprehensive analysis: + +```json +{ + "tool": "batch_extract_from_repository", + "input": { + "project_id": "myapp", + "repo_path": "/repos/myapp", + "max_commits": 100, + "file_patterns": ["*.py", "*.js", "*.go"] + } +} +``` + +## LLM Provider Comparison + +### Ollama (Local) + +**Pros**: +- Free and private +- No API limits +- Works offline +- Full control + +**Cons**: +- Requires powerful hardware +- Slower than cloud +- Manual model management + +**Recommended Models**: +- `llama3.2` (8B) - Good balance +- `mistral` (7B) - Fast +- `qwen2.5` (14B) - Better quality (needs 16GB+ RAM) + +### OpenAI + +**Pros**: +- Best quality +- Fast responses +- No infrastructure needed + +**Cons**: +- Costs money +- Requires internet +- Data sent to OpenAI + +**Cost** (Nov 2024): +- GPT-4o: $5/$15 per 1M tokens (in/out) +- GPT-4o-mini: $0.15/$0.60 per 1M tokens +- Embeddings: $0.02 per 1M tokens + +### Google Gemini + +**Pros**: +- Cost-effective +- Good quality +- Fast + +**Cons**: +- Requires internet +- Data sent to Google + +**Cost**: +- Gemini 1.5 Flash: Lower cost +- Gemini 1.5 Pro: Higher quality +- Free tier available + +## Performance Optimization + +### Ollama GPU Acceleration + +```yaml +# Add to docker-compose.full.yml +services: + ollama: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] +``` + +### Neo4j Performance for Large Scale + +```bash +# In docker-compose.full.yml +NEO4J_server_memory_heap_initial__size=4G +NEO4J_server_memory_heap_max__size=8G +NEO4J_server_memory_pagecache_size=4G +NEO4J_dbms_memory_transaction_total_max=2G +``` + +### LLM Context Optimization + +```python +# Use context packing to stay within token limits +tool: code_graph_pack_context +input: { + "entry_points": ["src/main.py"], + "task_type": "implement", + "token_budget": 8000 # Adjust based on model +} +``` + +## Cost Estimation + +### Local Deployment (Ollama) +- **VPS**: $40-80/month (32GB RAM, 8 cores) +- **GPU VPS**: $100-200/month (with GPU) +- **LLM**: $0 +- **Embeddings**: $0 +- **Total**: $40-200/month + +### Cloud Deployment (OpenAI) +- **VPS**: $10-20/month (8GB RAM) +- **LLM**: $20-100/month (depends on usage) +- **Embeddings**: $1-5/month +- **Total**: $31-125/month + +### Hybrid (Ollama Embeddings + OpenAI LLM) +- **VPS**: $10-20/month +- **LLM**: $20-100/month +- **Embeddings**: $0 (local) +- **Total**: $30-120/month + +## Production Deployment + +See [Production Setup Guide](production.md) for: +- High availability configuration +- Backup strategies +- Monitoring setup +- Security hardening +- Scaling considerations + +## Troubleshooting + +### LLM Generation Fails + +```bash +# Check Ollama +curl http://localhost:11434/api/generate -d '{ + "model": "llama3.2", + "prompt": "test" +}' + +# Check model is pulled +ollama list + +# View Ollama logs +docker logs codebase-rag-ollama +``` + +### Out of Memory Errors + +```bash +# Check memory usage +docker stats + +# Reduce model size +ollama pull llama3.2:3b # Smaller 3B model + +# Or increase Docker memory limit +# Docker Desktop: Settings → Resources → Memory +``` + +### Slow Response Times + +```bash +# Enable GPU acceleration (if available) +# Check GPU is detected +nvidia-smi + +# Or switch to smaller model +OLLAMA_MODEL=mistral # 7B instead of 13B + +# Or use cloud LLM for faster responses +LLM_PROVIDER=openai +``` + +## Next Steps + +- [Knowledge RAG Guide](../guide/knowledge/overview.md) - Document processing +- [Auto-Extraction Guide](../guide/memory/extraction.md) - Automated memory capture +- [Production Setup](production.md) - Deploy at scale diff --git a/docs/deployment/minimal.md b/docs/deployment/minimal.md new file mode 100644 index 0000000..27f04f2 --- /dev/null +++ b/docs/deployment/minimal.md @@ -0,0 +1,340 @@ +# Minimal Mode Deployment + +Minimal Mode provides **Code Graph functionality only** - no LLM or embedding model required. Perfect for: + +- Resource-constrained environments +- Privacy-sensitive projects +- Cost-conscious deployments +- Pure code analysis without AI + +## Features Available + +### ✅ What's Included + +- **Repository Ingestion**: Parse and index code repositories +- **Fulltext Search**: Fast code search using Neo4j native indexes +- **Graph Traversal**: Navigate code relationships (calls, imports, inheritance) +- **Impact Analysis**: Find what code depends on a given symbol +- **Context Packing**: Intelligently select relevant code for LLM context + +### ❌ What's Not Included + +- Vector similarity search (no embeddings) +- Memory Store for AI agents +- LLM-powered auto-extraction +- Knowledge RAG document Q&A + +## System Requirements + +### Minimum +- **CPU**: 2 cores +- **RAM**: 4GB +- **Disk**: 10GB SSD +- **Docker**: 20.10+ +- **Neo4j**: 5.0+ (included) + +### Recommended +- **CPU**: 4 cores +- **RAM**: 8GB +- **Disk**: 50GB SSD + +## Quick Start + +### 1. Clone and Configure + +```bash +# Clone repository +git clone https://github.com/royisme/codebase-rag.git +cd codebase-rag + +# Copy minimal environment template +cp docker/.env.template/.env.minimal .env + +# Edit configuration +nano .env +``` + +### 2. Configure Environment + +Edit `.env`: + +```bash +# Neo4j Configuration (required) +NEO4J_URI=bolt://neo4j:7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=change_this_password # ⚠️ Change this! +NEO4J_DATABASE=neo4j + +# Deployment Mode +DEPLOYMENT_MODE=minimal +ENABLE_KNOWLEDGE_RAG=false +ENABLE_AUTO_EXTRACTION=false +``` + +### 3. Start Services + +```bash +# Using Makefile (recommended) +make docker-minimal + +# Or using docker-compose directly +docker-compose -f docker/docker-compose.minimal.yml up -d + +# Or using helper script +./scripts/docker-deploy.sh +# Choose option 1: Minimal +``` + +### 4. Verify Deployment + +```bash +# Check containers +docker ps +# Should show: codebase-rag-mcp-minimal and codebase-rag-neo4j + +# Check Neo4j +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p your_password +# Run: RETURN 'Connected' as status; + +# View logs +docker logs codebase-rag-mcp-minimal +``` + +## MCP Client Configuration + +Configure Claude Desktop or VS Code to use the minimal MCP server: + +### Claude Desktop + +Edit `~/Library/Application Support/Claude/claude_desktop_config.json`: + +```json +{ + "mcpServers": { + "codebase-rag-minimal": { + "command": "docker", + "args": [ + "exec", + "-i", + "codebase-rag-mcp-minimal", + "python", + "start_mcp.py", + "--mode=minimal" + ] + } + } +} +``` + +### VS Code with MCP Extension + +Add to VS Code settings: + +```json +{ + "mcp.servers": { + "codebase-rag-minimal": { + "command": "docker", + "args": ["exec", "-i", "codebase-rag-mcp-minimal", "python", "start_mcp.py", "--mode=minimal"], + "type": "stdio" + } + } +} +``` + +## Available MCP Tools + +Minimal mode provides 4 core Code Graph tools: + +### 1. code_graph_ingest_repo + +Index a code repository: + +```json +{ + "local_path": "/repos/myproject", + "mode": "full" +} +``` + +### 2. code_graph_fulltext_search + +Search code by keywords: + +```json +{ + "query": "authentication middleware", + "language": "python", + "limit": 20 +} +``` + +### 3. code_graph_impact_analysis + +Find code dependencies: + +```json +{ + "symbol": "UserService.authenticate", + "direction": "reverse" +} +``` + +### 4. code_graph_pack_context + +Build intelligent context for LLM: + +```json +{ + "entry_points": ["src/api/routes.py"], + "task_type": "implement", + "token_budget": 8000 +} +``` + +## Usage Examples + +### Example 1: Index and Search + +```bash +# 1. Ingest repository +# (Via Claude or MCP client) +Tool: code_graph_ingest_repo +Input: {"local_path": "/repos/myapp", "mode": "full"} + +# 2. Search for authentication code +Tool: code_graph_fulltext_search +Input: {"query": "JWT token validation", "language": "python"} + +# 3. Analyze impact of changing auth function +Tool: code_graph_impact_analysis +Input: {"symbol": "validate_token", "direction": "reverse"} +``` + +### Example 2: Prepare Context for Code Review + +```bash +# Pack relevant context for reviewing auth changes +Tool: code_graph_pack_context +Input: { + "entry_points": ["src/auth/jwt.py", "src/middleware/auth.py"], + "task_type": "review", + "token_budget": 12000 +} +``` + +## Performance Optimization + +### Neo4j Tuning + +For large repositories, adjust Neo4j memory in `docker-compose.minimal.yml`: + +```yaml +services: + neo4j: + environment: + - NEO4J_server_memory_heap_initial__size=2G + - NEO4J_server_memory_heap_max__size=4G + - NEO4J_server_memory_pagecache_size=2G +``` + +### Ingestion Performance + +```bash +# Incremental updates for large repos +Tool: code_graph_ingest_repo +Input: {"local_path": "/repos/myapp", "mode": "incremental"} + +# Full re-index when needed +Input: {"local_path": "/repos/myapp", "mode": "full"} +``` + +## Monitoring + +Check system health: + +```bash +# Container stats +docker stats codebase-rag-mcp-minimal codebase-rag-neo4j + +# Neo4j query performance +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p password +# Run: CALL dbms.listQueries(); + +# View ingestion logs +docker logs -f codebase-rag-mcp-minimal +``` + +## Upgrading to Standard/Full Mode + +When you need more features: + +```bash +# Stop minimal mode +docker-compose -f docker/docker-compose.minimal.yml down + +# Copy and configure for standard mode +cp docker/.env.template/.env.standard .env +nano .env # Add embedding configuration + +# Start standard mode +docker-compose -f docker/docker-compose.standard.yml up -d +``` + +Your Neo4j data persists, so existing code graphs are preserved. + +## Troubleshooting + +### Neo4j Connection Failed + +```bash +# Check Neo4j status +docker logs codebase-rag-neo4j + +# Verify Neo4j is ready +docker exec codebase-rag-neo4j neo4j status + +# Test connection +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p password +``` + +### Ingestion Stuck + +```bash +# Check MCP server logs +docker logs codebase-rag-mcp-minimal + +# Check disk space +df -h + +# Restart if needed +docker restart codebase-rag-mcp-minimal +``` + +### Poor Search Results + +```bash +# Rebuild fulltext indexes +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p password + +# Run these queries: +CALL db.index.fulltext.drop('code_search'); +CALL db.index.fulltext.createNodeIndex('code_search', ['Function', 'Class'], ['name', 'content']); +``` + +## Cost Analysis + +Minimal mode is the most cost-effective option: + +- **Infrastructure**: ~$5-10/month (small VPS) +- **LLM costs**: $0 (no LLM required) +- **Embedding costs**: $0 (no embeddings) +- **Total**: ~$5-10/month for hosting only + +Perfect for individual developers and small teams! + +## Next Steps + +- [Docker Guide](docker.md) - Advanced Docker configuration +- [Code Graph User Guide](../guide/code-graph/overview.md) - Learn all features +- [Production Setup](production.md) - Deploy to production diff --git a/docs/deployment/overview.md b/docs/deployment/overview.md new file mode 100644 index 0000000..0d78941 --- /dev/null +++ b/docs/deployment/overview.md @@ -0,0 +1,355 @@ +# Deployment Overview + +Choose the right deployment mode based on your needs and available infrastructure. + +## 🎯 Deployment Modes + +### Minimal - Code Graph Only + +**Perfect for**: Developers who want code intelligence without LLM overhead + +```yaml +Requirements: + - Neo4j database + - Docker & docker-compose + - No LLM needed ✓ + - No embedding model needed ✓ + +Resources: + - Image size: ~500MB + - Memory: ~1GB RAM + - Startup time: ~5 seconds +``` + +**Available Features**: + +- ✅ Repository ingestion and code parsing +- ✅ File relationship discovery (imports, dependencies) +- ✅ Impact analysis (who depends on this file?) +- ✅ Context packing for AI assistants +- ✅ Full-text search on file paths and content +- ❌ Memory Store +- ❌ Knowledge RAG +- ❌ Auto-extraction + +**Use When**: + +- You want code navigation and analysis only +- You don't need LLM-powered features +- You're working in air-gapped environments +- You want minimal resource usage + +[→ Minimal Deployment Guide](minimal.md){ .md-button .md-button--primary } + +--- + +### Standard - Code Graph + Memory + +**Perfect for**: Teams building project knowledge bases + +```yaml +Requirements: + - Neo4j database + - Docker & docker-compose + - Embedding model (Ollama/OpenAI/Gemini) ✓ + - No LLM needed ✓ + +Resources: + - Image size: ~600MB + - Memory: ~2GB RAM + - Startup time: ~8 seconds +``` + +**Available Features**: + +- ✅ All Minimal features +- ✅ Manual memory management (add/update/delete) +- ✅ Vector-based memory search +- ✅ Project memory summaries +- ✅ Memory superseding (track decision changes) +- ❌ Auto-extraction from git/conversations +- ❌ Knowledge RAG + +**Use When**: + +- You want to maintain project decision logs +- You need searchable team knowledge +- You have access to an embedding service +- You prefer manual curation over auto-extraction + +[→ Standard Deployment Guide](standard.md){ .md-button .md-button--primary } + +--- + +### Full - All Features + +**Perfect for**: Teams wanting complete AI-powered capabilities + +```yaml +Requirements: + - Neo4j database + - Docker & docker-compose + - LLM (Ollama/OpenAI/Gemini/OpenRouter) ✓ + - Embedding model ✓ + +Resources: + - Image size: ~800MB + - Memory: ~4GB RAM (+ LLM requirements) + - Startup time: ~15 seconds +``` + +**Available Features**: + +- ✅ All Standard features +- ✅ Automatic memory extraction from: + - Git commits + - AI conversations + - Code comments (TODO/FIXME/NOTE) + - Q&A sessions +- ✅ Knowledge base RAG: + - Document ingestion + - Intelligent Q&A + - Multi-format support +- ✅ Batch repository analysis + +**Use When**: + +- You want fully automated knowledge extraction +- You need document Q&A capabilities +- You have LLM infrastructure available +- You want maximum AI assistance + +[→ Full Deployment Guide](full.md){ .md-button .md-button--primary } + +--- + +## 🔄 Mode Comparison Matrix + +| Feature Category | Minimal | Standard | Full | +|------------------|---------|----------|------| +| **Code Graph** | +| Repository ingestion | ✅ | ✅ | ✅ | +| Incremental updates | ✅ | ✅ | ✅ | +| File search | ✅ | ✅ | ✅ | +| Impact analysis | ✅ | ✅ | ✅ | +| Context packing | ✅ | ✅ | ✅ | +| **Memory Store** | +| Add memory | ❌ | ✅ | ✅ | +| Search memories | ❌ | ✅ (vector) | ✅ (vector) | +| Update/delete | ❌ | ✅ | ✅ | +| Supersede | ❌ | ✅ | ✅ | +| Extract from git | ❌ | ❌ | ✅ (LLM) | +| Extract from chat | ❌ | ❌ | ✅ (LLM) | +| Extract from code | ❌ | ❌ | ✅ (LLM) | +| **Knowledge RAG** | +| Add documents | ❌ | ❌ | ✅ | +| Query knowledge | ❌ | ❌ | ✅ (LLM) | +| Vector search | ❌ | ❌ | ✅ | +| **Infrastructure** | +| Neo4j | Required | Required | Required | +| Embedding | - | Required | Required | +| LLM | - | - | Required | +| **Performance** | +| Image size | 500MB | 600MB | 800MB | +| RAM usage | 1GB | 2GB | 4GB+ | +| Startup time | 5s | 8s | 15s | + +## 🏗️ Architecture Diagrams + +### Minimal Mode Architecture + +```mermaid +graph TB + subgraph "Client" + A[Claude Desktop / API Client] + end + + subgraph "Docker Network" + B[MCP Server
Minimal] + C[(Neo4j
Graph DB)] + end + + subgraph "Code Graph Services" + D[Code Ingestor] + E[Graph Service] + F[Ranker] + G[Pack Builder] + end + + A -->|MCP/REST| B + B --> D + B --> E + B --> F + B --> G + D -->|Store| C + E -->|Query| C + + style B fill:#90EE90 + style C fill:#87CEEB +``` + +### Standard Mode Architecture + +```mermaid +graph TB + subgraph "Client" + A[Claude Desktop / API Client] + end + + subgraph "Docker Network" + B[MCP Server
Standard] + C[(Neo4j
Graph DB)] + end + + subgraph "Code Graph Services" + D[Code Ingestor] + E[Graph Service] + end + + subgraph "Memory Services" + F[Memory Store] + end + + subgraph "External" + G[Embedding Service
Ollama/OpenAI] + end + + A -->|MCP/REST| B + B --> D + B --> E + B --> F + D -->|Store| C + E -->|Query| C + F -->|Store/Search| C + F -->|Vectorize| G + + style B fill:#FFD700 + style C fill:#87CEEB + style G fill:#FFA07A +``` + +### Full Mode Architecture + +```mermaid +graph TB + subgraph "Client" + A[Claude Desktop / API Client] + end + + subgraph "Docker Network" + B[MCP Server
Full] + C[(Neo4j
Graph DB)] + D[Ollama
Optional] + end + + subgraph "All Services" + E[Code Graph] + F[Memory Store] + G[Knowledge RAG] + H[Memory Extractor] + end + + subgraph "External/Optional" + I[LLM Service
OpenAI/Gemini] + J[Embedding Service] + end + + A -->|MCP/REST| B + B --> E + B --> F + B --> G + B --> H + E -->|Store| C + F -->|Store/Search| C + G -->|Store/Query| C + F -->|Vectorize| J + G -->|Generate| I + H -->|Analyze| I + + D -.->|Local LLM| I + D -.->|Local Embed| J + + style B fill:#FF6347 + style C fill:#87CEEB + style D fill:#DDA0DD +``` + +## 🚀 Quick Decision Guide + +Use this flowchart to choose your deployment mode: + +```mermaid +graph TD + A[Start] --> B{Do you need
LLM features?} + B -->|No| C{Do you need
memory search?} + B -->|Yes| D[Full Mode] + C -->|No| E[Minimal Mode] + C -->|Yes| F{Can you provide
embedding service?} + F -->|Yes| G[Standard Mode] + F -->|No| E + + E --> H[✓ Code Graph only
✓ No external deps
✓ Fast & lightweight] + G --> I[✓ Code Graph
✓ Memory Store
⚠ Need embedding] + D --> J{Do you have
local GPU?} + J -->|Yes| K[Use with-ollama profile] + J -->|No| L[Use cloud LLM] + K --> M[✓ All features
✓ Self-hosted
⚠ High resources] + L --> N[✓ All features
✓ Lower resources
⚠ API costs] + + style E fill:#90EE90 + style G fill:#FFD700 + style K fill:#FF6347 + style L fill:#FF6347 +``` + +## 📋 Pre-Deployment Checklist + +### For All Modes + +- [ ] Docker installed (version 20.10+) +- [ ] docker-compose installed (version 1.29+) +- [ ] At least 4GB free disk space +- [ ] Ports 7474, 7687, 8000 available +- [ ] `.env` file configured + +### Additional for Standard Mode + +- [ ] Embedding service available: + - [ ] Local Ollama running, or + - [ ] OpenAI API key, or + - [ ] Google API key for Gemini + +### Additional for Full Mode + +- [ ] LLM service available: + - [ ] Local Ollama running, or + - [ ] OpenAI API key, or + - [ ] Google API key, or + - [ ] OpenRouter API key +- [ ] Embedding service (same as Standard) +- [ ] For local Ollama: GPU with 8GB+ VRAM (optional but recommended) + +## 🔄 Switching Between Modes + +You can switch deployment modes at any time. Data in Neo4j is preserved. + +```bash +# Stop current deployment +make docker-stop + +# Start different mode +make docker-minimal # or +make docker-standard # or +make docker-full +``` + +!!! warning "Configuration Required" + When switching to Standard or Full mode, update your `.env` file with required API keys and service URLs. + +## 📚 Next Steps + +- [Minimal Deployment Guide](minimal.md) +- [Standard Deployment Guide](standard.md) +- [Full Deployment Guide](full.md) +- [Production Setup](production.md) +- [Docker Guide](docker.md) diff --git a/docs/deployment/production.md b/docs/deployment/production.md new file mode 100644 index 0000000..a60f22f --- /dev/null +++ b/docs/deployment/production.md @@ -0,0 +1,515 @@ +# Production Deployment + +This guide covers deploying Code Graph Knowledge System to production, including documentation hosting on vantagecraft.dev. + +## 📝 Documentation Deployment (vantagecraft.dev) + +### Option 1: GitHub Pages (Recommended) + +Deploy documentation automatically using GitHub Actions. + +#### Prerequisites + +- GitHub repository +- Domain `vantagecraft.dev` with DNS access + +#### Step 1: Configure DNS + +Add a CNAME record for your documentation subdomain: + +```dns +Type: CNAME +Name: docs +Value: royisme.github.io +TTL: 3600 +``` + +Or for root domain: + +```dns +Type: A +Name: @ +Value: 185.199.108.153 +Value: 185.199.109.153 +Value: 185.199.110.153 +Value: 185.199.111.153 +``` + +#### Step 2: Configure GitHub Pages + +1. Create `docs/CNAME` file: + +```bash +echo "code-graph.vantagecraft.dev" > docs/CNAME +``` + +2. Enable GitHub Pages in repository settings: + - Go to Settings → Pages + - Source: GitHub Actions + +#### Step 3: Deploy + +The GitHub Actions workflow will automatically deploy on push to main: + +```bash +git add . +git commit -m "Add documentation" +git push origin main +``` + +Your documentation will be available at: **https://code-graph.vantagecraft.dev** + +### Option 2: Self-Hosted (Nginx) + +Host documentation on your own server. + +#### Prerequisites + +- Server with Nginx +- Domain configured +- SSL certificate (Let's Encrypt recommended) + +#### Step 1: Build Documentation + +```bash +# Install dependencies +pip install mkdocs-material mkdocs-minify-plugin mkdocs-git-revision-date-localized-plugin + +# Build +mkdocs build + +# Output in site/ directory +``` + +#### Step 2: Configure Nginx + +```nginx +# /etc/nginx/sites-available/code-graph.vantagecraft.dev + +server { + listen 80; + server_name code-graph.vantagecraft.dev; + return 301 https://$server_name$request_uri; +} + +server { + listen 443 ssl http2; + server_name code-graph.vantagecraft.dev; + + ssl_certificate /etc/letsencrypt/live/code-graph.vantagecraft.dev/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/code-graph.vantagecraft.dev/privkey.pem; + + root /var/www/code-graph.vantagecraft.dev; + index index.html; + + location / { + try_files $uri $uri/ =404; + } + + # Gzip compression + gzip on; + gzip_vary on; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + # Cache static assets + location ~* \.(jpg|jpeg|png|gif|ico|css|js|svg|woff|woff2)$ { + expires 1y; + add_header Cache-Control "public, immutable"; + } +} +``` + +#### Step 3: Deploy + +```bash +# Copy built site to server +rsync -avz site/ user@server:/var/www/code-graph.vantagecraft.dev/ + +# Reload Nginx +ssh user@server 'sudo nginx -t && sudo systemctl reload nginx' +``` + +#### Step 4: SSL Certificate (Let's Encrypt) + +```bash +# On server +sudo apt install certbot python3-certbot-nginx + +# Obtain certificate +sudo certbot --nginx -d code-graph.vantagecraft.dev + +# Auto-renewal is configured automatically +``` + +### Option 3: Cloudflare Pages + +Deploy to Cloudflare Pages for global CDN. + +#### Step 1: Connect Repository + +1. Go to Cloudflare Pages dashboard +2. Create new project from GitHub +3. Select your repository + +#### Step 2: Configure Build + +```yaml +Build command: mkdocs build +Build output directory: site +Root directory: / +``` + +#### Step 3: Custom Domain + +1. Add custom domain: `code-graph.vantagecraft.dev` +2. Cloudflare will configure DNS automatically + +--- + +## 🚀 Application Production Deployment + +### Docker Swarm Deployment + +For production workloads, use Docker Swarm or Kubernetes. + +#### Single Node Setup + +```bash +# Initialize swarm +docker swarm init + +# Deploy stack +docker stack deploy -c docker-compose.full.yml codebase-rag +``` + +#### Stack Configuration + +```yaml +# docker-compose.prod.yml +version: '3.8' + +services: + neo4j: + image: neo4j:5.15-enterprise + deploy: + replicas: 1 + resources: + limits: + memory: 4G + reservations: + memory: 2G + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + volumes: + - neo4j_data:/data + environment: + - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes + # ... other production configs + + mcp: + image: royisme/codebase-rag:full + deploy: + replicas: 2 + update_config: + parallelism: 1 + delay: 10s + resources: + limits: + memory: 2G + reservations: + memory: 1G + environment: + # Production environment variables +``` + +### Kubernetes Deployment + +#### Prerequisites + +- Kubernetes cluster (1.24+) +- kubectl configured +- Helm 3+ + +#### Step 1: Create Namespace + +```bash +kubectl create namespace codebase-rag +``` + +#### Step 2: Deploy Neo4j + +```yaml +# neo4j-deployment.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: neo4j + namespace: codebase-rag +spec: + serviceName: neo4j + replicas: 1 + selector: + matchLabels: + app: neo4j + template: + metadata: + labels: + app: neo4j + spec: + containers: + - name: neo4j + image: neo4j:5.15-community + ports: + - containerPort: 7474 + name: http + - containerPort: 7687 + name: bolt + env: + - name: NEO4J_AUTH + valueFrom: + secretKeyRef: + name: neo4j-auth + key: auth + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 10Gi +``` + +#### Step 3: Deploy Application + +```yaml +# mcp-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-full + namespace: codebase-rag +spec: + replicas: 3 + selector: + matchLabels: + app: mcp-full + template: + metadata: + labels: + app: mcp-full + spec: + containers: + - name: mcp-full + image: royisme/codebase-rag:full + ports: + - containerPort: 8000 + env: + - name: NEO4J_URI + value: "bolt://neo4j:7687" + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: neo4j-auth + key: password + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1000m" + livenessProbe: + httpGet: + path: /api/v1/health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /api/v1/health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 +``` + +#### Step 4: Create Service + +```yaml +# service.yaml +apiVersion: v1 +kind: Service +metadata: + name: mcp-full + namespace: codebase-rag +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 8000 + protocol: TCP + selector: + app: mcp-full +``` + +#### Step 5: Deploy + +```bash +kubectl apply -f neo4j-deployment.yaml +kubectl apply -f mcp-deployment.yaml +kubectl apply -f service.yaml +``` + +--- + +## 🔒 Security Best Practices + +### 1. Environment Variables + +Never commit secrets to git: + +```bash +# Use Kubernetes secrets +kubectl create secret generic app-secrets \ + --from-literal=neo4j-password=xxx \ + --from-literal=openai-api-key=xxx \ + -n codebase-rag +``` + +### 2. Network Security + +```bash +# Restrict Neo4j access +# Only allow from application pods +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: neo4j-policy +spec: + podSelector: + matchLabels: + app: neo4j + ingress: + - from: + - podSelector: + matchLabels: + app: mcp-full +``` + +### 3. TLS/SSL + +Use cert-manager for automatic certificate management: + +```bash +# Install cert-manager +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.0/cert-manager.yaml + +# Create issuer +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: your-email@vantagecraft.dev + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: nginx +``` + +### 4. Rate Limiting + +```nginx +# Nginx ingress annotation +nginx.ingress.kubernetes.io/limit-rps: "10" +nginx.ingress.kubernetes.io/limit-connections: "5" +``` + +--- + +## 📊 Monitoring + +### Prometheus Metrics + +Application exposes metrics at `/metrics`: + +```yaml +# prometheus-servicemonitor.yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: mcp-full +spec: + selector: + matchLabels: + app: mcp-full + endpoints: + - port: http + path: /metrics +``` + +### Logging + +Use ELK stack or Loki for centralized logging: + +```yaml +# fluent-bit configmap +[OUTPUT] + Name es + Match * + Host elasticsearch + Port 9200 + Index codebase-rag +``` + +--- + +## 🔄 Backup & Recovery + +### Neo4j Backup + +```bash +# Manual backup +docker exec codebase-rag-neo4j \ + neo4j-admin database dump neo4j \ + --to=/backups/neo4j-$(date +%Y%m%d).dump + +# Automated backup (cron) +0 2 * * * /usr/local/bin/backup-neo4j.sh +``` + +### Restore + +```bash +# Stop services +docker-compose down + +# Restore backup +docker run --rm \ + -v neo4j_data:/data \ + -v $(pwd)/backups:/backups \ + neo4j:5.15 \ + neo4j-admin database load neo4j \ + --from=/backups/neo4j-20240101.dump + +# Start services +docker-compose up -d +``` + +--- + +## 📚 Next Steps + +- [System Architecture](../architecture/design.md) - Scalability and disaster recovery +- [Troubleshooting Guide](../troubleshooting.md) - Common production issues +- [Docker Guide](docker.md) - Advanced Docker configuration +- [FAQ](../faq.md) - Frequently asked questions diff --git a/docs/deployment/standard.md b/docs/deployment/standard.md new file mode 100644 index 0000000..296c73f --- /dev/null +++ b/docs/deployment/standard.md @@ -0,0 +1,389 @@ +# Standard Mode Deployment + +Standard Mode adds **Memory Store with vector search** to Code Graph functionality. Requires embedding model but no LLM. + +## What You Get + +### Minimal Mode Features + +- **Memory Store**: Persistent project knowledge for AI agents +- **Vector Search**: Semantic similarity search in memories +- **Memory Management**: Add, search, update, delete memories +- **Memory Evolution**: Supersede outdated decisions + +### Use Cases +- AI agent long-term memory across sessions +- Project decision tracking with semantic search +- Team preference documentation +- Problem-solution repository + +## System Requirements + +### Minimum +- **CPU**: 4 cores +- **RAM**: 8GB (for local embeddings) +- **Disk**: 20GB SSD +- **Docker**: 20.10+ + +### With Cloud Embeddings +- **CPU**: 2 cores +- **RAM**: 4GB +- **OpenAI/Gemini API key** + +## Quick Start + +### 1. Choose Embedding Provider + +=== "Ollama (Local, Free)" + + ```bash + # Install Ollama + curl -fsSL https://ollama.com/install.sh | sh + + # Pull embedding model + ollama pull nomic-embed-text + + # Verify + curl http://localhost:11434/api/embeddings \ + -d '{"model":"nomic-embed-text","prompt":"test"}' + ``` + +=== "OpenAI (Cloud, Best Quality)" + + ```bash + # Get API key from https://platform.openai.com/api-keys + export OPENAI_API_KEY=sk-proj-... + ``` + +=== "Google Gemini (Cloud, Cost-Effective)" + + ```bash + # Get API key from https://makersuite.google.com/app/apikey + export GOOGLE_API_KEY=AIza... + ``` + +### 2. Configure Environment + +```bash +# Copy standard template +cp docker/.env.template/.env.standard .env + +# Edit configuration +nano .env +``` + +Example configuration: + +```bash +# Neo4j +NEO4J_URI=bolt://neo4j:7687 +NEO4J_USER=neo4j +NEO4J_PASSWORD=your_secure_password +NEO4J_DATABASE=neo4j + +# Deployment Mode +DEPLOYMENT_MODE=standard +ENABLE_KNOWLEDGE_RAG=false +ENABLE_AUTO_EXTRACTION=false + +# Embedding Provider (choose one) +EMBEDDING_PROVIDER=ollama + +# Ollama Configuration +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_EMBEDDING_MODEL=nomic-embed-text + +# Or OpenAI +# EMBEDDING_PROVIDER=openai +# OPENAI_API_KEY=sk-... +# OPENAI_EMBEDDING_MODEL=text-embedding-3-small +``` + +### 3. Start Services + +```bash +make docker-standard + +# Or +docker-compose -f docker/docker-compose.standard.yml up -d +``` + +### 4. Verify Deployment + +```bash +# Check containers +docker ps + +# Test embedding +curl http://localhost:11434/api/embeddings \ + -d '{"model":"nomic-embed-text","prompt":"test query"}' + +# Check Neo4j vector index +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p password +# Run: SHOW INDEXES; +``` + +## Available MCP Tools + +Standard mode provides **11 tools** (4 Code Graph + 7 Memory): + +### Memory Management Tools + +**1. add_memory** - Save project knowledge +```json +{ + "project_id": "myapp", + "memory_type": "decision", + "title": "Use PostgreSQL for main database", + "content": "Selected PostgreSQL over MySQL", + "reason": "Need advanced JSON support", + "importance": 0.9, + "tags": ["database", "architecture"] +} +``` + +**2. search_memories** - Semantic search +```json +{ + "project_id": "myapp", + "query": "database decisions", + "memory_type": "decision", + "min_importance": 0.7, + "limit": 10 +} +``` + +**3. get_memory** - Retrieve specific memory +```json +{ + "memory_id": "mem_123456" +} +``` + +**4. update_memory** - Modify existing memory +```json +{ + "memory_id": "mem_123456", + "title": "Updated title", + "importance": 0.95 +} +``` + +**5. delete_memory** - Soft delete memory +```json +{ + "memory_id": "mem_123456", + "reason": "No longer relevant" +} +``` + +**6. supersede_memory** - Replace with new memory +```json +{ + "old_memory_id": "mem_123456", + "new_title": "Migrate to PostgreSQL 16", + "new_content": "Upgrading from PostgreSQL 14", + "new_reason": "Performance improvements", + "new_importance": 0.9 +} +``` + +**7. get_project_summary** - Overview of all memories +```json +{ + "project_id": "myapp" +} +``` + +## Usage Examples + +### Example 1: AI Agent Workflow + +```bash +# Agent starts working on authentication feature + +# 1. Search for related decisions +Tool: search_memories +Input: { + "project_id": "myapp", + "query": "authentication security", + "memory_type": "decision" +} + +# 2. Implement feature following past decisions + +# 3. Save new decision +Tool: add_memory +Input: { + "project_id": "myapp", + "memory_type": "decision", + "title": "Use JWT with RS256", + "content": "Implemented JWT authentication with RS256 signing", + "reason": "More secure than HS256, supports key rotation", + "importance": 0.9, + "tags": ["auth", "security"] +} +``` + +### Example 2: Track Problem Solutions + +```bash +# Encountered Redis connection issue in Docker + +Tool: add_memory +Input: { + "project_id": "myapp", + "memory_type": "experience", + "title": "Redis Docker networking issue", + "content": "Redis connection fails with localhost in Docker", + "reason": "Must use service name 'redis' instead of localhost", + "importance": 0.7, + "tags": ["docker", "redis", "networking"] +} + +# Later, search for Redis issues +Tool: search_memories +Input: { + "project_id": "myapp", + "query": "Redis connection problems", + "memory_type": "experience" +} +``` + +### Example 3: Update Outdated Decision + +```bash +# Original decision to use MySQL +Old Memory ID: mem_abc123 + +# Decided to migrate to PostgreSQL +Tool: supersede_memory +Input: { + "old_memory_id": "mem_abc123", + "new_title": "Migrate to PostgreSQL", + "new_content": "Migrating from MySQL to PostgreSQL", + "new_reason": "Need advanced features and better performance", + "new_importance": 0.95, + "new_tags": ["database", "migration"] +} +``` + +## Memory Best Practices + +### Importance Scoring +- **0.9-1.0**: Critical architectural decisions, security findings +- **0.7-0.8**: Important technical choices +- **0.5-0.6**: Team preferences, conventions +- **0.3-0.4**: Future plans, minor notes + +### Effective Tagging +```bash +# Domain tags +"database", "api", "frontend", "auth" + +# Type tags +"performance", "security", "bug", "optimization" + +# Status tags +"critical", "deprecated", "planned" +``` + +### When to Save Memories +- After making architecture decisions +- When solving tricky bugs +- When establishing team conventions +- When discovering important limitations + +## Performance Considerations + +### Embedding Model Selection + +**Local (Ollama)**: +- `nomic-embed-text`: Best quality, 768 dimensions +- `mxbai-embed-large`: Faster, good quality +- `all-minilm`: Lightweight, 384 dimensions + +**Cloud**: +- OpenAI `text-embedding-3-small`: $0.02/1M tokens +- OpenAI `text-embedding-3-large`: $0.13/1M tokens +- Gemini `embedding-001`: Free tier available + +### Vector Index Tuning + +```cypher +// Check vector index status +SHOW INDEXES; + +// Rebuild if needed +DROP INDEX memory_content_vector IF EXISTS; +CREATE VECTOR INDEX memory_content_vector +FOR (m:Memory) ON (m.embedding) +OPTIONS {indexConfig: { + `vector.dimensions`: 768, + `vector.similarity_function`: 'cosine' +}}; +``` + +## Cost Analysis + +### With Local Ollama +- **Infrastructure**: ~$10-20/month (VPS with 8GB RAM) +- **Embedding**: $0 (local) +- **Total**: ~$10-20/month + +### With OpenAI Embeddings +- **Infrastructure**: ~$5-10/month (small VPS) +- **Embeddings**: ~$0.02 per 1M tokens +- **Typical usage**: ~$1-5/month for embeddings +- **Total**: ~$6-15/month + +## Upgrading to Full Mode + +When you need LLM-powered features: + +```bash +# Stop standard mode +docker-compose -f docker/docker-compose.standard.yml down + +# Configure for full mode +cp docker/.env.template/.env.full .env +nano .env # Add LLM configuration + +# Start full mode +docker-compose -f docker/docker-compose.full.yml up -d +``` + +## Troubleshooting + +### Embedding Generation Fails + +```bash +# Check Ollama logs +docker logs codebase-rag-ollama + +# Test embedding locally +curl http://localhost:11434/api/embeddings \ + -d '{"model":"nomic-embed-text","prompt":"test"}' + +# Restart Ollama +docker restart codebase-rag-ollama +``` + +### Vector Search Returns No Results + +```bash +# Check if vector index exists +docker exec -it codebase-rag-neo4j cypher-shell -u neo4j -p password +# Run: SHOW INDEXES; + +# Check memory count +# Run: MATCH (m:Memory) RETURN count(m); + +# Verify embeddings exist +# Run: MATCH (m:Memory) WHERE m.embedding IS NOT NULL RETURN count(m); +``` + +## Next Steps + +- [Memory Store User Guide](../guide/memory/overview.md) - Detailed features +- [Full Mode](full.md) - Upgrade for all features +- [Production Setup](production.md) - Deploy to production diff --git a/docs/development/changelog-automation.md b/docs/development/changelog-automation.md new file mode 100644 index 0000000..f8702e7 --- /dev/null +++ b/docs/development/changelog-automation.md @@ -0,0 +1,508 @@ +# Automatic Changelog Generation + +Complete guide to automatically generating changelogs from git commits. + +## Overview + +Instead of manually writing changelog entries, we automatically generate them from git commit messages using **Conventional Commits** format. + +**Benefits**: +- ✅ Never forget to update changelog +- ✅ Consistent formatting +- ✅ Automatic categorization +- ✅ Less manual work +- ✅ Traceable to specific commits + +## Conventional Commits Format + +All commits should follow this format: + +``` +(): + + + +