diff --git a/.env b/.env index 4749dd7..7d35216 100644 --- a/.env +++ b/.env @@ -4,7 +4,8 @@ OLLAMA_PORT=11434 OLLAMA_MODEL=mistral CHUNK_SIZE=3000 TOKEN_ENCODING=cl100k_base -PROMPT_FILE=prompts/default.txt +PROMPT_FILE=prompts/mistral.txt HEALTH_CHECK_TIMEOUT=5.0 OLLAMA_TIMEOUT=30.0 -OLLAMA_MEMORY=12G \ No newline at end of file +OLLAMA_MEMORY=12G +DEFAULT_CHUNK_TIME=40.0 \ No newline at end of file diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 92fd2ac..0fe9602 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -22,7 +22,7 @@ jobs: - name: Install Dependencies run: | cd backend - uv venv + uv .venv source .venv/bin/activate uv pip install -r requirements.txt diff --git a/README.md b/README.md index 7ec517f..3f3779a 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ Analyze, summarize, and explain information in PDF textbooks and whitepapers using OCR and AI analysis. Upload PDFs to get markdown summaries and analysis with real-time progress tracking. ## Features -- PDF text extraction with OCR +- PDF text extraction with PyMuPDF - Real-time tracking of analysis progress -- Chunked file upload +- Chunked file upload to Ollama - Automatic error recovery - Memory-efficient processing - Markdown formatted output @@ -57,8 +57,9 @@ bun run dev # Backend (default: http://localhost:8000) cd backend -python -m venv venv -source venv/bin/activate +pip install uv +uv .venv +source .venv/bin/activate pip install -r requirements.txt uvicorn main:app --reload ``` diff --git a/backend/Dockerfile b/backend/Dockerfile index 131955d..62656ea 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,8 @@ FROM python:3.12-slim WORKDIR /app +ARG VERSION + LABEL version=$VERSION # Update pip @@ -17,4 +19,4 @@ COPY requirements.txt . RUN uv pip install --system -r requirements.txt COPY . . -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "$BACKEND_PORT", "--lifespan=on", "--log-level=info"] +CMD uvicorn main:app --host 0.0.0.0 --port $BACKEND_PORT --lifespan=on --log-level=info diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 32285f5..737bce7 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -11,6 +11,7 @@ class Settings(BaseSettings): PROMPT_FILE: str = os.environ.get("PROMPT_FILE", "prompts/default.txt") HEALTH_CHECK_TIMEOUT: float = float(os.environ.get("HEALTH_CHECK_TIMEOUT", "5.0")) OLLAMA_TIMEOUT: float = float(os.environ.get("OLLAMA_TIMEOUT", "30.0")) + DEFAULT_CHUNK_TIME: float = float(os.environ.get("DEFAULT_CHUNK_TIME", "30.0")) class Config: env_file = ".env" diff --git a/backend/app/services/ai.py b/backend/app/services/ai.py index a3a321d..0b9f4bf 100644 --- a/backend/app/services/ai.py +++ b/backend/app/services/ai.py @@ -1,7 +1,8 @@ from tenacity import retry, stop_after_attempt, wait_exponential import ollama +import json from fastapi import WebSocket -from typing import Optional +from typing import Optional, Dict, Any from app.core.logging import logger from app.core.config import settings @@ -12,10 +13,11 @@ async def load_prompt(prompt_path: str = None) -> str: """ if prompt_path is None: prompt_path = settings.PROMPT_FILE - try: with open(prompt_path, 'r') as file: - return file.read().strip() + content = file.read().strip() + logger.info(f"Loaded prompt from {prompt_path} (length: {len(content)} chars)") + return content except FileNotFoundError: logger.warning(f"Prompt file {prompt_path} not found, using default prompt") return """You are processing part of a document. Your task is to create a concise summary of this section while maintaining context with previous sections. @@ -27,39 +29,173 @@ async def load_prompt(prompt_path: str = None) -> str: 4. Use clear paragraph breaks for different topics 5. Avoid repeating information that was covered in previous sections""" -@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10)) -async def process_chunk(chunk: str, websocket: WebSocket, chunk_index: int, total_chunks: int, previous_summary: Optional[str] = None) -> str: +def extract_tag_content(text: str, tag: str) -> str: + """Extract content between XML-style tags, handling nested tags.""" + start_tag = f"<{tag}>" + end_tag = f"" + result_start = "" + result_end = "" + try: - prompt_template = await load_prompt('default') - context = f"This is chunk {chunk_index + 1} of {total_chunks}." + # First try direct tag extraction + start = text.index(start_tag) + len(start_tag) + end = text.index(end_tag) + return text[start:end].strip() + except ValueError: + try: + # Try extracting from within tags + result_content_start = text.index(result_start) + len(result_start) + result_content_end = text.index(result_end) + result_content = text[result_content_start:result_content_end].strip() + + # Now try to find our tag within the result content + start = result_content.index(start_tag) + len(start_tag) + end = result_content.index(end_tag) + return result_content[start:end].strip() + except ValueError: + return "" + +def validate_markdown_structure(summary: str) -> str: + """Ensure summary has proper markdown structure""" + lines = summary.split('\n') + if not any(line.startswith('#') for line in lines): + return f"# Summary\n\n{summary}" + return summary + +@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10)) +async def process_chunk( + chunk: str, + websocket: WebSocket, + chunk_index: int, + total_chunks: int, + previous_context: Optional[Dict[str, Any]] = None +) -> Dict[str, str]: + """ + Process a document chunk and return both context and summary. + + Args: + chunk: The text chunk to process + websocket: WebSocket connection for status updates + chunk_index: Current chunk index + total_chunks: Total number of chunks + previous_context: Context from previous chunk processing - # Pass more focused context about previous content - if previous_summary: - # Extract key points from previous summary to maintain flow - key_points = previous_summary.split('\n')[-5:] # Take last few key points - points_text = '\n'.join(key_points) - context = f"{context}\n\nKey points from previous section:\n{points_text}\n\nContinue the document flow, focusing on new information while maintaining narrative coherence." + Returns: + Dict containing 'context' and 'summary' keys + """ + try: + prompt_template = await load_prompt() - full_prompt = f"{prompt_template}\n\n{context}\n\nText to analyze:\n{chunk}" + # Initialize or maintain context + if previous_context is None: + previous_context = { + "metadata": { + "partial_titles": [], + "current_depth": 1, + "pending_sections": [] + }, + "content": { + "active_concepts": [], + "knowledge_chain": [] + } + } - logger.info(f"Processing chunk {chunk_index + 1}/{total_chunks} (length: {len(chunk)} chars):\n{chunk}") - logger.info("Chunk boundary marker ----") + # Prepare input section + inputs = f""" +{chunk}: str # Current text section for analysis +{chunk_index == 0}: bool # Start of new document flag +{json.dumps(previous_context, indent=2)}: dict # Previous context +""" + full_prompt = f"{prompt_template}\n\n{inputs}" + + logger.info(f"Processing chunk {chunk_index + 1}/{total_chunks} (length: {len(chunk)} chars)") + try: response = ollama.chat( model=settings.OLLAMA_MODEL, - messages=[{ - 'role': 'user', - 'content': full_prompt - }], + messages=[ + { + 'role': 'system', + 'content': '''Your response must be in this exact format: + +{ + "metadata": { + "partial_titles": [], + "current_depth": 1, + "pending_sections": [] + }, + "content": { + "active_concepts": [], + "knowledge_chain": [] + } +} + + + +# [Section Title] + +## Overview +[One paragraph overview] + +## New Concepts +### [Concept Name] +- Definition: [Clear definition] +- Example: [Concrete example] +- Prerequisites: [Required concepts] + +## Technical Implementation +[Implementation details with examples] + +## Related Concepts +[List of related concepts with brief connections] +''' + }, + { + 'role': 'user', + 'content': full_prompt + } + ], stream=False ) result = response['message']['content'] - logger.info(f"AI Output for chunk {chunk_index + 1}/{total_chunks} (length: {len(result)} chars):\n{result}") - return result + + # Add detailed logging of AI response + logger.debug(f"Raw AI response for chunk {chunk_index + 1}:\n{result}") + + # Extract both context and summary + context = extract_tag_content(result, "context") + summary = extract_tag_content(result, "summary") + + # Log extracted content + logger.debug(f"Extracted context for chunk {chunk_index + 1}:\n{context}") + logger.debug(f"Extracted summary for chunk {chunk_index + 1}:\n{summary}") + + if not context or not summary: + logger.warning(f"Missing context or summary in AI response for chunk {chunk_index + 1}") + logger.warning("AI response structure:\n" + "\n".join( + f"- Line {i+1}: {line[:100]}..." for i, line in enumerate(result.split('\n')) + )) + + # Parse context back into dictionary if present + try: + context_dict = json.loads(context) if context else previous_context + except json.JSONDecodeError: + logger.error(f"Failed to parse context JSON for chunk {chunk_index + 1}") + logger.error(f"Invalid context content:\n{context}") + context_dict = previous_context + + logger.info(f"AI Output for chunk {chunk_index + 1}/{total_chunks} (context: {len(context)} chars, summary: {len(summary)} chars)") + logger.info(f"Context: {context_dict}") + logger.info(f"Summary: {summary}") + + return { + "context": context_dict, + "summary": summary or result # Fallback to full response if no summary tag + } except Exception as e: - logger.error(f"Ollama processing failed: {str(e)}") + logger.error(f"AI processing error for chunk {chunk_index + 1}: {str(e)}") await websocket.send_json({ 'error': f'AI processing failed: {str(e)}' }) @@ -67,7 +203,4 @@ async def process_chunk(chunk: str, websocket: WebSocket, chunk_index: int, tota except Exception as e: logger.error(f"Chunk processing failed: {str(e)}") - await websocket.send_json({ - 'error': f'Chunk processing failed: {str(e)}' - }) raise \ No newline at end of file diff --git a/backend/app/services/pdf.py b/backend/app/services/pdf.py index bdd92c4..dc433a9 100644 --- a/backend/app/services/pdf.py +++ b/backend/app/services/pdf.py @@ -1,11 +1,13 @@ import fitz from fastapi import WebSocket +from typing import Dict, Any +import time from app.core.logging import logger from app.services.text_extraction import process_pdf_page from app.services.ai import process_chunk from app.utils.text import split_into_chunks -from app.utils.time import estimate_processing_time, estimate_remaining_time +from app.utils.time import estimator from app.core.config import settings async def process_pdf(pdf_path: str, websocket: WebSocket) -> str: @@ -51,37 +53,39 @@ async def process_pdf(pdf_path: str, websocket: WebSocket) -> str: 'total_chunks': total_chunks, 'current_chunk': 0, 'progress': 0, - 'estimated_time': estimate_processing_time(total_chunks) + 'estimated_time': estimator.estimate_total_time(total_chunks) }) # Process chunks with AI summaries = [] - running_summary = None + current_context: Dict[str, Any] = None for i, chunk in enumerate(chunks): - summary = await process_chunk( + result = await process_chunk( chunk, websocket, chunk_index=i, total_chunks=total_chunks, - previous_summary=running_summary + previous_context=current_context ) - summaries.append(summary) - # Keep only the most recent summary for context - running_summary = summary + summaries.append(result['summary']) + current_context = result['context'] await websocket.send_json({ 'status': 'analyzing', 'current_chunk': i + 1, 'total_chunks': total_chunks, 'progress': (i + 1) / total_chunks, - 'estimated_remaining': estimate_remaining_time(i + 1, total_chunks) + 'estimated_remaining': estimator.estimate_remaining_time(i + 1, total_chunks) }) # Combine summaries with proper section numbering and formatting sections = [] for i, summary in enumerate(summaries, 1): - section_header = f"## Section {i}" - sections.append(f"{section_header}\n\n{summary}") + if summary.startswith('#'): # If summary already has a heading + sections.append(summary) + else: # Add section number if no heading + section_header = f"## Section {i}" + sections.append(f"{section_header}\n\n{summary}") final_summary = "# Document Summary\n\n" + "\n\n".join(sections) logger.info(f"Final Combined Summary:\n{final_summary}") diff --git a/backend/app/utils/time.py b/backend/app/utils/time.py index b09eb94..d27a03a 100644 --- a/backend/app/utils/time.py +++ b/backend/app/utils/time.py @@ -1,20 +1,49 @@ -def estimate_processing_time(total_chunks: int) -> str: - """ - Estimate total processing time based on number of chunks. - """ - total_seconds = total_chunks * 30 - if total_seconds < 60: - return f"{total_seconds}s" - minutes = total_seconds // 60 - return f"{minutes}m" +from typing import List +from statistics import mean +from app.core.config import settings -def estimate_remaining_time(current_chunk: int, total_chunks: int) -> str: - """ - Estimate remaining processing time. - """ - remaining_chunks = total_chunks - current_chunk - remaining_seconds = remaining_chunks * 30 - if remaining_seconds < 60: - return f"{remaining_seconds}s" - minutes = remaining_seconds // 60 - return f"{minutes}m" \ No newline at end of file +class TimeEstimator: + def __init__(self): + self.processing_times: List[float] = [] + self.default_chunk_time = settings.DEFAULT_CHUNK_TIME # Add to settings + + def add_processing_time(self, seconds: float) -> None: + """Record actual processing time for a chunk.""" + self.processing_times.append(seconds) + + def get_average_chunk_time(self) -> float: + """Get average processing time per chunk.""" + if not self.processing_times: + return self.default_chunk_time + return mean(self.processing_times) + + def format_time(self, seconds: float) -> str: + """Format time duration with appropriate units and precision.""" + if seconds < 60: + return f"{int(seconds)}s" + + minutes = int(seconds // 60) + remaining_seconds = int(seconds % 60) + + if minutes == 0: + return f"{remaining_seconds}s" + elif remaining_seconds == 0: + return f"{minutes}m" + else: + return f"{minutes}m {remaining_seconds}s" + + def estimate_total_time(self, total_chunks: int) -> str: + """Estimate total processing time based on actual performance.""" + avg_time = self.get_average_chunk_time() + total_seconds = total_chunks * avg_time + return self.format_time(total_seconds) + + def estimate_remaining_time(self, current_chunk: int, total_chunks: int) -> str: + """Estimate remaining time based on actual performance.""" + remaining_chunks = total_chunks - current_chunk + avg_time = self.get_average_chunk_time() + remaining_seconds = remaining_chunks * avg_time + return self.format_time(remaining_seconds) + +# Global estimator instance +estimator = TimeEstimator() \ No newline at end of file diff --git a/backend/prompts/default.txt b/backend/prompts/default.txt index 446f15f..b807709 100644 --- a/backend/prompts/default.txt +++ b/backend/prompts/default.txt @@ -1,3 +1,17 @@ + +{$TEXT_CHUNK} +{$IS_FIRST_CHUNK} +{$PREVIOUS_SUMMARY} + + + +1. Analyze current content and context +2. Extract key technical information +3. Create structured summary +4. Format with proper markdown + + + You are an AI assistant creating a focused technical book summary. GUIDELINES: @@ -30,4 +44,5 @@ Remember: - Maintain narrative flow with previous sections - Focus on technical content and concepts - Be concise but thorough -- Avoid repeating information from previous sections \ No newline at end of file +- Avoid repeating information from previous sections + \ No newline at end of file diff --git a/backend/prompts/gpt.txt b/backend/prompts/gpt.txt new file mode 100644 index 0000000..6cb5c18 --- /dev/null +++ b/backend/prompts/gpt.txt @@ -0,0 +1,141 @@ + +{$TEXT_CHUNK}: The current section of text for analysis. +{$IS_FIRST_CHUNK}: Boolean indicating if this is the start of a new document. +{$PREVIOUS_CONTEXT}: Context from prior chunks, including key concepts, summaries, and dependencies. + + + +1. Establish document metadata handling +2. Define content processing workflow +3. Specify progressive knowledge building +4. Detail markdown formatting rules +5. Define educational structuring principles + + + +You will create educational summaries of technical non-fiction content that maintain the pedagogical value of the original while being more concise. Your goal is to help readers grasp complex technical concepts efficiently while ensuring proper knowledge progression. + +It is absolutely paramount that you do not repeat information from previous sections. Ignore personal information like acknowledgements, dedications, etc. and focus on the technical content. + +Input Variables: +- TEXT_CHUNK: Current portion of text to analyze +- IS_FIRST_CHUNK: Boolean indicating if this is the start of a new document +- PREVIOUS_CONTEXT: Previously generated context including metadata, key concepts, and content structure + +First, analyze the text chunk: + + +1. Document identifiers: + - Title and subtitle + - Chapter numbers and titles + - Section headings + - Version information if present + +2. Content structure: + - Major topic divisions + - Concept hierarchies + - Knowledge dependencies + - Technical terminology introductions + + +Then process the content following these principles: + +1. Progressive Knowledge Building: + - Identify prerequisite concepts + - Track concept introduction order + - Maintain pedagogical sequence + - Link related concepts + +2. Content Processing: + - Remove redundant explanations + - Preserve technical accuracy + - Maintain example clarity + - Keep critical context + +3. Educational Value: + - Clarify complex concepts + - Keep illuminating examples + - Preserve learning scaffolds + - Maintain conceptual flow + +Generate your summary in Markdown using this structure: + +```markdown +# Title + +## Metadata +- Source: [book title] +- Section: [current section] +- Prerequisites: [required prior knowledge] + +## Conceptual Overview +[High-level explanation of main ideas] + +## Key Concepts +### [Concept Name] +[Clear, concise explanation] +[Practical example or application] + +## Technical Details +### [Technical Topic] +[Detailed technical explanation] +```code sample or configuration example``` + +## Related Concepts +- Links to previously introduced concepts +- Forward references to advanced topics + +## Summary +[Concise recap of main points] +``` + +Follow these formatting guidelines: +1. Structural Elements: + - Use proper heading hierarchy (h1 > h2 > h3) + - Include clear section breaks + - Maintain consistent indentation + +2. Technical Content: + - Code blocks for syntax, configs, commands + - Tables for structured data + - Blockquotes for important definitions + - Inline code for technical terms + +3. Concept Presentation: + - Bold for new term introduction + - Italic for emphasis + - Lists for sequential steps + - Diagrams described in text + +Before outputting, analyze your work: + + +1. Verify: + - Technical accuracy maintained + - Prerequisites properly identified + - Concepts clearly explained + - Examples support understanding + - Progressive learning path preserved + +2. Ensure: + - No orphaned references + - Clear concept connections + - Proper term introduction + - Logical flow maintained + +3. Quality Check: + - After generating the summary: + - Ensure all concepts are logically linked. + - Avoid redundancy with `$PREVIOUS_CONTEXT`. + - Verify technical clarity and Markdown formatting consistency. + + +Output your result in tags, preceded by tags containing: +- Current document position +- Active concept threads +- Pending introductions +- Knowledge dependencies + +Remember: Focus on maintaining educational value while increasing efficiency of learning. Every included element should serve the reader's understanding of the material. + + \ No newline at end of file diff --git a/backend/prompts/mistral.txt b/backend/prompts/mistral.txt new file mode 100644 index 0000000..a5735f3 --- /dev/null +++ b/backend/prompts/mistral.txt @@ -0,0 +1,286 @@ + +{$TEXT_CHUNK}: str # Current text section for analysis +{$IS_FIRST_CHUNK}: bool # Start of new document flag +{$PREVIOUS_CONTEXT}: dict { + "metadata": { + "partial_titles": [str], # Incomplete section/chapter titles + "current_depth": int, # Current heading depth + "pending_sections": [ # Sections waiting for completion + { + "text": str, # Partial section text + "depth": int, # Heading depth + "status": "incomplete" | "complete" + } + ] + }, + "content": { + "active_concepts": [str], # Currently discussed technical concepts + "knowledge_chain": [ # Prerequisite relationships + { + "concept": str, + "requires": [str] + } + ] + } +} + + +1. Process document metadata +2. Extract and track concepts +3. Generate educational summary +4. Perform quality validation +5. Format output in Markdown + + +Your task: Create technical summaries that teach effectively while being concise. + +1. Process Text Structure +DO: +- Join split headings from chunk boundaries + Example: If previous chunk ends with "3.1 Advanced" and current starts with "Neural Networks", combine as "3.1 Advanced Neural Networks" +- Track section depth using #, ##, ### in Markdown +- Note incomplete sections in pending_sections list + +DON'T: +- Create new section numbers +- Modify existing heading hierarchy +- Lose partial headings at chunk boundaries + +2. Handle Technical Concepts +DO: +- Track concept dependencies explicitly + Example: If explaining backpropagation, list "gradients", "chain rule" as prerequisites +- Define new terms when first introduced + Example: "Gradient descent: An optimization algorithm that..." +- Link related concepts using inline references + +DON'T: +- Redefine terms from previous chunks +- Break prerequisite chains +- Skip concept definitions + +3. Generate Summary Content +DO: +- Use concrete examples for abstract concepts + Example: Explain CNNs using image processing examples +- Keep critical implementation details + Example: Include key parameters in code examples +- Link to previous concepts using exact terms + +DON'T: +- Include personal anecdotes +- Repeat examples from previous chunks +- Use ambiguous references + +4. Handle Edge Cases + +Boundary Content: +DO: +- Tables: Store partial tables in pending_sections until complete + Example: If table header is in previous chunk, reference it in current chunk +- Code: Buffer incomplete functions until full definition available + Example: Store partial function in pending_sections.code_blocks +- Equations: Track equation numbers and parts across chunks + Example: Tag equation fragments with shared ID +- Lists: Maintain list context across chunk boundaries + Example: Continue numbering from previous chunk + +DON'T: +- Start new table header if previous table incomplete +- Generate partial code blocks +- Fragment equations mid-expression +- Reset list numbering arbitrarily + +References: +DO: +- Track unresolved forward references in pending_concepts +- Maintain citation context across chunks +- Cache figure/table references until content appears +- Link cross-references to specific chunk IDs + +DON'T: +- Introduce citations without context +- Leave unresolved references +- Drop figure/table numbers + +Formatting: +DO: +- Preserve nested list depth across chunks +- Maintain consistent code indentation +- Convert PDF-specific formatting to Markdown +- Track open/close states of nested structures + +DON'T: +- Break nested lists mid-hierarchy +- Mix formatting standards +- Lose code block language specifications + +5. Content Organization Rules + +Metadata Handling: +DO: +- Store book metadata (titles, figures, tables) in separate metadata registry +- Reference figures/tables only when discussing their content +- Group chapter summaries into coherent units + +DON'T: +- Mix metadata listings with content summaries +- List figures/tables without context +- Duplicate structural information across chunks + +Content Flow: +DO: +- Use transition markers between chunks: + { + "previous_topic": "last major concept from previous chunk", + "current_topic": "main concept in this chunk", + "continuation_type": "expansion|new_topic|example" + } +- Track narrative threads across chunks: + { + "thread_id": "unique_identifier", + "status": "ongoing|concluded", + "key_points": ["point1", "point2"], + "dependencies": ["thread_id1", "thread_id2"] + } + +DON'T: +- Start new sections with generic transitions +- Repeat context unnecessarily +- Break narrative flow between chunks + +Example Resolution: +DO: +- Tag examples with unique IDs: + { + "example_id": "netflix_aws_outage", + "first_mention": chunk_id, + "components": ["availability", "resilience"], + "status": "complete|partial" + } +- Track example components across chunks +- Ensure complete example coverage + +DON'T: +- Fragment examples across chunks without tracking +- Repeat example setup in multiple chunks +- Leave examples incomplete + +6. Enhanced Quality Validation + +Before Generating Output: +1. Narrative Consistency Check: + - Scan previous chunks for related content + - Identify potential duplications + - Verify example completeness + - Check thread continuity + +2. Reference Validation: + - Verify all figures/tables are contextually introduced + - Confirm cross-references resolve to available content + - Check citation completeness + - Validate external link context + +3. Content Deduplication: + - Compare current content against previous chunks + - Flag potential repetition of: + - Examples + - Definitions + - Technical explanations + - Resolve overlapping content + +4. Structure Verification: + - Validate heading hierarchy + - Check list consistency + - Verify section transitions + - Confirm metadata placement + +5. Knowledge Flow Analysis: + - Verify concept prerequisites + - Check example dependencies + - Validate learning progression + - Confirm narrative thread completion + +If Validation Fails: +1. Log specific validation failure +2. Attempt auto-correction if possible +3. Flag content for human review if needed +4. Provide specific error context in output + +7. Format Consistency and Self-Review + +Before finalizing output: +1. Format Verification: + - Ensure all headings follow consistent hierarchy + - Verify concept formatting patterns: + * New terms: `**term**` on first use + * Technical references: `inline code` + * Section headers: proper markdown levels + - Check list formatting consistency + - Validate code block language tags + +2. Content Self-Review: + - Compare concept explanations across chunks + - Verify technical accuracy of summaries + - Check for logical flow between sections + - Ensure prerequisites are properly linked + +3. Summary Enhancement: + - Review initial draft for clarity + - Strengthen concept connections + - Add missing context where needed + - Remove redundant explanations + +4. Final Quality Gates: + - Concept consistency check + - Format compliance verification + - Technical accuracy validation + - Educational value assessment + +If any check fails: +1. Log the specific isYsue +2. Apply correction rules +3. Re-verify after changes +4. Document changes in context + +Required Output Format: + +```markdown +# [Complete Section Title] + +## Overview +[One concise paragraph explaining the main topic and its significance] + +## Key Concepts +[For each new concept:] +### **[Concept Name]** +- **Definition**: [Clear, concise definition] +- **Example**: [Concrete, practical example] +- **Prerequisites**: `[concept1]`, `[concept2]` + +## Technical Implementation +- Code blocks must be complete and runnable +- Include typical parameter values +- Note common pitfalls + +Example: +```python +# CNN layer implementation +conv_layer = nn.Conv2d( + in_channels=3, # RGB input + out_channels=16, # Number of filters + kernel_size=3, # 3x3 kernel + padding=1 # Preserve spatial dimensions +) +``` + +## Related Concepts +Only list concepts that are: +- Directly prerequisite to this section +- Immediately built upon by this section +Example: For CNNs, list "neural networks", "backpropagation", link to "pooling layers" + +Generate output in tags containing: +1. Updated metadata and concept tracking +2. Formatted technical content summary + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1c9b069..00c7662 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,9 @@ services: - "${FRONTEND_PORT}:${FRONTEND_PORT}" networks: - app_network + environment: + VERSION: ${FE_VERSION} + FRONTEND_PORT: ${FRONTEND_PORT} backend: build: @@ -20,7 +23,10 @@ services: - "${BACKEND_PORT}:${BACKEND_PORT}" networks: - app_network + depends_on: + - ollama environment: + BACKEND_PORT: ${BACKEND_PORT} CORS_ORIGINS: '["http://localhost:${FRONTEND_PORT}"]' OLLAMA_HOST: "http://ollama:${OLLAMA_PORT}" OLLAMA_MODEL: ${OLLAMA_MODEL} @@ -29,9 +35,8 @@ services: PROMPT_FILE: ${PROMPT_FILE} HEALTH_CHECK_TIMEOUT: ${HEALTH_CHECK_TIMEOUT} OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT} + DEFAULT_CHUNK_TIME: ${DEFAULT_CHUNK_TIME} PYTHONUNBUFFERED: 1 - depends_on: - - ollama ollama: build: ./ollama diff --git a/docs/setup/local.md b/docs/setup/local.md index 318d3c9..6b2f4e0 100644 --- a/docs/setup/local.md +++ b/docs/setup/local.md @@ -25,9 +25,10 @@ bun install bun run dev cd ../backend -python -m venv venv -source venv/bin/activate -pip install -r requirements.txt +pip install uv +uv .venv +source .venv/bin/activate +uv pip install -r requirements.txt uvicorn main:app --reload ```