refactor: improve code quality and streamline unit tests

.pre-commit-config.yaml

@@ -1,36 +1,68 @@
-# Pre-commit hooks configuration
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
-      - id: check-json
-      - id: check-toml
-      - id: check-added-large-files
-      - id: check-merge-conflict
-      - id: debug-statements
+# Pre-commit hooks for madengine
+# See https://pre-commit.com for more information
+# Install with: pip install pre-commit && pre-commit install
 
+repos:
+  # Code formatting
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3
+        args: ['--line-length=88']
 
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+  # Import sorting
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
     hooks:
       - id: isort
+        args: ['--profile=black', '--line-length=88']
 
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+  # Linting
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.1.1
     hooks:
       - id: flake8
+        args:
+          - '--max-line-length=88'
+          - '--extend-ignore=E203,E501,W503'
+          - '--exclude=venv,build,dist,.git,__pycache__,.pytest_cache'
 
+  # Type checking (optional - can be slow)
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.11.2
     hooks:
       - id: mypy
-        additional_dependencies: [types-requests, types-PyYAML]
-        exclude: ^(tests/|scripts/)
+        additional_dependencies:
+          - types-PyYAML
+          - types-toml
+        args: ['--ignore-missing-imports', '--check-untyped-defs']
+        files: ^src/madengine/
+        exclude: ^(tests/|src/madengine/scripts/)
+
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: '\.md$'
+      - id: end-of-file-fixer
+        exclude: '\.md$'
+      - id: check-yaml
+        args: ['--allow-multiple-documents']
+      - id: check-json
+      - id: check-toml
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: mixed-line-ending
+        args: ['--fix=lf']
+
+  # Security checks
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.10
+    hooks:
+      - id: bandit
+        args: ['-c', 'pyproject.toml', '--skip', 'B101']
+        additional_dependencies: ['bandit[toml]']

.pre-commit-setup.md

@@ -0,0 +1,62 @@
+# Pre-commit Hooks Setup
+
+## Installation
+
+```bash
+# Install pre-commit
+pip install pre-commit
+
+# Install the git hooks
+pre-commit install
+
+# (Optional) Run against all files to verify
+pre-commit run --all-files
+```
+
+## What Gets Checked
+
+- **black**: Code formatting (88 char line length)
+- **isort**: Import statement sorting  
+- **flake8**: Linting and style checking
+- **mypy**: Type checking (on src/madengine only)
+- **bandit**: Security vulnerability scanning
+- **General checks**: trailing whitespace, YAML/JSON validity, large files, merge conflicts
+
+## Usage
+
+Pre-commit will now run automatically on `git commit`. If any hook fails:
+
+1. Review the output
+2. Fix the issues (many are auto-fixed)
+3. Stage the changes: `git add <files>`
+4. Commit again: `git commit`
+
+## Skipping Hooks (Not Recommended)
+
+```bash
+# Skip all hooks (emergency only)
+git commit --no-verify
+
+# Skip specific hooks
+SKIP=mypy git commit
+```
+
+## Manual Run
+
+```bash
+# Run on staged files
+pre-commit run
+
+# Run on all files
+pre-commit run --all-files
+
+# Run specific hook
+pre-commit run black --all-files
+```
+
+## Updating Hooks
+
+```bash
+# Update to latest versions
+pre-commit autoupdate
+```

pyproject.toml

@@ -180,3 +180,7 @@ exclude_lines = [
     "class .*\\bProtocol\\):",
     "@(abc\\.)?abstractmethod",
 ]
+
+[tool.bandit]
+exclude_dirs = ["tests", "venv", "build", "dist", ".git"]
+skips = ["B101"]  # Skip assert_used (used in tests)

src/madengine/cli/commands/run.py

@@ -22,7 +22,7 @@
 from madengine.core.errors import (
     BuildError,
     ConfigurationError,
-    RuntimeError as MADRuntimeError,
+    ExecutionError,
 )
 
 from ..constants import (
@@ -436,7 +436,7 @@ def run(
             for suggestion in e.suggestions:
                 console.print(f"  • {suggestion}")
         raise typer.Exit(ExitCode.BUILD_FAILURE)
-    except MADRuntimeError as e:
+    except ExecutionError as e:
         # Runtime execution errors
         console.print(f"💥 [bold red]Runtime error: {e}[/bold red]")
         if hasattr(e, 'suggestions') and e.suggestions:

src/madengine/core/errors.py

@@ -109,19 +109,23 @@ def __init__(self, message: str, context: Optional[ErrorContext] = None, **kwarg
         )
 
 
-class RuntimeError(MADEngineError):
+class ExecutionError(MADEngineError):
     """Runtime execution errors."""
-    
+
     def __init__(self, message: str, context: Optional[ErrorContext] = None, **kwargs):
         super().__init__(
-            message, 
-            ErrorCategory.RUNTIME, 
-            context, 
+            message,
+            ErrorCategory.RUNTIME,
+            context,
             recoverable=False,
             **kwargs
         )
 
 
+# Backward compatibility alias
+RuntimeError = ExecutionError
+
+
 class BuildError(MADEngineError):
     """Build and compilation errors."""
 

src/madengine/database/mongodb.py

@@ -737,13 +737,13 @@ def __init__(self, args):
     def run(self) -> bool:
         """Execute the MongoDB upload operation."""
         logger.warning("MongoDBHandler is deprecated. Use upload_file_to_mongodb instead.")
-        
-        print("\n" + "=" * 80)
-        print("📤 UPLOADING TO MONGODB")
-        print("=" * 80)
-        print(f"📂 File: {self.file_path}")
-        print(f"🗄️  Database: {self.database_name}")
-        print(f"📊 Collection: {self.collection_name}")
+
+        console.print("\n" + "=" * 80)
+        console.print("[bold blue]📤 UPLOADING TO MONGODB[/bold blue]")
+        console.print("=" * 80)
+        console.print(f"📂 File: [cyan]{self.file_path}[/cyan]")
+        console.print(f"🗄️  Database: [cyan]{self.database_name}[/cyan]")
+        console.print(f"📊 Collection: [cyan]{self.collection_name}[/cyan]")
 
         try:
             # Parse unique fields if provided
@@ -760,28 +760,28 @@ def run(self) -> bool:
                 config=self.config,
                 options=options
             )
-            
-            print(f"✅ Successfully processed {result.documents_processed} documents")
-            print(f"   Inserted: {result.documents_inserted}")
-            print(f"   Updated: {result.documents_updated}")
-            print("=" * 80 + "\n")
-            
+
+            console.print(f"✅ [bold green]Successfully processed {result.documents_processed} documents[/bold green]")
+            console.print(f"   Inserted: {result.documents_inserted}")
+            console.print(f"   Updated: {result.documents_updated}")
+            console.print("=" * 80 + "\n")
+
             self.return_status = True
-            
+
         except FileNotFoundError as e:
-            print(f"❌ Error: {e}")
+            console.print(f"[bold red]❌ Error:[/bold red] {e}")
             self.return_status = False
         except ConnectionFailure as e:
-            print(f"❌ MongoDB connection failed: {e}")
-            print("💡 Tip: Check MONGO_HOST, MONGO_PORT, MONGO_USER, MONGO_PASSWORD")
+            console.print(f"[bold red]❌ MongoDB connection failed:[/bold red] {e}")
+            console.print("[yellow]💡 Tip: Check MONGO_HOST, MONGO_PORT, MONGO_USER, MONGO_PASSWORD[/yellow]")
             self.return_status = False
         except ValueError as e:
-            print(f"❌ Invalid file: {e}")
+            console.print(f"[bold red]❌ Invalid file:[/bold red] {e}")
             self.return_status = False
         except Exception as e:
-            print(f"❌ Unexpected error: {e}")
+            console.print(f"[bold red]❌ Unexpected error:[/bold red] {e}")
             logger.exception("MongoDB upload failed")
             self.return_status = False
-        
-        print("=" * 80 + "\n")
+
+        console.print("=" * 80 + "\n")
         return self.return_status

src/madengine/orchestration/run_orchestrator.py

@@ -26,7 +26,7 @@
 from madengine.core.errors import (
     BuildError,
     ConfigurationError,
-    RuntimeError as MADRuntimeError,
+    ExecutionError,
     create_error_context,
     handle_error,
 )
@@ -158,7 +158,7 @@ def execute(
 
         Raises:
             ConfigurationError: If neither manifest nor tags provided
-            MADRuntimeError: If execution fails
+            ExecutionError: If execution fails
         """
         self.rich_console.print(f"\n[dim]{'=' * 60}[/dim]")
         self.rich_console.print("[bold blue]🚀 RUN PHASE[/bold blue]")
@@ -321,14 +321,14 @@ def execute(
                 self._cleanup_model_dir_copies()
                 raise
 
-        except (ConfigurationError, MADRuntimeError, BuildError):
+        except (ConfigurationError, ExecutionError, BuildError):
             raise
         except Exception as e:
             context = create_error_context(
                 operation="run_phase",
                 component="RunOrchestrator",
             )
-            raise MADRuntimeError(
+            raise ExecutionError(
                 f"Run phase failed: {e}",
                 context=context,
                 suggestions=[
@@ -604,7 +604,7 @@ def _execute_local(self, manifest_file: str, timeout: int) -> Dict:
             )
 
             if not compatible_images:
-                raise MADRuntimeError(
+                raise ExecutionError(
                     f"No compatible images for GPU vendor '{runtime_gpu_vendor}' and architecture '{runtime_gpu_arch}'",
                     context=create_error_context(
                         operation="filter_images",