Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 42 additions & 33 deletions cortex/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,14 @@
from dataclasses import asdict, dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any, List, Optional, Tuple
from typing import Any

from rich import box
from rich.console import Console
from rich.panel import Panel
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
from rich.table import Table

from cortex.branding import CORTEX_CYAN, console, cx_header, cx_print
from cortex.branding import console, cx_header, cx_print

# Model recommendations based on system capabilities
MODEL_REQUIREMENTS = {
Expand Down Expand Up @@ -118,7 +117,9 @@ def _get_system_info(self) -> dict:
elif platform.system() == "Darwin":
result = subprocess.run(
["sysctl", "-n", "machdep.cpu.brand_string"],
capture_output=True, text=True, timeout=5
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0:
info["cpu_model"] = result.stdout.strip()
Expand All @@ -139,8 +140,7 @@ def _get_system_info(self) -> dict:
break
elif platform.system() == "Darwin":
result = subprocess.run(
["sysctl", "-n", "hw.memsize"],
capture_output=True, text=True, timeout=5
["sysctl", "-n", "hw.memsize"], capture_output=True, text=True, timeout=5
)
if result.returncode == 0:
mem_bytes = int(result.stdout.strip())
Expand All @@ -160,7 +160,9 @@ def _detect_nvidia_gpu(self) -> bool:
try:
result = subprocess.run(
["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],
capture_output=True, text=True, timeout=10
capture_output=True,
text=True,
timeout=10,
)
return result.returncode == 0 and result.stdout.strip() != ""
except Exception:
Expand All @@ -171,7 +173,9 @@ def _get_nvidia_vram(self) -> int:
try:
result = subprocess.run(
["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=10
capture_output=True,
text=True,
timeout=10,
)
if result.returncode == 0:
return int(result.stdout.strip().split("\n")[0])
Expand Down Expand Up @@ -223,7 +227,7 @@ def _benchmark_cpu(self) -> BenchmarkResult:
score=score,
raw_value=round(avg_time * 1000, 2),
unit="ms",
description="Matrix computation speed"
description="Matrix computation speed",
)

def _benchmark_memory(self) -> BenchmarkResult:
Expand All @@ -250,7 +254,7 @@ def _benchmark_memory(self) -> BenchmarkResult:

# Calculate approximate bandwidth (bytes per second)
bytes_processed = size * 8 * 2 # 8 bytes per int, 2 operations
bandwidth_gbps = (bytes_processed / avg_time) / (1024 ** 3)
bandwidth_gbps = (bytes_processed / avg_time) / (1024**3)

# Score based on bandwidth
# Baseline: 10 GB/s = 50, 50 GB/s = 100, 1 GB/s = 10
Expand All @@ -267,7 +271,7 @@ def _benchmark_memory(self) -> BenchmarkResult:
score=score,
raw_value=round(bandwidth_gbps, 2),
unit="GB/s",
description="Memory throughput"
description="Memory throughput",
)

def _benchmark_gpu(self, system_info: dict) -> BenchmarkResult:
Expand Down Expand Up @@ -298,7 +302,7 @@ def _benchmark_gpu(self, system_info: dict) -> BenchmarkResult:
score=score,
raw_value=vram_mb,
unit="MB",
description="NVIDIA GPU VRAM"
description="NVIDIA GPU VRAM",
)

elif system_info.get("has_apple_silicon"):
Expand All @@ -320,7 +324,7 @@ def _benchmark_gpu(self, system_info: dict) -> BenchmarkResult:
score=score,
raw_value=int(ram_gb * 1024),
unit="MB (unified)",
description="Apple Silicon unified memory"
description="Apple Silicon unified memory",
)

else:
Expand All @@ -330,7 +334,7 @@ def _benchmark_gpu(self, system_info: dict) -> BenchmarkResult:
score=15,
raw_value=0,
unit="MB",
description="No dedicated GPU detected"
description="No dedicated GPU detected",
)

def _benchmark_inference_simulation(self) -> BenchmarkResult:
Expand All @@ -348,9 +352,11 @@ def _benchmark_inference_simulation(self) -> BenchmarkResult:
# Simulate embedding lookup (string hashing)
embeddings = [hash(token) % 10000 for token in tokens]
# Simulate attention (nested loops)
attention = sum(embeddings[i] * embeddings[j]
for i in range(min(50, len(embeddings)))
for j in range(min(50, len(embeddings))))
attention = sum(
embeddings[i] * embeddings[j]
for i in range(min(50, len(embeddings)))
for j in range(min(50, len(embeddings)))
)
_ = attention
elapsed = time.perf_counter() - start

Expand All @@ -372,7 +378,7 @@ def _benchmark_inference_simulation(self) -> BenchmarkResult:
score=score,
raw_value=round(tokens_per_sec / 1000, 2),
unit="K tok/s",
description="Simulated inference throughput"
description="Simulated inference throughput",
)

def _benchmark_token_generation(self) -> BenchmarkResult:
Expand All @@ -390,8 +396,10 @@ def _benchmark_token_generation(self) -> BenchmarkResult:
context = [0] * 10
for _ in range(sequence_length):
# Simulate softmax over vocabulary
logits = [(hash((i, tuple(context[-10:]))) % 1000) / 1000
for i in range(min(1000, vocab_size))]
logits = [
(hash((i, tuple(context[-10:]))) % 1000) / 1000
for i in range(min(1000, vocab_size))
]
next_token = max(range(len(logits)), key=lambda i: logits[i])
generated.append(next_token)
context.append(next_token)
Expand All @@ -415,7 +423,7 @@ def _benchmark_token_generation(self) -> BenchmarkResult:
score=score,
raw_value=round(tokens_per_sec, 1),
unit="tok/s",
description="Simulated generation speed"
description="Simulated generation speed",
)

def _calculate_overall_score(self, results: list[BenchmarkResult]) -> tuple[int, str]:
Expand Down Expand Up @@ -579,8 +587,9 @@ def run(self, save_history: bool = True) -> BenchmarkReport:
report.overall_score, report.rating = self._calculate_overall_score(report.results)

# Get model recommendations
report.can_run, report.needs_upgrade, report.upgrade_suggestion = \
report.can_run, report.needs_upgrade, report.upgrade_suggestion = (
self._get_model_recommendations(report.system_info, report.overall_score)
)

# Save to history
if save_history:
Expand Down Expand Up @@ -633,11 +642,7 @@ def display_report(self, report: BenchmarkReport):
else:
score_str = f"[red]{result.score}/100[/red]"

table.add_row(
result.name,
score_str,
f"{result.raw_value} {result.unit}"
)
table.add_row(result.name, score_str, f"{result.raw_value} {result.unit}")

console.print(table)
console.print()
Expand All @@ -650,12 +655,16 @@ def display_report(self, report: BenchmarkReport):
else:
score_color = "red"

score_content = f"[bold {score_color}]{report.overall_score}/100[/bold {score_color}] ({report.rating})"
console.print(Panel(
f"[bold]OVERALL SCORE:[/bold] {score_content}",
border_style="cyan",
box=box.ROUNDED,
))
score_content = (
f"[bold {score_color}]{report.overall_score}/100[/bold {score_color}] ({report.rating})"
)
console.print(
Panel(
f"[bold]OVERALL SCORE:[/bold] {score_content}",
border_style="cyan",
box=box.ROUNDED,
)
)
console.print()

# Model recommendations
Expand Down
6 changes: 3 additions & 3 deletions cortex/branding.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
- Consistent visual language
"""

from typing import List, Optional, Tuple

from rich import box
from rich.console import Console
from rich.panel import Panel
Expand Down Expand Up @@ -318,7 +316,9 @@ def cx_error(message: str) -> None:

def cx_warning(message: str) -> None:
"""Print a warning message with warning icon."""
console.print(f"[{CORTEX_WARNING}]⚠[/{CORTEX_WARNING}] [{CORTEX_WARNING}]{message}[/{CORTEX_WARNING}]")
console.print(
f"[{CORTEX_WARNING}]⚠[/{CORTEX_WARNING}] [{CORTEX_WARNING}]{message}[/{CORTEX_WARNING}]"
)


def cx_info(message: str) -> None:
Expand Down
44 changes: 26 additions & 18 deletions cortex/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1648,9 +1648,7 @@ def progress_callback(message: str, percent: float) -> None:
"success",
)
if result.duration_seconds:
console.print(
f"[dim]Completed in {result.duration_seconds:.1f}s[/dim]"
)
console.print(f"[dim]Completed in {result.duration_seconds:.1f}s[/dim]")
elif result.status == UpdateStatus.PENDING:
# Dry run
cx_print(
Expand Down Expand Up @@ -2931,9 +2929,7 @@ def main():
f"[cyan]🔔 Cortex update available:[/cyan] "
f"[green]{update_release.version}[/green]"
)
console.print(
" [dim]Run 'cortex update' to upgrade[/dim]"
)
console.print(" [dim]Run 'cortex update' to upgrade[/dim]")
console.print()
except Exception:
pass # Don't block CLI on update check failures
Expand Down Expand Up @@ -2987,7 +2983,7 @@ def main():
nargs="?",
default="status",
choices=["status", "diagnose", "deps"],
help="Action: status (default), diagnose, deps"
help="Action: status (default), diagnose, deps",
)
systemd_parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")

Expand All @@ -2998,9 +2994,11 @@ def main():
nargs="?",
default="status",
choices=["status", "modes", "switch", "apps"],
help="Action: status (default), modes, switch, apps"
help="Action: status (default), modes, switch, apps",
)
gpu_parser.add_argument(
"mode", nargs="?", help="Mode for switch action (integrated/hybrid/nvidia)"
)
gpu_parser.add_argument("mode", nargs="?", help="Mode for switch action (integrated/hybrid/nvidia)")
gpu_parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")

# Printer/Scanner setup command
Expand All @@ -3010,7 +3008,7 @@ def main():
nargs="?",
default="status",
choices=["status", "detect"],
help="Action: status (default), detect"
help="Action: status (default), detect",
)
printer_parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")

Expand Down Expand Up @@ -3477,7 +3475,8 @@ def main():
help="Action to perform (default: status)",
)
wifi_parser.add_argument(
"-v", "--verbose",
"-v",
"--verbose",
action="store_true",
help="Enable verbose output",
)
Expand All @@ -3504,7 +3503,8 @@ def main():
help="Truncation mode for large input (default: middle)",
)
stdin_parser.add_argument(
"-v", "--verbose",
"-v",
"--verbose",
action="store_true",
help="Enable verbose output",
)
Expand All @@ -3524,7 +3524,8 @@ def main():
help="Package constraints (format: pkg:constraint:source)",
)
deps_parser.add_argument(
"-v", "--verbose",
"-v",
"--verbose",
action="store_true",
help="Enable verbose output",
)
Expand All @@ -3539,7 +3540,8 @@ def main():
help="Action to perform (default: check)",
)
health_parser.add_argument(
"-v", "--verbose",
"-v",
"--verbose",
action="store_true",
help="Enable verbose output",
)
Expand Down Expand Up @@ -3574,18 +3576,17 @@ def main():
return cli.systemd(
args.service,
action=getattr(args, "action", "status"),
verbose=getattr(args, "verbose", False)
verbose=getattr(args, "verbose", False),
)
elif args.command == "gpu":
return cli.gpu(
action=getattr(args, "action", "status"),
mode=getattr(args, "mode", None),
verbose=getattr(args, "verbose", False)
verbose=getattr(args, "verbose", False),
)
elif args.command == "printer":
return cli.printer(
action=getattr(args, "action", "status"),
verbose=getattr(args, "verbose", False)
action=getattr(args, "action", "status"), verbose=getattr(args, "verbose", False)
)
elif args.command == "ask":
return cli.ask(args.question)
Expand Down Expand Up @@ -3624,25 +3625,30 @@ def main():
return cli.env(args)
elif args.command == "upgrade":
from cortex.licensing import open_upgrade_page

open_upgrade_page()
return 0
elif args.command == "license":
from cortex.licensing import show_license_status

show_license_status()
return 0
elif args.command == "activate":
from cortex.licensing import activate_license

return 0 if activate_license(args.license_key) else 1
elif args.command == "update":
return cli.update(args)
elif args.command == "wifi":
from cortex.wifi_driver import run_wifi_driver

return run_wifi_driver(
action=getattr(args, "action", "status"),
verbose=getattr(args, "verbose", False),
)
elif args.command == "stdin":
from cortex.stdin_handler import run_stdin_handler

return run_stdin_handler(
action=getattr(args, "action", "info"),
max_lines=getattr(args, "max_lines", 1000),
Expand All @@ -3651,13 +3657,15 @@ def main():
)
elif args.command == "deps":
from cortex.semver_resolver import run_semver_resolver

return run_semver_resolver(
action=getattr(args, "action", "analyze"),
packages=getattr(args, "packages", None),
verbose=getattr(args, "verbose", False),
)
elif args.command == "health":
from cortex.health_score import run_health_check

return run_health_check(
action=getattr(args, "action", "check"),
verbose=getattr(args, "verbose", False),
Expand Down
Loading
Loading