diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..7b27466b --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +# Environment configuration template for Exosphere State Manager +# Copy this file to .env and fill in your actual values. +# NEVER commit .env with real secrets to the repository. + +# MongoDB connection string (required) +MONGO_URI=mongodb+srv://username:password@cluster.example.mongodb.net/ + +# MongoDB database name (optional, defaults to exosphere) +MONGO_DATABASE_NAME=exosphere + +# API authentication secret - generate a secure random string (required) +# Example: openssl rand -hex 32 +STATE_MANAGER_SECRET=changeme + +# Encryption key for secrets - generate a secure random string (required) +# Example: openssl rand -hex 32 +SECRETS_ENCRYPTION_KEY=changeme + +# API key for dashboard access (required) +EXOSPHERE_API_KEY=changeme + +# TTL in days for runs collection (optional, defaults to 30) +# Override with RUN_TTL_DAYS for runs-specific TTL +TTL_DAYS=30 +RUN_TTL_DAYS=30 diff --git a/.gitignore b/.gitignore index 774ebbf6..5d1b9e3b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,26 @@ -# Ignore temp directory and temp files at repository root -/temp* -!/temp/.gitkeep \ No newline at end of file +# node +node_modules/ +npm-debug.log +yarn-debug.log +yarn-error.log + +# env (keep .env.example tracked as a safe template) +.env +.env.local +.env.*.local +.env.backup + +# python +__pycache__/ +*.py[cod] +.venv/ +venv/ + +# OS / IDE +.DS_Store +.vscode/ +.idea/ + +# logs +*.log +logs/ diff --git a/README.md b/README.md index 58620a0e..dd0cf851 100644 --- a/README.md +++ b/README.md @@ -235,9 +235,30 @@ Create the runtime and register your nodes: Get Exosphere running locally in under 2 minutes: +### 1. Set up environment variables + +```bash +# Copy the example environment file +cp .env.example .env + +# Generate secure secrets (REQUIRED!) +# For STATE_MANAGER_SECRET and EXOSPHERE_API_KEY: +openssl rand -base64 32 + +# For SECRETS_ENCRYPTION_KEY (must be a valid Fernet key): +python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" + +# Edit .env and add your generated secrets +nano .env +``` + +> **🔒 Security Note**: Never commit `.env` file or use default secrets in production! See [SECURITY_SETUP.md](SECURITY_SETUP.md) for detailed security configuration. + +### 2. Start the services + ```bash # Option 1: With cloud MongoDB (recommended) -echo "MONGO_URI=your-mongodb-connection-string" > .env +# Ensure MONGO_URI is set in your .env file curl -O https://raw.githubusercontent.com/exospherehost/exospherehost/main/docker-compose/docker-compose.yml docker compose up -d @@ -248,14 +269,15 @@ docker compose -f docker-compose-with-mongodb.yml up -d **Environment Configuration:** - Docker Compose automatically loads `.env` files from the working directory -- Create your `.env` file in the same directory as your docker-compose file +- All secrets must be configured before starting services +- Use `.env.example` as a template - never commit `.env` to version control Access your services: - **Dashboard**: `http://localhost:3000` - **API**: `http://localhost:8000` -> **📝 Note**: This configuration is for **development and testing only**. For production deployments, environment variable customization, and advanced configuration options, please read the complete **[Docker Compose Setup Guide](https://docs.exosphere.host/docker-compose-setup)**. +> **📝 Note**: This configuration requires proper environment setup. For production deployments, environment variable customization, and advanced configuration options, please read the complete **[Docker Compose Setup Guide](https://docs.exosphere.host/docker-compose-setup)** and **[Security Setup Guide](SECURITY_SETUP.md)**. ## 📚 Documentation & Resources diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ee013a76 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,58 @@ +## Security: Configure via .env (local only). Never commit real secrets. +## Use .env.example for placeholder values and copy to .env for local runs. +services: + exosphere-state-manager: + build: + context: ./state-manager + dockerfile: Dockerfile + image: exosphere-state-manager:local + container_name: exosphere-state-manager + restart: unless-stopped + environment: + - MONGO_URI=${MONGO_URI:?MONGO_URI must be set} + - STATE_MANAGER_SECRET=${STATE_MANAGER_SECRET:?STATE_MANAGER_SECRET must be set} + - MONGO_DATABASE_NAME=${MONGO_DATABASE_NAME:-exosphere} + - SECRETS_ENCRYPTION_KEY=${SECRETS_ENCRYPTION_KEY:?SECRETS_ENCRYPTION_KEY must be set} + - TTL_DAYS=${TTL_DAYS:-30} + - RUN_TTL_DAYS=${RUN_TTL_DAYS:-30} + + ports: + - "8000:8000" + networks: + - exosphere-network + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + exosphere-dashboard: + image: ghcr.io/exospherehost/exosphere-dashboard:${EXOSPHERE_TAG:-latest} + pull_policy: always + container_name: exosphere-dashboard + restart: unless-stopped + environment: + # Server-side secure configuration (NOT exposed to browser) + - EXOSPHERE_STATE_MANAGER_URI=${EXOSPHERE_STATE_MANAGER_URI:-http://exosphere-state-manager:8000} + - EXOSPHERE_API_KEY=${EXOSPHERE_API_KEY:?EXOSPHERE_API_KEY must be set} + # Client-side configuration (exposed to browser) + - NEXT_PUBLIC_DEFAULT_NAMESPACE=${NEXT_PUBLIC_DEFAULT_NAMESPACE:-default} + depends_on: + exosphere-state-manager: + condition: service_healthy + ports: + - "3000:3000" + networks: + - exosphere-network + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/", "||", "exit", "1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + +networks: + exosphere-network: + driver: bridge + attachable: true diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..1d245b69 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,162 @@ +{ + "name": "exospherehost_fork", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "mongodb": "^7.0.0" + } + }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.2.tgz", + "integrity": "sha512-QgA5AySqB27cGTXBFmnpifAi7HxoGUeezwo6p9dI03MuDB6Pp33zgclqVb6oVK3j6I9Vesg0+oojW2XxB59SGg==", + "license": "MIT", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "license": "MIT" + }, + "node_modules/@types/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==", + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, + "node_modules/bson": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.0.0.tgz", + "integrity": "sha512-Kwc6Wh4lQ5OmkqqKhYGKIuELXl+EPYSCObVE6bWsp1T/cGkOCBN0I8wF/T44BiuhHyNi1mmKVPXk60d41xZ7kw==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "license": "MIT" + }, + "node_modules/mongodb": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.0.0.tgz", + "integrity": "sha512-vG/A5cQrvGGvZm2mTnCSz1LUcbOPl83hfB6bxULKQ8oFZauyox/2xbZOoGNl+64m8VBrETkdGCDBdOsCr3F3jg==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^7.0.0", + "mongodb-connection-string-url": "^7.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": ">=7.0.0 <7.1.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.0.tgz", + "integrity": "sha512-irhhjRVLE20hbkRl4zpAYLnDMM+zIZnp0IDB9akAFFUZp/3XdOfwwddc7y6cNvF2WCEtfTYRwYbIfYa2kVY0og==", + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^13.0.0", + "whatwg-url": "^14.1.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "license": "MIT", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..af87bd6d --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "mongodb": "^7.0.0" + } +} diff --git a/state-manager/app/config/settings.py b/state-manager/app/config/settings.py index 75cea36d..3f7e5931 100644 --- a/state-manager/app/config/settings.py +++ b/state-manager/app/config/settings.py @@ -14,6 +14,9 @@ class Settings(BaseModel): secrets_encryption_key: str = Field(..., description="Key for encrypting secrets") trigger_workers: int = Field(default=1, description="Number of workers to run the trigger cron") trigger_retention_hours: int = Field(default=720, description="Number of hours to retain completed/failed triggers before cleanup") + ttl_days: int = Field(default=30, description="TTL in days for TTL indexes") + # Specific TTL for runs collection; overrides ttl_days when used for runs + run_ttl_days: int = Field(30, env="RUN_TTL_DAYS", description="TTL in days for runs collection") @classmethod def from_env(cls) -> "Settings": @@ -23,7 +26,9 @@ def from_env(cls) -> "Settings": state_manager_secret=os.getenv("STATE_MANAGER_SECRET"), # type: ignore secrets_encryption_key=os.getenv("SECRETS_ENCRYPTION_KEY"), # type: ignore trigger_workers=int(os.getenv("TRIGGER_WORKERS", 1)), # type: ignore - trigger_retention_hours=int(os.getenv("TRIGGER_RETENTION_HOURS", 720)) # type: ignore + trigger_retention_hours=int(os.getenv("TRIGGER_RETENTION_HOURS", 720)), # type: ignore + ttl_days=int(os.getenv("TTL_DAYS", 30)), # type: ignore + run_ttl_days=int(os.getenv("RUN_TTL_DAYS", os.getenv("TTL_DAYS", 30))) # type: ignore ) diff --git a/state-manager/app/main.py b/state-manager/app/main.py index 0486a0c4..01a54be3 100644 --- a/state-manager/app/main.py +++ b/state-manager/app/main.py @@ -6,6 +6,8 @@ from fastapi.middleware.cors import CORSMiddleware from contextlib import asynccontextmanager from pymongo import AsyncMongoClient +from pymongo.asynchronous.database import AsyncDatabase +from typing import List, Optional # injecting singletons from .singletons.logs_manager import LogsManager @@ -34,46 +36,203 @@ # importing database health check function from .utils.check_database_health import check_database_health -#scheduler +# scheduler from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger from .tasks.trigger_cron import trigger_cron # init tasks from .tasks.init_tasks import init_tasks - + # Define models list DOCUMENT_MODELS = [State, GraphTemplate, RegisteredNode, Store, Run, DatabaseTriggers] scheduler = AsyncIOScheduler() +# Use LogsManager for consistent structured logging across the app +logger = LogsManager().get_logger() + + +async def ensure_ttl_indexes( + db: AsyncDatabase, + ttl_days: int = 30, + collections: Optional[List[str]] = None, + timestamp_field: str = "created_at" +) -> None: + """ + Ensure TTL indexes exist on the specified collections. + + Creates a TTL (Time To Live) index that automatically deletes documents after + they reach a certain age based on a timestamp field. If an index already exists + with a different expireAfterSeconds value, it will be updated using collMod. + + Args: + db: async pymongo Database instance (from AsyncMongoClient) + ttl_days: number of days after which documents should expire (default: 30) + collections: list of collection names to apply TTL to (defaults to ['runs']) + timestamp_field: name of the timestamp field to create TTL index on (default: 'created_at') + """ + if collections is None: + collections = ["runs"] + + ttl_seconds = int(ttl_days) * 24 * 3600 + index_name = f"ttl_{timestamp_field}_index" + + for coll_name in collections: + try: + coll = db.get_collection(coll_name) + + # Check for existing indexes + existing_indexes = await coll.index_information() + + if index_name in existing_indexes: + # Index exists, check if expireAfterSeconds differs + existing_ttl = existing_indexes[index_name].get("expireAfterSeconds") + + if existing_ttl is None or existing_ttl != ttl_seconds: + # TTL value differs, update using collMod + logger.warning( + "ttl_index_exists_mismatch", + collection=coll_name, + index_name=index_name, + timestamp_field=timestamp_field, + existing_ttl=existing_ttl if existing_ttl is not None else None, + desired_ttl=ttl_seconds, + ) + + try: + # Use collMod to update the expireAfterSeconds value + await db.command({ + "collMod": coll_name, + "index": { + "keyPattern": { timestamp_field: 1 }, + "expireAfterSeconds": ttl_seconds + } + }) + + logger.info( + "ttl_index_updated", + collection=coll_name, + index_name=index_name, + timestamp_field=timestamp_field, + previous_ttl=existing_ttl if existing_ttl is not None else None, + new_ttl=ttl_seconds, + ) + except Exception as mod_error: + logger.error( + "ttl_index_update_failed", + collection=coll_name, + index_name=index_name, + error=str(mod_error), + exc_info=True, + ) + + # If collMod fails, drop and recreate + await coll.drop_index(index_name) + logger.info( + "ttl_index_dropped", + collection=coll_name, + index_name=index_name, + ) + + await coll.create_index( + [(timestamp_field, 1)], + expireAfterSeconds=ttl_seconds, + name=index_name + ) + + logger.info( + "ttl_index_recreated", + collection=coll_name, + index_name=index_name, + timestamp_field=timestamp_field, + ttl_seconds=ttl_seconds, + ttl_days=ttl_days, + ) + else: + # Index exists with correct TTL value + logger.info( + "ttl_index_exists_correct", + collection=coll_name, + index_name=index_name, + timestamp_field=timestamp_field, + ttl_seconds=ttl_seconds, + ttl_days=ttl_days, + ) + else: + # Index doesn't exist, create it + await coll.create_index( + [(timestamp_field, 1)], + expireAfterSeconds=ttl_seconds, + name=index_name + ) + + logger.info( + "ttl_index_created", + collection=coll_name, + index_name=index_name, + timestamp_field=timestamp_field, + ttl_seconds=ttl_seconds, + ttl_days=ttl_days, + ) + + except Exception as e: + # Log error but don't crash the server + logger.error( + "ttl_index_ensure_failed", + collection=coll_name, + index_name=index_name, + timestamp_field=timestamp_field, + error=str(e), + exc_info=True, + ) + + logger.info("ttl_index_setup_completed", collections=collections, ttl_days=ttl_days, timestamp_field=timestamp_field) + + @asynccontextmanager async def lifespan(app: FastAPI): - # begaining of the server - logger = LogsManager().get_logger() - logger.info("server starting") + # beginning of the server + logger.info("server_starting") # Get settings settings = get_settings() - # initializing beanie + # initializing beanie (and Mongo client) client = AsyncMongoClient(settings.mongo_uri) db = client[settings.mongo_database_name] + + # Initialize beanie models (this registers document models / collections) await init_beanie(db, document_models=DOCUMENT_MODELS) - logger.info("beanie dbs initialized") + logger.info("beanie_initialized") + + # --- ENSURE TTL INDEXES (conservative, before init_tasks) --- + # Start with 'runs' only to be safe. Add other collections once you confirm. + # Uses 'created_at' field by default (can be customized via timestamp_field parameter) + logger.info("ttl_index_creation_start") + try: + # Default TTL is 30 days; override via env var RUN_TTL_DAYS + await ensure_ttl_indexes(db, ttl_days=settings.run_ttl_days, collections=["runs"]) + logger.info("ttl_index_creation_completed", collections=["runs"], ttl_days=settings.run_ttl_days) + except Exception as e: + # By default we log and continue. If you want fail-fast, replace with `raise`. + logger.error("ttl_index_creation_exception", error=str(e), exc_info=True) # performing init tasks + logger.info("init_tasks_start") await init_tasks() - logger.info("init tasks completed") + logger.info("init_tasks_completed") # initialize secret if not settings.state_manager_secret: + # this is critical — fail immediately raise ValueError("STATE_MANAGER_SECRET is not set") - logger.info("secret initialized") + logger.info("secret_initialized") # perform database health check await check_database_health(DOCUMENT_MODELS) + # schedule the cron job scheduler.add_job( trigger_cron, CronTrigger.from_crontab("* * * * *"), @@ -81,7 +240,7 @@ async def lifespan(app: FastAPI): misfire_grace_time=60, coalesce=True, max_instances=1, - id="every_minute_task" + id="every_minute_task", ) scheduler.start() @@ -91,7 +250,7 @@ async def lifespan(app: FastAPI): # end of the server await client.close() scheduler.shutdown() - logger.info("server stopped") + logger.info("server_stopped") app = FastAPI( @@ -109,18 +268,19 @@ async def lifespan(app: FastAPI): }, ) -# Add middlewares in inner-to-outer order (last added runs first on request): -# 1) UnhandledExceptions (inner) -app.add_middleware(UnhandledExceptionsMiddleware) -# 2) Request ID (middle) -app.add_middleware(RequestIdMiddleware) -# 3) CORS (outermost) -app.add_middleware(CORSMiddleware, **get_cors_config()) +# Add middlewares in inner-to-outer order (first added is innermost; last added runs first on request): +# 1) CORS (innermost, closest to route handler) +app.add_middleware(CORSMiddleware, **get_cors_config()) +# 2) Request ID (middle) +app.add_middleware(RequestIdMiddleware) +# 3) UnhandledExceptions (outermost, global catch-all for exceptions from any middleware or route) +app.add_middleware(UnhandledExceptionsMiddleware) @app.get("/health") def health() -> dict: return {"message": "OK"} + app.include_router(global_router) -app.include_router(router) \ No newline at end of file +app.include_router(router) diff --git a/state-manager/pyproject.toml b/state-manager/pyproject.toml index e80bdaea..1ae1504b 100644 --- a/state-manager/pyproject.toml +++ b/state-manager/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "json-schema-to-pydantic>=0.4.1", "pytest-cov>=6.2.1", "python-dotenv>=1.1.1", + "pydantic-settings>=2.0.0", "structlog>=25.4.0", "uvicorn>=0.35.0", ]