diff --git a/Dockerfile b/Dockerfile index 98b6fc2..3aeaecf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,37 @@ -FROM python:3.12-slim-bookworm AS builder +FROM python:3.12-slim-bookworm AS base -# Copy uv from external repository -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /uvx /bin/ +# Python optimizations +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 -# Set the working directory for the build stage WORKDIR /app -# Copy only necessary files for installing dependencies -COPY ./pyproject.toml . -COPY ./uv.lock . -COPY ./README.md . +FROM base AS builder +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /bin/ -# RUN uv cache dir -# RUN uv sync +COPY ./pyproject.toml ./uv.lock ./ +COPY ./bases ./bases +COPY ./components ./components +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + +FROM base AS dev +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv uvx/ /bin/ + +WORKDIR /app + +COPY --from=builder /app/.venv /app/.venv + +ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONPATH="/app" + +COPY ./projects/api_public/pyproject.toml ./projects/api_public/uv.lock ./ +COPY ./bases ./bases +COPY ./components ./components + +# this isntalls dev dependencies +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen CMD [ "sleep", "infinity" ] \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-debug.yml similarity index 98% rename from docker-compose-dev.yml rename to docker-compose-debug.yml index 82bdf87..8d44d9f 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-debug.yml @@ -120,17 +120,18 @@ services: build: context: ./ dockerfile: Dockerfile + target: dev # command: ["uv", "run", "bases/bot_detector/hiscore_scraper/core.py"] # command: uv run uvicorn bases.bot_detector.api_public.src.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.website.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --reload --port 5000 ports: - - 5000:5000 # api_publics + - 5000 # api endpoint + - 8000 # metrics endpoint volumes: - ./bases:/app/bases - ./components:/app/components - ./projects:/app/projects - - uv_cache:/root/.cache/uv command: ["sleep", "infinity"] # network_mode: host # needed if we do any portforwarding from the cluster networks: diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml new file mode 100644 index 0000000..92c754b --- /dev/null +++ b/docker-compose-prod.yml @@ -0,0 +1,395 @@ +services: + # KAFKA + kafka: + image: apache/kafka:3.7.2 + container_name: kafka + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_NUM_PARTITIONS: 3 + ports: + - 9092:9092 + healthcheck: + test: + [ + "CMD", + "./opt/kafka/bin/kafka-topics.sh", + "--list", + "--bootstrap-server", + "localhost:9092" + ] + interval: 30s + timeout: 10s + retries: 5 + networks: + - botdetector-network + kafka_setup: + container_name: kafka_setup + image: bd/kafka_setup # tags the image if build + build: + context: ./_kafka + # command: ["sleep", "infinity"] + environment: + - KAFKA_BROKER=kafka:9092 + networks: + - botdetector-network + depends_on: + kafka: + condition: service_healthy + kafdrop: + container_name: kafdrop + image: obsidiandynamics/kafdrop:latest + environment: + - KAFKA_BROKERCONNECT=kafka:9092 + - JVM_OPTS=-Xms32M -Xmx64M + - SERVER_SERVLET_CONTEXTPATH=/ + ports: + - 9042:9000 # too many things use 9000 like portainer.. lets use 9042 instead? + restart: on-failure + networks: + - botdetector-network + depends_on: + kafka: + condition: service_healthy + # MYSQL + mysql: + container_name: mysql + image: bd/mysql # tags the image if build + build: + context: ./_mysql + environment: + - MYSQL_ROOT_PASSWORD=root_bot_buster + # - MYSQL_DATABASE=playerdata + ports: + - 3307:3306 + networks: + - botdetector-network + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost -u root -proot_bot_buster"] + interval: 10s + retries: 3 + start_period: 30s + timeout: 5s + mysql_setup: + container_name: mysql_setup + image: bd/mysql_setup # tags the image if build + build: + context: ./_mysql_data + # command: ["sleep", "infinity"] + environment: + - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata + - DEBUG=False + networks: + - botdetector-network + depends_on: + mysql: + condition: service_healthy + # MINIO + minio: + image: minio/minio + expose: + - "9000" + - "9001" + ports: + - 9001 + environment: + MINIO_ROOT_USER: "minio_user" + MINIO_ROOT_PASSWORD: "minio_password" + volumes: + - ./_minio_data:/data + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 + interval: 1s + timeout: 10s + retries: 5 + command: server /data --console-address ":9001" + networks: + - botdetector-network + # Create a bucket named "bucket" if it doesn't exist + minio-create-bucket: + image: minio/mc + depends_on: + minio: + condition: service_healthy + entrypoint: > + bash -c " + mc alias set minio http://minio:9000 minio_user minio_password && + if ! mc ls minio/bucket; then + mc mb minio/models + else + echo 'bucket already exists' + fi + " + # COMPONENTS + hiscore_scraper: + container_name: hiscore_scraper + image: bd/hiscore_scraper # tags the image if build + build: + context: ./ + dockerfile: ./projects/hiscore_scraper/Dockerfile + target: production + # command: ["uv", "run", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] + networks: + - botdetector-network + env_file: + - .env + volumes: + - uv_cache:/root/.cache/uv + depends_on: + kafka_setup: + condition: service_completed_successfully + runemetrics_scraper: + container_name: runemetrics_scraper + image: bd/runemetrics_scraper # tags the image if build + build: + context: ./ + dockerfile: ./projects/runemetrics_scraper/Dockerfile + target: production + # command: ["sleep", "infinity"] + # command: ["uv", "run", "bot_detector/runemetrics_scraper/src/core.py"] + networks: + - botdetector-network + env_file: + - .env + volumes: + - uv_cache:/root/.cache/uv + depends_on: + kafka_setup: + condition: service_completed_successfully + worker_hiscore: + container_name: worker_hiscore + image: bd/worker_hiscore # tags the image if build + build: + context: ./ + dockerfile: ./projects/worker_hiscore/Dockerfile + target: production + # command: ["sleep", "infinity"] + # command: ["uv", "run", "bot_detector/worker_hiscore/src/core.py"] + networks: + - botdetector-network + env_file: + - .env + volumes: + - uv_cache:/root/.cache/uv + depends_on: + kafka: + condition: service_healthy + kafka_setup: + condition: service_completed_successfully + mysql: + condition: service_healthy + mysql_setup: + condition: service_completed_successfully + worker_report: + container_name: worker_report + image: bd/worker_report # tags the image if build + build: + context: ./ + dockerfile: ./projects/worker_report/Dockerfile + target: production + # command: ["sleep", "infinity"] + networks: + - botdetector-network + env_file: + - .env + volumes: + - uv_cache:/root/.cache/uv + depends_on: + kafka_setup: + condition: service_completed_successfully + mysql_setup: + condition: service_completed_successfully + worker_ml: + container_name: worker_ml + image: bd/worker_ml # tags the image if build + build: + context: ./ + dockerfile: ./projects/worker_ml/Dockerfile + target: production + # command: ["sleep", "infinity"] + environment: + - DATABASE_URL=mysql+asyncmy://ml-worker:ml_worker_pw@mysql:3306/playerdata + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + - BASE_URL=http://api_ml:5000 + - MAX_MESSAGES=10 + - MAX_INTERVAL_MS=5000 + - MODEL_NAME=multi_model_v1 + networks: + - botdetector-network + volumes: + - uv_cache:/root/.cache/uv + depends_on: + kafka_setup: + condition: service_completed_successfully + mysql_setup: + condition: service_completed_successfully + api_ml: + condition: service_healthy + + scrape_task_producer: + container_name: scrape_task_producer + image: bd/scrape_task_producer # tags the image if build + build: + context: ./ + dockerfile: ./projects/scrape_task_producer/Dockerfile + target: production + # command: ["sleep", "infinity"] + networks: + - botdetector-network + env_file: + - .env + volumes: + - uv_cache:/root/.cache/uv + depends_on: + kafka_setup: + condition: service_completed_successfully + mysql_setup: + condition: service_completed_successfully + job_prune_hs_data: + container_name: job_prune_hs_data + image: bd/job_prune_hs_data # tags the image if build + build: + context: ./ + dockerfile: ./projects/job_prune_hs_data/Dockerfile + target: production + # command: ["sleep", "infinity"] + networks: + - botdetector-network + environment: + - LIMIT=100 + - DATABASE_URL=mysql+asyncmy://job-prune-hs:job_prune_hs_pw@mysql:3306/playerdata + env_file: + - .env + volumes: + - uv_cache:/root/.cache/uv + depends_on: + mysql_setup: + condition: service_completed_successfully + api_public: + container_name: api_public + image: bd/api_public:prod + build: + context: . + dockerfile: ./projects/api_public/Dockerfile + target: prod + # command: ["sleep", "infinity"] + env_file: + - .env + environment: + - KAFKA_HOST=kafka:9092 + - DATABASE_URL=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 + volumes: + - ./bases:/app/bases + - ./components:/app/components + ports: + - 5000 + networks: + - botdetector-network + depends_on: + kafka_setup: + condition: service_completed_successfully + mysql_setup: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + wait_for_api_public: + image: alpine:latest + container_name: wait_for_api_public + command: ["sh", "-c", "echo 'api_public healthy'"] + depends_on: + api_public: + condition: service_healthy + networks: + - botdetector-network + website: + container_name: website + image: bd/website + build: + context: . + dockerfile: ./projects/website/Dockerfile + env_file: + - .env + environment: + BD_TOKEN: "" + ENV: "DEV" + RELEASE_VERSION: "0.1" + PATREON_CLIENT_ID: "" + PATREON_CLIENT_SECRET: "" + volumes: + - uv_cache:/root/.cache/uv + ports: + - 5000 + networks: + - botdetector-network + healthcheck: + test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/monitoring\").getcode()==200 else 1)'"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + api_ml: + container_name: api_ml + image: bd/api_ml + build: + context: . + dockerfile: ./projects/api_ml/Dockerfile + target: builder + # command: ["sleep", "infinity"] + command: > + sh -c "cd ../.. && projects/api_ml/.venv/bin/uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir bases/bot_detector/api_ml" + env_file: + - .env + environment: + - MLFLOW_S3_ENDPOINT_URL=http://minio:9000 + - AWS_ACCESS_KEY_ID=minio_user + - AWS_SECRET_ACCESS_KEY=minio_password + - UV_HTTP_TIMEOUT=120 + volumes: + - ./bases:/app/bases + - ./components:/app/components + - uv_cache:/root/.cache/uv + ports: + - 5001:5000 + networks: + - botdetector-network + depends_on: + minio: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "python -c 'import urllib.request,sys; sys.exit(0 if urllib.request.urlopen(\"http://localhost:5000/\").getcode()==200 else 1)'"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + +networks: + botdetector-network: + name: bd-network + +# To create the docker volume for uv cache (run only once) +# docker volume create uv_cache + +# To copy existing cache from host to docker volume (run only once) +# docker run --rm \ +# -v uv_cache:/data \ +# -v ~/.cache/uv:/source \ +# alpine sh -c "cp -r /source/* /data/" +volumes: + uv_cache: + external: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ea124cd..4ffd3f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -276,14 +276,12 @@ services: condition: service_completed_successfully api_public: container_name: api_public - image: bd/api_public + image: bd/api_public:dev build: context: . dockerfile: ./projects/api_public/Dockerfile - target: builder + target: dev # command: ["sleep", "infinity"] - command: > - sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.bot_detector.api_public.src.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" env_file: - .env environment: @@ -293,9 +291,8 @@ services: - POOL_RECYCLE=60 - POOL_TIMEOUT=30 volumes: - - ./bases:/bases - - ./components:/components - - uv_cache:/root/.cache/uv + - ./bases:/app/bases + - ./components:/app/components ports: - 5000 networks: @@ -352,10 +349,8 @@ services: build: context: . dockerfile: ./projects/api_ml/Dockerfile - target: builder + target: dev # command: ["sleep", "infinity"] - command: > - sh -c "cd ../.. && projects/api_ml/.venv/bin/uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir bases/bot_detector/api_ml" env_file: - .env environment: diff --git a/projects/api_ml/Dockerfile b/projects/api_ml/Dockerfile index d8c340c..bf9e21e 100644 --- a/projects/api_ml/Dockerfile +++ b/projects/api_ml/Dockerfile @@ -1,34 +1,57 @@ -FROM python:3.12-slim-bookworm AS builder +FROM python:3.12-slim-bookworm AS base -# Copy uv from external repository -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /uvx /bin/ +# Python optimizations +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 -# set the working directory WORKDIR /app -# Copy only necessary files to run the projects +FROM base AS builder +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /bin/ + +WORKDIR /app + +COPY ./projects/api_ml ./projects/api_ml COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects -WORKDIR /app/projects/api_ml +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev --project ./projects/api_ml + +FROM base AS dev +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv uvx/ /bin/ + +WORKDIR /app + +COPY --from=builder /app/projects/api_ml/.venv /app/.venv + +ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONPATH="/app" -# install dependencies via RUN uv build -ENV UV_HTTP_TIMEOUT=120 -# ENV UV_PYTHON_CACHE_DIR=/root/.cache/uv/python -# RUN --mount=type=cache,target=/root/.cache/uv \ -# uv sync --frozen --no-editable -RUN uv sync --frozen --no-editable +COPY ./projects/api_ml ./projects/api_ml +COPY ./bases ./bases +COPY ./components ./components + +# this installs dev dependencies +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --project ./projects/api_ml \ + && cp -r projects/api_ml/.venv/ .venv/ + +CMD ["uvicorn", "bot_detector.api_ml.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--reload"] + +FROM base AS prod + +WORKDIR /app -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Create a secure user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser --home /app --no-create-home --disabled-password --uid 5678 appuser -# Creates a non-root user with an explicit UID and adds permission to access the /project folder -RUN adduser -u 5678 --disabled-password --gecos "" appuser +COPY --from=builder --chown=appuser /app/projects/api_ml/.venv /app/.venv -WORKDIR /app/projects/api_ml -COPY --from=builder --chown=appuser /app/projects/api_ml/.venv /app/projects/api_ml/.venv +ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONPATH="/app" USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_ml.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD ["python3", "-m", "uvicorn", "bot_detector.api_ml.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file diff --git a/projects/api_public/Dockerfile b/projects/api_public/Dockerfile index c7b3656..ece9e6b 100644 --- a/projects/api_public/Dockerfile +++ b/projects/api_public/Dockerfile @@ -1,30 +1,57 @@ -FROM python:3.12-slim-bookworm AS builder +FROM python:3.12-slim-bookworm AS base -# Copy uv from external repository -COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /uvx /bin/ +# Python optimizations +ENV PYTHONUNBUFFERED=1 +ENV UV_COMPILE_BYTECODE=1 -# set the working directory WORKDIR /app -# Copy only necessary files to run the projects +FROM base AS builder +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv /bin/ + +WORKDIR /app + +COPY ./projects/api_public ./projects/api_public COPY ./bases ./bases COPY ./components ./components -COPY ./projects ./projects -WORKDIR /app/projects/api_public +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev --project ./projects/api_public + +FROM base AS dev +COPY --from=ghcr.io/astral-sh/uv:0.5.4 /uv uvx/ /bin/ + +WORKDIR /app + +COPY --from=builder /app/projects/api_public/.venv /app/.venv + +ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONPATH="/app" -# install dependencies via RUN uv build -RUN uv sync --frozen --no-editable +COPY ./projects/api_public ./projects/api_public +COPY ./bases ./bases +COPY ./components ./components + +# this installs dev dependencies +RUN --mount=type=cache,id=uv_cache,target=/root/.cache/uv \ + uv sync --frozen --project ./projects/api_public \ + && cp -r projects/api_public/.venv/ .venv/ + +CMD ["uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--reload"] + +FROM base AS prod + +WORKDIR /app -# Production stage: Prepare the final production environment -FROM python:3.12-slim-bookworm AS production +# Create a secure user +RUN addgroup --system appuser && \ + adduser --system --ingroup appuser --home /app --no-create-home --disabled-password --uid 5678 appuser -# Creates a non-root user with an explicit UID and adds permission to access the /project folder -RUN adduser -u 5678 --disabled-password --gecos "" appuser +COPY --from=builder --chown=appuser /app/projects/api_public/.venv /app/.venv -WORKDIR /app/projects/api_public -COPY --from=builder --chown=appuser /app/projects/api_public/.venv /app/projects/api_public/.venv +ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONPATH="/app" USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_public.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD ["python3", "-m", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file diff --git a/projects/api_public/pyproject.toml b/projects/api_public/pyproject.toml index ed7ff76..e76e18e 100644 --- a/projects/api_public/pyproject.toml +++ b/projects/api_public/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ ] [project.scripts] -scrape_task_producer = "bot_detector.api_public.core.server:run" +api_public = "bot_detector.api_public.core.server:run" [tool.hatch.build.hooks.polylith-bricks] packages = ["bot_detector"] @@ -30,4 +30,4 @@ packages = ["bot_detector"] "../../components/bot_detector/database" = "bot_detector/database" "../../components/bot_detector/kafka" = "bot_detector/kafka" "../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file