From 8d2fef26a6e5633f7d3fb5fca5b143e75cced145 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 12:53:32 -0700 Subject: [PATCH 01/17] Get MongoDB connection parameters from `.env` file --- .env.example | 7 +++++++ .gitignore | 7 +++++-- CONTRIBUTING.md | 34 ++++++++++++++++++++++------------ docker-compose.yml | 7 ++----- mongodb/README.md | 4 ++-- pyproject.toml | 1 + src/config.py | 34 ++++++++++++++++++++++++++++++++++ src/server.py | 20 ++++++++++++-------- src/tests/conftest.py | 35 +++++++++++++++++++++++++++++++++++ uv.lock | 16 ++++++++++++++++ 10 files changed, 136 insertions(+), 29 deletions(-) create mode 100644 .env.example create mode 100644 src/config.py create mode 100644 src/tests/conftest.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..92fc545 --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +# See `src/config.py` for documentation about these environment variables. + +MONGO_HOST=mongo +MONGO_PORT=27017 +MONGO_DATABASE=bertron +MONGO_USERNAME=admin +MONGO_PASSWORD=root diff --git a/.gitignore b/.gitignore index f0b4ac2..fefca85 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ # Ignore unrelated files created by Finder on macOS. .DS_Store -# Jupter Notebook checkpoints. +# Jupyter Notebook checkpoints. **/.ipynb_checkpoints/ # TODO: Document these. @@ -16,4 +16,7 @@ **/*.egg-info/ # Ignore Python bytecode files. -__pycache__ \ No newline at end of file +__pycache__ + +# Top-level environment configuration file. +/.env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c9542e..8dcafec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,49 +10,59 @@ Check https://docs.astral.sh/uv/#installation for alternative installation metho ## Preparing dev environment +Create your `.env` file (if you haven't already) and edit its contents to reflect +your environment. + +```sh +cp .env.example .env + +# (Optional) Edit its contents. +# vi .env ``` +> Note: Git will ignore your `.env` file. + +Create and activate a Python virtual environment. + +```sh uv venv source .venv/bin/activate ``` -You can also run commands without activating the dev environment with `uv run`, -for example you can run `ruff check` with -``` +You can also run commands without first activating the Python virtual environment, by prefixing them with `uv run`. For example, you can run `ruff check` with: +```sh uv run ruff check ``` -but the dev environment is useful if you're using an IDE. ## Adding dependencies You can add dependencies with `uv add`. The following command: -``` +```sh uv add polars duckdb ``` adds `polars` and `duckdb` as project dependencies. For dev dependencies (only needed for development, but not for using the package later), use `--dev`: -``` +```sh uv add --dev ruff ipykernel pytest pytest-cov mypy ``` ## Updating dependencies You can run -``` +```sh uv sync --upgrade ``` to install the latest dependency version that matches the version range in `pyproject.toml`. This will also update `uv.lock` to make the installation reproducible. -## Syncing dev environment +## Syncing the Python virtual environment -After adding or updating dependencies, -run -``` +After adding or updating dependencies, run +```sh uv sync --all-extras --dev ``` -to make sure the dev environment has the updated dependencies. +to make sure the Python virtual environment has the updated dependencies. --- diff --git a/docker-compose.yml b/docker-compose.yml index f93d5fe..44a6008 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,8 +30,8 @@ services: - "${MONGO_PORT:-27017}:27017" restart: unless-stopped environment: - MONGO_INITDB_ROOT_USERNAME: admin - MONGO_INITDB_ROOT_PASSWORD: root + MONGO_INITDB_ROOT_USERNAME: "${MONGO_USERNAME:-admin}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_PASSWORD:-root}" volumes: - mongo_data:/data/db @@ -40,9 +40,6 @@ services: build: { context: ".", dockerfile: Dockerfile, target: development } # This service should not start automatically - only run on demand profiles: ["tools"] - environment: - # Set the MongoDB connection string to connect to the mongo service - MONGO_URI: "mongodb://admin:root@mongo:27017" volumes: # Mount the root directory to access the ingest script and data files - ".:/app" diff --git a/mongodb/README.md b/mongodb/README.md index 78ae358..3994c8d 100644 --- a/mongodb/README.md +++ b/mongodb/README.md @@ -22,8 +22,8 @@ python ingest_data.py --input your_data_file.json ### Command-line arguments -- `--mongo-uri`: MongoDB connection URI (default: mongodb://localhost:27017) -- `--db-name`: MongoDB database name (default: bertron) +- `--mongo-uri`: MongoDB connection URI (default: `mongodb://localhost:27017`) +- `--db-name`: MongoDB database name (default: `bertron`) - `--schema-path`: Path or URL to the schema JSON file (default: `bertron_schema.json` in the current directory) - `--input`: Path to input JSON file or directory containing JSON files (required) diff --git a/pyproject.toml b/pyproject.toml index 96700c4..98b503c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "fastapi[standard]>=0.115.12", "jsonschema>=4.0.0", "nmdc-api-utilities>=0.3.9", + "pydantic-settings>=2.10.1", "pymongo>=4.13.1", "pytest>=8.4.0", "uvicorn>=0.34.3", diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..72aafc7 --- /dev/null +++ b/src/config.py @@ -0,0 +1,34 @@ +from typing import Optional + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + r""" + Configuration settings for the application. + + Order of precedence (highest to lowest) for settings values: + 1. Process-level environment variable definitions. + 2. Environment variable definitions loaded from a `.env` file. + 3. Default values defined in this class. + + References: + - https://docs.pydantic.dev/latest/concepts/pydantic_settings/#dotenv-env-support + """ + + # Load environment variable definitions from a `.env` file, if one exists, + # in the current working directory. We allow the `.env` file to contain + # extra environment variables (beyond those modeled below), since this + # repository currently contains a variety of independent applications. + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + # MongoDB connection settings. + mongo_host: str = "localhost" + mongo_port: int = 27017 # note: value in `.env` will be coerced into int + mongo_database: str = "bertron" + mongo_username: Optional[str] = None + mongo_password: Optional[str] = None + + +# Instantiate a settings object that can be imported into other modules. +settings = Settings() diff --git a/src/server.py b/src/server.py index 201d5e6..acc84b5 100644 --- a/src/server.py +++ b/src/server.py @@ -10,13 +10,17 @@ from lib.helpers import get_package_version from models import HealthResponse, VersionResponse +from config import settings as cfg # Set up logging logger = logging.getLogger(__name__) -# Connect to MongoDB. -# TODO: Get these values from environment variables instead of hard-coding them. -mongo_client = MongoClient("mongo:27017", username="admin", password="root") +# Connect to the MongoDB server. +mongo_client = MongoClient( + f"{cfg.mongo_host}:{cfg.mongo_port}", + username=cfg.mongo_username, + password=cfg.mongo_password, +) app = FastAPI( title="BERtron API", @@ -58,7 +62,7 @@ def get_version() -> VersionResponse: @app.get("/bertron") def get_all_entities(): r"""Get all documents from the entities collection.""" - db = mongo_client["bertron"] + db = mongo_client[cfg.mongo_database] # Check if the collection exists if "entities" not in db.list_collection_names(): @@ -104,7 +108,7 @@ def find_entities(query: MongoDBQuery): "sort": {"field1": 1, "field2": -1} } """ - db = mongo_client["bertron"] + db = mongo_client[cfg.mongo_database] # Check if the collection exists if "entities" not in db.list_collection_names(): @@ -153,7 +157,7 @@ def find_nearby_entities( Example: /bertron/geo/nearby?latitude=47.6062&longitude=-122.3321&radius_meters=10000 """ - db = mongo_client["bertron"] + db = mongo_client[cfg.mongo_database] # Check if the collection exists if "entities" not in db.list_collection_names(): @@ -215,7 +219,7 @@ def find_entities_in_bounding_box( Example: /bertron/geo/bbox?southwest_lat=47.5&southwest_lng=-122.4&northeast_lat=47.7&northeast_lng=-122.2 """ - db = mongo_client["bertron"] + db = mongo_client[cfg.mongo_database] # Check if the collection exists if "entities" not in db.list_collection_names(): @@ -274,7 +278,7 @@ def get_entity_by_id(id: str): Example: /bertron/emsl:12345 """ - db = mongo_client["bertron"] + db = mongo_client[cfg.mongo_database] # Check if the collection exists if "entities" not in db.list_collection_names(): diff --git a/src/tests/conftest.py b/src/tests/conftest.py new file mode 100644 index 0000000..a6e5e89 --- /dev/null +++ b/src/tests/conftest.py @@ -0,0 +1,35 @@ +r""" +This module contains `pytest` fixture definitions that `pytest` will automatically make available +to all tests within this directory and its descendant directories. + +From the `pytest` documentation: +> The `conftest.py` file serves as a means of providing fixtures for an entire directory. +> Fixtures defined in a `conftest.py` can be used by any test in that package without +> needing to import them (`pytest` will automatically discover them). +Source: https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files +""" + +import pytest + +from config import settings as cfg + + +# Note: We use `autouse=True` so that this fixture is automatically applied to each test +# within its scope (since we are in a `conftest.py` file, its scope consists of +# the current directory and all descendant directories). +@pytest.fixture(autouse=True) +def patched_cfg(): + r""" + A `pytest` fixture that temporarily patches the application configuration + so it references a test database. + """ + + test_database_name = "bertron_test" + main_database_name = cfg.mongo_database + assert main_database_name != test_database_name, ( + "The main database name matches the test database name. " + "Reconfigure your environment to ensure they differ." + ) + cfg.mongo_database = test_database_name + yield cfg + cfg.mongo_database = main_database_name diff --git a/uv.lock b/uv.lock index 4a408ea..03572a5 100644 --- a/uv.lock +++ b/uv.lock @@ -112,6 +112,7 @@ dependencies = [ { name = "fastapi", extra = ["standard"] }, { name = "jsonschema" }, { name = "nmdc-api-utilities" }, + { name = "pydantic-settings" }, { name = "pymongo" }, { name = "pytest" }, { name = "uvicorn" }, @@ -132,6 +133,7 @@ requires-dist = [ { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" }, { name = "jsonschema", specifier = ">=4.0.0" }, { name = "nmdc-api-utilities", specifier = ">=0.3.9" }, + { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "pymongo", specifier = ">=4.13.1" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "uvicorn", specifier = ">=0.34.3" }, @@ -1760,6 +1762,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757, upload-time = "2025-04-23T18:33:30.645Z" }, ] +[[package]] +name = "pydantic-settings" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/85/1ea668bbab3c50071ca613c6ab30047fb36ab0da1b92fa8f17bbc38fd36c/pydantic_settings-2.10.1.tar.gz", hash = "sha256:06f0062169818d0f5524420a360d632d5857b83cffd4d42fe29597807a1614ee", size = 172583, upload-time = "2025-06-24T13:26:46.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/f0/427018098906416f580e3cf1366d3b1abfb408a0652e9f31600c24a1903c/pydantic_settings-2.10.1-py3-none-any.whl", hash = "sha256:a60952460b99cf661dc25c29c0ef171721f98bfcb52ef8d9ea4c943d7c8cc796", size = 45235, upload-time = "2025-06-24T13:26:45.485Z" }, +] + [[package]] name = "pygments" version = "2.19.2" From cdeede51382532218d43841bb802016d539872a8 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 12:53:49 -0700 Subject: [PATCH 02/17] Add test that involves accessing MongoDB database --- src/tests/test_server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/tests/test_server.py b/src/tests/test_server.py index 1181047..b51128a 100644 --- a/src/tests/test_server.py +++ b/src/tests/test_server.py @@ -9,7 +9,7 @@ from fastapi.testclient import TestClient from starlette import status -from models import VersionResponse +from models import HealthResponse, VersionResponse from server import app @@ -28,7 +28,17 @@ def test_root_endpoint_redirects_to_api_docs(test_client: TestClient): def test_version_endpoint_returns_version_response(test_client: TestClient): response = test_client.get("/version") assert response.status_code == status.HTTP_200_OK + body = response.json() # Note: This will raise a `ValidationError` if the response is not # a valid `VersionResponse` (e.g. if it has extra fields or # its fields' values are of an incompatible data type). - _ = VersionResponse(**response.json()) + _ = VersionResponse(**body) + + +def test_health_endpoint_returns_health_response(test_client: TestClient): + response = test_client.get("/health") + assert response.status_code == status.HTTP_200_OK + body = response.json() + assert body["web_server"] is True + assert body["database"] is True + _ = HealthResponse(**body) From ae455cd9e611042f86401a72f6b7bf5a2a0a995d Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 12:59:38 -0700 Subject: [PATCH 03/17] Use `ruff` to format Python code --- src/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index a6e5e89..25d5e9a 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -23,7 +23,7 @@ def patched_cfg(): A `pytest` fixture that temporarily patches the application configuration so it references a test database. """ - + test_database_name = "bertron_test" main_database_name = cfg.mongo_database assert main_database_name != test_database_name, ( From 929fa82f0e21f96987b4ab5c4b35eb08eb1aa514 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 13:13:33 -0700 Subject: [PATCH 04/17] Configure GHA step to fail if container exits with non-zero code --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68281a8..882b589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,11 @@ jobs: - name: Spin up Docker Compose stack in background run: docker compose up --detach - # Note: This spins up the "test" container. + # Note: The `--exit-code-from test` option applies the exit code of the `test` container + # to the `docker compose` process, so that the GHA step fails if tests fail. + # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container - run: docker compose up test + run: docker compose up --exit-code-from test test # Note: This spins everything down. - name: Spin down Docker Compose stack From ac7a8ed5ef404210cad191119d2a51041d85ae23 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 13:24:38 -0700 Subject: [PATCH 05/17] Override `MONGO_HOST` environment variable for GHA tests --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 882b589..09a5892 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,11 +39,15 @@ jobs: - name: Spin up Docker Compose stack in background run: docker compose up --detach + # Note: The `--env MONGO_HOST=mongo` sets the `MONGO_HOST` environment variable + # within the `test` container, so that the `test` container accesses the + # MongoDB server whose hostname is `mongo` (instead of `localhost`). + # # Note: The `--exit-code-from test` option applies the exit code of the `test` container # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container - run: docker compose up --exit-code-from test test + run: docker compose up --env MONGO_HOST=mongo --exit-code-from test test # Note: This spins everything down. - name: Spin down Docker Compose stack From 09374c029792c8cb229cb849c9b8dff6bf11973b Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 13:28:27 -0700 Subject: [PATCH 06/17] Use standard shell syntax for setting environment variable --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09a5892..bbe37e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: - name: Spin up Docker Compose stack in background run: docker compose up --detach - # Note: The `--env MONGO_HOST=mongo` sets the `MONGO_HOST` environment variable + # Note: The `MONGO_HOST=mongo` sets the `MONGO_HOST` environment variable # within the `test` container, so that the `test` container accesses the # MongoDB server whose hostname is `mongo` (instead of `localhost`). # @@ -47,7 +47,7 @@ jobs: # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container - run: docker compose up --env MONGO_HOST=mongo --exit-code-from test test + run: MONGO_HOST=mongo docker compose up --exit-code-from test test # Note: This spins everything down. - name: Spin down Docker Compose stack From 0a5a48a82606d26e51b009d0033c606375aa59fd Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 13:33:50 -0700 Subject: [PATCH 07/17] Use same `MONGO_HOST` value for entire Docker Compose stack --- .github/workflows/ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbe37e4..9cf79c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,19 +35,21 @@ jobs: - name: Run tests locally run: uv run pytest tests + # Note: Before spinning up the Docker Compose stack, we customize `MONGO_HOST` + # so that containers access the MongoDB server at the host named `mongo` + # instead of at the host named `localhost`. + - name: Customize environment variable(s) for Docker Compose stack + run: export MONGO_HOST=mongo + # Note: This spins up containers running the default services. - name: Spin up Docker Compose stack in background run: docker compose up --detach - # Note: The `MONGO_HOST=mongo` sets the `MONGO_HOST` environment variable - # within the `test` container, so that the `test` container accesses the - # MongoDB server whose hostname is `mongo` (instead of `localhost`). - # # Note: The `--exit-code-from test` option applies the exit code of the `test` container # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container - run: MONGO_HOST=mongo docker compose up --exit-code-from test test + run: docker compose up --exit-code-from test test # Note: This spins everything down. - name: Spin down Docker Compose stack From f608fc755ef3ee0a426e067c3421223615fdf4a8 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 13:56:09 -0700 Subject: [PATCH 08/17] Customize `MONGO_HOST` in a GHA-compatible way --- .github/workflows/ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cf79c3..c851cd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,20 +35,22 @@ jobs: - name: Run tests locally run: uv run pytest tests - # Note: Before spinning up the Docker Compose stack, we customize `MONGO_HOST` - # so that containers access the MongoDB server at the host named `mongo` - # instead of at the host named `localhost`. - - name: Customize environment variable(s) for Docker Compose stack - run: export MONGO_HOST=mongo - # Note: This spins up containers running the default services. + # This does not include the `ingest` or `test` services, + # which aren't part of the default profile. - name: Spin up Docker Compose stack in background + env: + # Customize `MONGO_HOST` for all containers in the stack. + MONGO_HOST: mongo run: docker compose up --detach # Note: The `--exit-code-from test` option applies the exit code of the `test` container # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container + env: + # Customize `MONGO_HOST` for the `test` container. + MONGO_HOST: mongo run: docker compose up --exit-code-from test test # Note: This spins everything down. From 817dbef507845a0f75e93ac6950908f0f6190ffb Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:06:02 -0700 Subject: [PATCH 09/17] Customize `MONGO_USERNAME` and `MONGO_PASSWORD` for GHA container stack --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c851cd3..a5599a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,9 @@ jobs: # which aren't part of the default profile. - name: Spin up Docker Compose stack in background env: - # Customize `MONGO_HOST` for all containers in the stack. + # Customize MongoDB connection parameters for all containers in the stack. + MONGO_USERNAME: admin + MONGO_PASSWORD: root MONGO_HOST: mongo run: docker compose up --detach @@ -49,7 +51,9 @@ jobs: # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container env: - # Customize `MONGO_HOST` for the `test` container. + # Customize MongoDB connection parameters for the `test` container. + MONGO_USERNAME: admin + MONGO_PASSWORD: root MONGO_HOST: mongo run: docker compose up --exit-code-from test test From 7c6d6a4ae68df96d0b8ba91d9fb3ef3885f90afb Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:06:27 -0700 Subject: [PATCH 10/17] Prevent running GHA workflow twice in response to same event --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5599a3..d5d6f88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,11 @@ name: CI +# Configure GHA to run this workflow whenever either: +# (a) a pull request is opened or updated, or +# (b) a commit(s) is pushed into the `main` branch. on: - push: -# branches: -# - main - pull_request: + pull_request: { } + push: { branches: [ main ] } jobs: pr-check: From bb50b285db646de60869bb4234aa42c8d63e9456 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:19:58 -0700 Subject: [PATCH 11/17] Propagate host env vars into containers and run all tests locally also --- .github/workflows/ci.yml | 18 +++++------------- docker-compose.yml | 15 +++++++++++++++ src/config.py | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5d6f88..a877db5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,31 +33,23 @@ jobs: - name: Linting run: uv run ruff check -- src/ - - name: Run tests locally - run: uv run pytest tests - # Note: This spins up containers running the default services. # This does not include the `ingest` or `test` services, # which aren't part of the default profile. - name: Spin up Docker Compose stack in background - env: - # Customize MongoDB connection parameters for all containers in the stack. - MONGO_USERNAME: admin - MONGO_PASSWORD: root - MONGO_HOST: mongo run: docker compose up --detach # Note: The `--exit-code-from test` option applies the exit code of the `test` container # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container - env: - # Customize MongoDB connection parameters for the `test` container. - MONGO_USERNAME: admin - MONGO_PASSWORD: root - MONGO_HOST: mongo run: docker compose up --exit-code-from test test + # Run the tests outside of the Docker Compose stack, to ensure they'll pass for developers + # running them locally (e.g. via `uv run pytest`). + - name: Run tests locally + run: uv run pytest -v + # Note: This spins everything down. - name: Spin down Docker Compose stack run: docker compose down diff --git a/docker-compose.yml b/docker-compose.yml index 44a6008..9bdbae6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,15 @@ services: # Use the container image built in the "development" stage of the Dockerfile. build: { context: ".", dockerfile: Dockerfile, target: development } restart: unless-stopped + # Propagate environment variables from the host into the container, falling + # back to default values for variables not defined on the host. + # Docs: https://docs.docker.com/compose/how-tos/environment-variables/set-environment-variables/#use-the-environment-attribute + environment: + MONGO_HOST: ${MONGO_HOST:-mongo} + MONGO_PORT: ${MONGO_PORT:-27017} + MONGO_USERNAME: ${MONGO_USERNAME:-admin} + MONGO_PASSWORD: ${MONGO_PASSWORD:-root} + MONGO_DATABASE: ${MONGO_DATABASE:-bertron} ports: # Map a host port (by default, 8000, but it can be overridden via an # environment variable) to port 8000 of the container; the latter being @@ -53,6 +62,12 @@ services: build: { context: ".", dockerfile: Dockerfile, target: development } # This service should not start automatically - only run on demand profiles: ["tools"] + environment: + MONGO_HOST: ${MONGO_HOST:-mongo} + MONGO_PORT: ${MONGO_PORT:-27017} + MONGO_USERNAME: ${MONGO_USERNAME:-admin} + MONGO_PASSWORD: ${MONGO_PASSWORD:-root} + MONGO_DATABASE: ${MONGO_DATABASE:-bertron} # the test suite will disregard this depends_on: - app - mongo diff --git a/src/config.py b/src/config.py index 72aafc7..c5fbf05 100644 --- a/src/config.py +++ b/src/config.py @@ -25,9 +25,9 @@ class Settings(BaseSettings): # MongoDB connection settings. mongo_host: str = "localhost" mongo_port: int = 27017 # note: value in `.env` will be coerced into int - mongo_database: str = "bertron" mongo_username: Optional[str] = None mongo_password: Optional[str] = None + mongo_database: str = "bertron" # Instantiate a settings object that can be imported into other modules. From bf9575a27d8c9e40d7eef5f0940685bac0e3a028 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:33:44 -0700 Subject: [PATCH 12/17] Prevent local test step from using `.venv/` leftover by container --- .dockerignore | 8 ++++++++ .github/workflows/ci.yml | 5 +++++ .gitignore | 6 ++++-- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9b2c452 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +# This file specifies directories/files we want Docker to omit from the Docker build context. +# Docs: https://docs.docker.com/build/concepts/context/#dockerignore-files + +# Ignore Python virtual environments. +.venv/ + +# Ignore Python cache files. +__pycache__/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a877db5..5778185 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,11 @@ jobs: - name: Spin up `test` container run: docker compose up --exit-code-from test test + # Delete the Python virtual environment created by the container. Otherwise, + # it will conflict with the one created locally in the next step. + - name: Delete leftover Python virtual environment + run: rm -rf ./.venv + # Run the tests outside of the Docker Compose stack, to ensure they'll pass for developers # running them locally (e.g. via `uv run pytest`). - name: Run tests locally diff --git a/.gitignore b/.gitignore index fefca85..e82b666 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,10 @@ # Jupyter Notebook checkpoints. **/.ipynb_checkpoints/ -# TODO: Document these. -.venv +# Ignore Python virtual environments. +.venv/ + +# TODO: Document this. **/*.egg-info/ # Ignore Python bytecode files. From 77f175f5f133a63810085b3f20cd99811e4494f3 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:35:49 -0700 Subject: [PATCH 13/17] Refine comments --- .dockerignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9b2c452..6f94157 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,8 @@ # This file specifies directories/files we want Docker to omit from the Docker build context. # Docs: https://docs.docker.com/build/concepts/context/#dockerignore-files -# Ignore Python virtual environments. +# Omit Python virtual environments. .venv/ -# Ignore Python cache files. +# Omit Python cache files. __pycache__/ From 8dafd53a5b8e876be783fa3f8eeb4b761dbf9abb Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:36:15 -0700 Subject: [PATCH 14/17] Use privileged user to delete `.venv/` leftover by container --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5778185..965f360 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: # Delete the Python virtual environment created by the container. Otherwise, # it will conflict with the one created locally in the next step. - name: Delete leftover Python virtual environment - run: rm -rf ./.venv + run: sudo rm -rf ./.venv # Run the tests outside of the Docker Compose stack, to ensure they'll pass for developers # running them locally (e.g. via `uv run pytest`). From f849844d50db132e8b163e0fc1b583e2bca09fc5 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 14:37:46 -0700 Subject: [PATCH 15/17] Run tests only within Docker Compose stack, not on host (simpler setup) --- .github/workflows/ci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 965f360..2cfb6af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,16 +45,6 @@ jobs: - name: Spin up `test` container run: docker compose up --exit-code-from test test - # Delete the Python virtual environment created by the container. Otherwise, - # it will conflict with the one created locally in the next step. - - name: Delete leftover Python virtual environment - run: sudo rm -rf ./.venv - - # Run the tests outside of the Docker Compose stack, to ensure they'll pass for developers - # running them locally (e.g. via `uv run pytest`). - - name: Run tests locally - run: uv run pytest -v - # Note: This spins everything down. - name: Spin down Docker Compose stack run: docker compose down From 59c8a172f5847c23cd266fd6f480f91f372ce348 Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 16:40:06 -0700 Subject: [PATCH 16/17] Display error message when host or `.env` file lacks required env. var. --- .env.example | 2 +- .github/workflows/ci.yml | 12 ++++++++++++ docker-compose.yml | 32 +++++++++++++++++--------------- src/config.py | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index 92fc545..89f4b91 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,6 @@ MONGO_HOST=mongo MONGO_PORT=27017 -MONGO_DATABASE=bertron MONGO_USERNAME=admin MONGO_PASSWORD=root +MONGO_DATABASE=bertron diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cfb6af..6d0f5e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,12 +37,24 @@ jobs: # This does not include the `ingest` or `test` services, # which aren't part of the default profile. - name: Spin up Docker Compose stack in background + env: + MONGO_HOST: mongo + MONGO_PORT: 27017 + MONGO_USERNAME: admin + MONGO_PASSWORD: root + MONGO_DATABASE: bertron run: docker compose up --detach # Note: The `--exit-code-from test` option applies the exit code of the `test` container # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container + env: + MONGO_HOST: mongo + MONGO_PORT: 27017 + MONGO_USERNAME: admin + MONGO_PASSWORD: root + MONGO_DATABASE: bertron # the test suite will disregard this run: docker compose up --exit-code-from test test # Note: This spins everything down. diff --git a/docker-compose.yml b/docker-compose.yml index 9bdbae6..2da64b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,15 +3,17 @@ services: # Use the container image built in the "development" stage of the Dockerfile. build: { context: ".", dockerfile: Dockerfile, target: development } restart: unless-stopped - # Propagate environment variables from the host into the container, falling - # back to default values for variables not defined on the host. - # Docs: https://docs.docker.com/compose/how-tos/environment-variables/set-environment-variables/#use-the-environment-attribute + # Propagate environment variables from the host into the container. + # Note: The `:?` suffix makes it so Docker Compose displays an error if the + # environment variable is either (a) undefined, or (b) empty, in the + # host environment. + # Docs: https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/#additional-information environment: - MONGO_HOST: ${MONGO_HOST:-mongo} - MONGO_PORT: ${MONGO_PORT:-27017} - MONGO_USERNAME: ${MONGO_USERNAME:-admin} - MONGO_PASSWORD: ${MONGO_PASSWORD:-root} - MONGO_DATABASE: ${MONGO_DATABASE:-bertron} + MONGO_HOST: ${MONGO_HOST:?} + MONGO_PORT: ${MONGO_PORT:?} + MONGO_USERNAME: ${MONGO_USERNAME:?} + MONGO_PASSWORD: ${MONGO_PASSWORD:?} + MONGO_DATABASE: ${MONGO_DATABASE:?} ports: # Map a host port (by default, 8000, but it can be overridden via an # environment variable) to port 8000 of the container; the latter being @@ -39,8 +41,8 @@ services: - "${MONGO_PORT:-27017}:27017" restart: unless-stopped environment: - MONGO_INITDB_ROOT_USERNAME: "${MONGO_USERNAME:-admin}" - MONGO_INITDB_ROOT_PASSWORD: "${MONGO_PASSWORD:-root}" + MONGO_INITDB_ROOT_USERNAME: "${MONGO_USERNAME:?}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_PASSWORD:?}" volumes: - mongo_data:/data/db @@ -63,11 +65,11 @@ services: # This service should not start automatically - only run on demand profiles: ["tools"] environment: - MONGO_HOST: ${MONGO_HOST:-mongo} - MONGO_PORT: ${MONGO_PORT:-27017} - MONGO_USERNAME: ${MONGO_USERNAME:-admin} - MONGO_PASSWORD: ${MONGO_PASSWORD:-root} - MONGO_DATABASE: ${MONGO_DATABASE:-bertron} # the test suite will disregard this + MONGO_HOST: ${MONGO_HOST:?} + MONGO_PORT: ${MONGO_PORT:?} + MONGO_USERNAME: ${MONGO_USERNAME:?} + MONGO_PASSWORD: ${MONGO_PASSWORD:?} + MONGO_DATABASE: ${MONGO_DATABASE:?} # the test suite will disregard this depends_on: - app - mongo diff --git a/src/config.py b/src/config.py index c5fbf05..962f506 100644 --- a/src/config.py +++ b/src/config.py @@ -24,7 +24,7 @@ class Settings(BaseSettings): # MongoDB connection settings. mongo_host: str = "localhost" - mongo_port: int = 27017 # note: value in `.env` will be coerced into int + mongo_port: int = 27017 # note: env. var. value will be coerced into int mongo_username: Optional[str] = None mongo_password: Optional[str] = None mongo_database: str = "bertron" From cc54a5266bd889569ffef8b90a31d9fc70afcd4d Mon Sep 17 00:00:00 2001 From: eecavanna Date: Sat, 19 Jul 2025 16:44:24 -0700 Subject: [PATCH 17/17] Define environment variables for `docker compose down` step --- .github/workflows/ci.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d0f5e8..10f2458 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,14 @@ jobs: name: PR checks runs-on: ubuntu-latest + # Define environment variables that will be available to all steps within this job. + env: + MONGO_HOST: mongo + MONGO_PORT: 27017 + MONGO_USERNAME: admin + MONGO_PASSWORD: root + MONGO_DATABASE: bertron + steps: - uses: actions/checkout@v4 @@ -37,24 +45,12 @@ jobs: # This does not include the `ingest` or `test` services, # which aren't part of the default profile. - name: Spin up Docker Compose stack in background - env: - MONGO_HOST: mongo - MONGO_PORT: 27017 - MONGO_USERNAME: admin - MONGO_PASSWORD: root - MONGO_DATABASE: bertron run: docker compose up --detach # Note: The `--exit-code-from test` option applies the exit code of the `test` container # to the `docker compose` process, so that the GHA step fails if tests fail. # Reference: https://docs.docker.com/reference/cli/docker/compose/up/ - name: Spin up `test` container - env: - MONGO_HOST: mongo - MONGO_PORT: 27017 - MONGO_USERNAME: admin - MONGO_PASSWORD: root - MONGO_DATABASE: bertron # the test suite will disregard this run: docker compose up --exit-code-from test test # Note: This spins everything down.