From 328461580ad92e2f1553682ffcd2c3793bbcb3e3 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 21 Apr 2026 07:51:06 -0500 Subject: [PATCH 1/2] Implement logger to stdout and file --- Makefile | 4 ++- README.md | 6 +++- app/apilogger.py | 85 +++++++++++++++++++++++++++++++++++++++++------- app/main.py | 10 +++--- 4 files changed, 86 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 14195d59..e8ca8cd3 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ VENV := .venv BIN := $(VENV)/bin UV := uv PIP := $(BIN)/pip +LOG_FILE := runtime-logs.log +IRI_LOG_FILE ?= $(LOG_FILE) STAMP_VENV := $(VENV)/.created STAMP_DEPS := $(VENV)/.deps @@ -34,6 +36,7 @@ dev: deps IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + IRI_LOG_FILE="$${IRI_LOG_FILE:-$${LOG_FILE:-$(IRI_LOG_FILE)}}" \ DEMO_QUEUE_UPDATE_SECS=2 \ OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://localhost:8000' fastapi dev @@ -78,4 +81,3 @@ ARGS ?= # call it via: make manage-globus ARGS=scopes-show manage-globus: deps @source local.env && $(BIN)/python ./tools/manage_globus.py $(ARGS) - diff --git a/README.md b/README.md index b47226df..0f8f49ce 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,11 @@ Links to data, created by this api, will concatenate these values producing link - `IRI_API_PARAMS`: as described above, this is a way to customize the API meta-data - `IRI_API_ADAPTER_*`: these values specify the business logic for the per-api-group implementation of a facility_adapter. For example: `IRI_API_ADAPTER_status=myfacility.MyFacilityStatusAdapter` would load the implementation of the `app.routers.status.facility_adapter.FacilityAdapter` abstract class to handle the `status` business logic for your facility. - `IRI_SHOW_MISSING_ROUTES`: hide api groups that don't have an `IRI_API_ADAPTER_*` environment variable defined, if set to `true`. This way if your facility only wishes to expose some api groups but not others, they can be hidden. (Defaults to `false`.) +- `LOG_LEVEL`: logging level for the API and adapters. Defaults to `DEBUG`. +- `IRI_LOG_FILE`: file path for API logs. Logs always go to stdout; when this is set, logs also go to the file. +- `LOG_FILE`: fallback file path for API logs when `IRI_LOG_FILE` is not set. + +For local development, `make` writes logs to `runtime-logs.log` by default. Use `make LOG_FILE=/tmp/iri-api.log` or `make IRI_LOG_FILE=/tmp/iri-api.log` to choose a different file. You can also put either variable in `local.env`. ## Docker support @@ -142,4 +147,3 @@ You can optionally use globus for authorization. Steps to use globus: - Specify the monitoring endpoint by setting the [OpenTelemetry](https://opentelemetry.io/docs/zero-code/python/) env vars - Add additional routers for other API-s - Add authenticated API-s via an [OAuth2 integration](https://fastapi.tiangolo.com/tutorial/security/oauth2-jwt/) - diff --git a/app/apilogger.py b/app/apilogger.py index 80403a9f..1bf7302c 100644 --- a/app/apilogger.py +++ b/app/apilogger.py @@ -1,5 +1,9 @@ """Logging utilities for the IRI Facility API.""" + import logging +import os +import sys +from pathlib import Path LEVELS = {"FATAL": logging.FATAL, "ERROR": logging.ERROR, @@ -7,22 +11,81 @@ "INFO": logging.INFO, "DEBUG": logging.DEBUG} +DEFAULT_FORMAT = "%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s" +DEFAULT_DATE_FORMAT = "%a, %d %b %Y %H:%M:%S" +IRI_HANDLER_ATTR = "_iri_facility_api_handler" -def get_stream_logger(name: str = __name__, level: str = "DEBUG") -> logging.Logger: +_CONFIGURED = False + + +def _level(level: str | int | None) -> int: + if isinstance(level, int): + return level + return LEVELS.get(str(level or "INFO").upper(), logging.INFO) + + +def _log_file_path() -> Path | None: + log_file = os.environ.get("IRI_LOG_FILE") or os.environ.get("LOG_FILE") + return Path(log_file) if log_file else None + + +def configure_logging(level: str | int | None = None) -> None: """ - Return a configured Stream logger. + Configure root logging for the API. + + Logs always go to stdout. If IRI_LOG_FILE or LOG_FILE is set, logs also go + to that file. """ - logger = logging.getLogger(name) + global _CONFIGURED + + log_level = _level(level or os.environ.get("LOG_LEVEL")) + root = logging.getLogger() + root.setLevel(log_level) - if not logger.handlers: - handler = logging.StreamHandler() + if _CONFIGURED: + for handler in root.handlers: + if getattr(handler, IRI_HANDLER_ATTR, False): + handler.setLevel(log_level) + return - formatter = logging.Formatter("%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s", datefmt="%a, %d %b %Y %H:%M:%S") + formatter = logging.Formatter(DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT) - handler.setFormatter(formatter) - logger.addHandler(handler) + for handler in root.handlers[:]: + if getattr(handler, IRI_HANDLER_ATTR, False): + root.removeHandler(handler) + handler.close() + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(log_level) + stdout_handler.setFormatter(formatter) + setattr(stdout_handler, IRI_HANDLER_ATTR, True) + root.addHandler(stdout_handler) + + log_file = _log_file_path() + if log_file: + if log_file.parent != Path("."): + log_file.parent.mkdir(parents=True, exist_ok=True) + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(log_level) + file_handler.setFormatter(formatter) + setattr(file_handler, IRI_HANDLER_ATTR, True) + root.addHandler(file_handler) + + _CONFIGURED = True + + +def get_stream_logger(name: str = __name__, level: str = "DEBUG") -> logging.Logger: + """ + Return a logger using the shared API stdout and optional file logging setup. + """ + configure_logging(level) + + logger = logging.getLogger(name) + logger.setLevel(_level(level)) + logger.propagate = True - logger.setLevel(LEVELS.get(level, logging.DEBUG)) - logger.propagate = False + for handler in logger.handlers[:]: + logger.removeHandler(handler) + handler.close() - return logger \ No newline at end of file + return logger diff --git a/app/main.py b/app/main.py index 62147590..49317718 100644 --- a/app/main.py +++ b/app/main.py @@ -11,6 +11,9 @@ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from . import config +from .apilogger import configure_logging + from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility from app.routers.status import status @@ -19,12 +22,7 @@ from app.routers.filesystem import filesystem from app.routers.task import task -from . import config - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(name)s: %(message)s" -) +configure_logging(config.LOG_LEVEL) # ------------------------------------------------------------------ # OpenTelemetry Tracing Configuration From 14f29a47d9dc5e7f5a823feafa9f4c3cbc48e90b Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 24 Apr 2026 09:46:20 -0500 Subject: [PATCH 2/2] Log rotation --- Makefile | 3 +++ README.md | 4 +++- app/apilogger.py | 19 ++++++++++++++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e8ca8cd3..2d252b26 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,8 @@ UV := uv PIP := $(BIN)/pip LOG_FILE := runtime-logs.log IRI_LOG_FILE ?= $(LOG_FILE) +LOG_ROTATION_DAYS := 5 +IRI_LOG_ROTATION_DAYS ?= $(LOG_ROTATION_DAYS) STAMP_VENV := $(VENV)/.created STAMP_DEPS := $(VENV)/.deps @@ -37,6 +39,7 @@ dev: deps IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ IRI_LOG_FILE="$${IRI_LOG_FILE:-$${LOG_FILE:-$(IRI_LOG_FILE)}}" \ + IRI_LOG_ROTATION_DAYS="$${IRI_LOG_ROTATION_DAYS:-$${LOG_ROTATION_DAYS:-$(IRI_LOG_ROTATION_DAYS)}}" \ DEMO_QUEUE_UPDATE_SECS=2 \ OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://localhost:8000' fastapi dev diff --git a/README.md b/README.md index 0f8f49ce..f5a1de24 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,10 @@ Links to data, created by this api, will concatenate these values producing link - `LOG_LEVEL`: logging level for the API and adapters. Defaults to `DEBUG`. - `IRI_LOG_FILE`: file path for API logs. Logs always go to stdout; when this is set, logs also go to the file. - `LOG_FILE`: fallback file path for API logs when `IRI_LOG_FILE` is not set. +- `IRI_LOG_ROTATION_DAYS`: number of daily rotated log files to retain. Defaults to `5`. +- `LOG_ROTATION_DAYS`: fallback retention when `IRI_LOG_ROTATION_DAYS` is not set. -For local development, `make` writes logs to `runtime-logs.log` by default. Use `make LOG_FILE=/tmp/iri-api.log` or `make IRI_LOG_FILE=/tmp/iri-api.log` to choose a different file. You can also put either variable in `local.env`. +For local development, `make` writes logs to `runtime-logs.log` by default and keeps `5` daily rotated files. Use `make LOG_FILE=/tmp/iri-api.log`, `make IRI_LOG_FILE=/tmp/iri-api.log`, or `make LOG_ROTATION_DAYS=10` to override those defaults. You can also put the same variables in `local.env`. ## Docker support diff --git a/app/apilogger.py b/app/apilogger.py index 1bf7302c..344bd65c 100644 --- a/app/apilogger.py +++ b/app/apilogger.py @@ -3,6 +3,7 @@ import logging import os import sys +from logging.handlers import TimedRotatingFileHandler from pathlib import Path LEVELS = {"FATAL": logging.FATAL, @@ -14,6 +15,7 @@ DEFAULT_FORMAT = "%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s" DEFAULT_DATE_FORMAT = "%a, %d %b %Y %H:%M:%S" IRI_HANDLER_ATTR = "_iri_facility_api_handler" +DEFAULT_ROTATION_DAYS = 5 _CONFIGURED = False @@ -29,6 +31,15 @@ def _log_file_path() -> Path | None: return Path(log_file) if log_file else None +def _rotation_days() -> int: + raw_days = os.environ.get("IRI_LOG_ROTATION_DAYS") or os.environ.get("LOG_ROTATION_DAYS") + try: + days = int(raw_days) if raw_days is not None else DEFAULT_ROTATION_DAYS + except ValueError: + days = DEFAULT_ROTATION_DAYS + return max(days, 0) + + def configure_logging(level: str | int | None = None) -> None: """ Configure root logging for the API. @@ -65,7 +76,13 @@ def configure_logging(level: str | int | None = None) -> None: if log_file: if log_file.parent != Path("."): log_file.parent.mkdir(parents=True, exist_ok=True) - file_handler = logging.FileHandler(log_file) + file_handler = TimedRotatingFileHandler( + log_file, + when="midnight", + interval=1, + backupCount=_rotation_days(), + encoding="utf-8", + ) file_handler.setLevel(log_level) file_handler.setFormatter(formatter) setattr(file_handler, IRI_HANDLER_ATTR, True)