From e600aa436cf0fe5cb2122f280a868ab8be1a7cf6 Mon Sep 17 00:00:00 2001 From: spomichter Date: Sun, 8 Mar 2026 19:11:08 +0000 Subject: [PATCH] fix: restore RotatingFileHandler to prevent OOM from unbounded log growth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daemon PR (#1436) replaced RotatingFileHandler with plain FileHandler to avoid a theoretical race when forkserver workers rotate the same file. However, this removed the 10 MiB × 20 backup cap entirely, causing unbounded log growth. Camera + LCM at 30 fps writes ~100 MB/min of JSON logs, leading to OOM and full system crashes after 5-10 minutes. Restore RotatingFileHandler with the original limits. The forkserver rotation race can lose a few interleaved lines but never caused issues in practice, whereas unbounded growth is a production-breaking OOM. --- dimos/utils/logging_config.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/dimos/utils/logging_config.py b/dimos/utils/logging_config.py index f159d13d43..bf7632fa60 100644 --- a/dimos/utils/logging_config.py +++ b/dimos/utils/logging_config.py @@ -62,7 +62,13 @@ def set_run_log_dir(log_dir: str | Path) -> None: for i, handler in enumerate(logger_obj.handlers): if isinstance(handler, logging.FileHandler) and handler.baseFilename != str(new_path): handler.close() - new_handler = logging.FileHandler(new_path, mode="a", encoding="utf-8") + new_handler = logging.handlers.RotatingFileHandler( + new_path, + mode="a", + maxBytes=10 * 1024 * 1024, # 10 MiB + backupCount=20, + encoding="utf-8", + ) new_handler.setLevel(handler.level) new_handler.setFormatter(handler.formatter) logger_obj.handlers[i] = new_handler @@ -273,12 +279,16 @@ def setup_logger(*, level: int | None = None) -> Any: console_handler.setFormatter(console_formatter) stdlib_logger.addHandler(console_handler) - # Plain FileHandler (not RotatingFileHandler) — rotation is unsafe with - # forkserver workers writing to the same file. Per-run logs are scoped to - # a single run so unbounded growth is not a concern. - file_handler = logging.FileHandler( + # RotatingFileHandler with a size cap to prevent unbounded log growth. + # Multiple forkserver workers may each open their own handler to the same + # file — a concurrent rotate can lose a few lines, but that is far + # preferable to unbounded growth causing OOM on resource-constrained + # devices (cameras + LCM at 30 fps can write ~100 MB/min of JSON logs). + file_handler = logging.handlers.RotatingFileHandler( log_file_path, mode="a", + maxBytes=10 * 1024 * 1024, # 10 MiB + backupCount=20, encoding="utf-8", )