From b2c7b516cb6c0140e3b32d31f96da4cb9198524c Mon Sep 17 00:00:00 2001 From: Timothy Nunn Date: Thu, 8 Jan 2026 10:24:55 +0000 Subject: [PATCH 1/6] Write an additional log file in PROCESS working directory --- process/main.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/process/main.py b/process/main.py index 97c074e6b1..78abe46899 100644 --- a/process/main.py +++ b/process/main.py @@ -214,6 +214,9 @@ def parse_args(self, args): self.args = parser.parse_args(args) # Store namespace object of the args + wkdir_path_length = len(self.args.input.lower().replace("in.dat", "")) + setup_loggers(Path(self.args.input[:wkdir_path_length])) + def run_mode(self): """Determine how to run Process.""" if self.args.version: @@ -729,15 +732,27 @@ def costs(self, value: CostsProtocol): logging_model_handler.setFormatter(logging_formatter) -def setup_loggers(): +def setup_loggers(process_working_directory: Path | None = None): """A function that adds our handlers to the appropriate logger object.""" # Only add our handlers if PROCESS is being run as an application # This should allow it to be used as a package (e.g. people import models that log) # without creating a process.log file... people can then handle our logs as they wish. + + # These are the list of handlers to add only if a root logger has not already been created. + handlers = [logging_stream_handler, logging_file_handler] + + if process_working_directory is not None: + logging_file_input_location_handler = logging.FileHandler( + f"{process_working_directory.as_posix()}process.log", mode="w" + ) + logging_file_input_location_handler.setLevel(logging.INFO) + logging_file_input_location_handler.setFormatter(logging_formatter) + handlers.append(logging_file_input_location_handler) + # Using basicConfig adds these handlers to the root logger iff the root logger has not # been setup yet. This means that during testing these hanlders won't be present, which # will ensure they do not conflict with the pytest handlers. - logging.basicConfig(handlers=[logging_stream_handler, logging_file_handler]) + logging.basicConfig(handlers=handlers) # However, this handler we know to be safe and necessary so we add it to the root logger # regardless of whether it has already been created. @@ -757,8 +772,6 @@ def main(args=None): :type args: list, optional """ - setup_loggers() - Process(args) From 9bc0532366e4089e64f2e1ee037e18be7095f7e1 Mon Sep 17 00:00:00 2001 From: Timothy Nunn Date: Mon, 12 Jan 2026 13:18:24 +0000 Subject: [PATCH 2/6] Initialise loggers in Single/VaryRun to enable non-CLI consistency --- process/main.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/process/main.py b/process/main.py index 78abe46899..538258c199 100644 --- a/process/main.py +++ b/process/main.py @@ -214,9 +214,6 @@ def parse_args(self, args): self.args = parser.parse_args(args) # Store namespace object of the args - wkdir_path_length = len(self.args.input.lower().replace("in.dat", "")) - setup_loggers(Path(self.args.input[:wkdir_path_length])) - def run_mode(self): """Determine how to run Process.""" if self.args.version: @@ -307,6 +304,8 @@ def run(self): config = RunProcessConfig(self.config_file) config.setup() + setup_loggers(config.wdir / "process.log") + init.init_all_module_vars() init.init_process() @@ -446,9 +445,12 @@ def set_mfile(self): """Set the mfile filename.""" self.mfile_path = Path(self.filename_prefix + "MFILE.DAT") - @staticmethod - def initialise(): + def initialise(self): """Run the init module to call all initialisation routines.""" + setup_loggers( + Path(self.output_path.as_posix().replace("OUT.DAT", "process.log")) + ) + initialise_imprad() # Reads in input file init.init_process() @@ -732,7 +734,7 @@ def costs(self, value: CostsProtocol): logging_model_handler.setFormatter(logging_formatter) -def setup_loggers(process_working_directory: Path | None = None): +def setup_loggers(working_directory_log_path: Path | None = None): """A function that adds our handlers to the appropriate logger object.""" # Only add our handlers if PROCESS is being run as an application # This should allow it to be used as a package (e.g. people import models that log) @@ -741,9 +743,9 @@ def setup_loggers(process_working_directory: Path | None = None): # These are the list of handlers to add only if a root logger has not already been created. handlers = [logging_stream_handler, logging_file_handler] - if process_working_directory is not None: + if working_directory_log_path is not None: logging_file_input_location_handler = logging.FileHandler( - f"{process_working_directory.as_posix()}process.log", mode="w" + working_directory_log_path.as_posix(), mode="w" ) logging_file_input_location_handler.setLevel(logging.INFO) logging_file_input_location_handler.setFormatter(logging_formatter) From a3c32fa782788ce0136ed415b70d25875e82be77 Mon Sep 17 00:00:00 2001 From: Timothy Nunn Date: Mon, 12 Jan 2026 13:25:40 +0000 Subject: [PATCH 3/6] Set working directory process.log to append mode --- process/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/main.py b/process/main.py index 538258c199..fba531fe4b 100644 --- a/process/main.py +++ b/process/main.py @@ -726,7 +726,7 @@ def costs(self, value: CostsProtocol): logging_stream_handler.setLevel(logging.CRITICAL) logging_stream_handler.setFormatter(logging_formatter) -logging_file_handler = logging.FileHandler("process.log", mode="w") +logging_file_handler = logging.FileHandler("process.log", mode="a") logging_file_handler.setLevel(logging.INFO) logging_file_handler.setFormatter(logging_formatter) From de80fe2961246378af38c2fbfae896d62d7b6a70 Mon Sep 17 00:00:00 2001 From: Timothy Nunn Date: Mon, 12 Jan 2026 13:32:22 +0000 Subject: [PATCH 4/6] Ensure VaryRun setup_loggers uses Path --- process/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/main.py b/process/main.py index fba531fe4b..04c68c5adf 100644 --- a/process/main.py +++ b/process/main.py @@ -304,7 +304,7 @@ def run(self): config = RunProcessConfig(self.config_file) config.setup() - setup_loggers(config.wdir / "process.log") + setup_loggers(Path(config.wdir) / "process.log") init.init_all_module_vars() init.init_process() From 3cf2ffbf870972014714e5fa14aadd39eaea296f Mon Sep 17 00:00:00 2001 From: Timothy Nunn Date: Thu, 15 Jan 2026 14:18:20 +0000 Subject: [PATCH 5/6] Simplify setting up of PROCESS package loggers --- process/main.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/process/main.py b/process/main.py index 04c68c5adf..b4c8bbfb2f 100644 --- a/process/main.py +++ b/process/main.py @@ -111,7 +111,7 @@ os.environ["PYTHON_PROCESS_ROOT"] = os.path.join(os.path.dirname(__file__)) -logger = logging.getLogger(__name__) +logger = logging.getLogger("process") class Process: @@ -736,12 +736,13 @@ def costs(self, value: CostsProtocol): def setup_loggers(working_directory_log_path: Path | None = None): """A function that adds our handlers to the appropriate logger object.""" - # Only add our handlers if PROCESS is being run as an application - # This should allow it to be used as a package (e.g. people import models that log) - # without creating a process.log file... people can then handle our logs as they wish. + # Remove all of the existing handlers from the 'process' package logger + logger.handlers.clear() - # These are the list of handlers to add only if a root logger has not already been created. - handlers = [logging_stream_handler, logging_file_handler] + # (Re)add the loggers to the 'process' package logger (and its children) + logger.addHandler(logging_stream_handler) + logger.addHandler(logging_file_handler) + logger.addHandler(logging_model_handler) if working_directory_log_path is not None: logging_file_input_location_handler = logging.FileHandler( @@ -749,17 +750,7 @@ def setup_loggers(working_directory_log_path: Path | None = None): ) logging_file_input_location_handler.setLevel(logging.INFO) logging_file_input_location_handler.setFormatter(logging_formatter) - handlers.append(logging_file_input_location_handler) - - # Using basicConfig adds these handlers to the root logger iff the root logger has not - # been setup yet. This means that during testing these hanlders won't be present, which - # will ensure they do not conflict with the pytest handlers. - logging.basicConfig(handlers=handlers) - - # However, this handler we know to be safe and necessary so we add it to the root logger - # regardless of whether it has already been created. - root_logger = logging.getLogger() - root_logger.addHandler(logging_model_handler) + logger.addHandler(logging_file_input_location_handler) def main(args=None): From 2c331385fdb22182158503dd498998c48029c50b Mon Sep 17 00:00:00 2001 From: Timothy Nunn Date: Thu, 15 Jan 2026 14:30:38 +0000 Subject: [PATCH 6/6] Disable most logger handlers during testing --- process/main.py | 11 ++++++++++- tests/conftest.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/process/main.py b/process/main.py index b4c8bbfb2f..fe0e4a9f48 100644 --- a/process/main.py +++ b/process/main.py @@ -111,6 +111,8 @@ os.environ["PYTHON_PROCESS_ROOT"] = os.path.join(os.path.dirname(__file__)) +PACKAGE_LOGGING = True +"""Can be set False to disable package-level logging, e.g. in the test suite""" logger = logging.getLogger("process") @@ -737,12 +739,19 @@ def costs(self, value: CostsProtocol): def setup_loggers(working_directory_log_path: Path | None = None): """A function that adds our handlers to the appropriate logger object.""" # Remove all of the existing handlers from the 'process' package logger + logger.handlers.clear() + # we always want to add this handler because otherwise PROCESS' error + # handling system won't work properly + logger.addHandler(logging_model_handler) + + if not PACKAGE_LOGGING: + return + # (Re)add the loggers to the 'process' package logger (and its children) logger.addHandler(logging_stream_handler) logger.addHandler(logging_file_handler) - logger.addHandler(logging_model_handler) if working_directory_log_path is not None: logging_file_input_location_handler = logging.FileHandler( diff --git a/tests/conftest.py b/tests/conftest.py index c711855233..3d1e657cce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from _pytest.fixtures import SubRequest from system_check import system_compatible +from process import main from process.log import logging_model_handler @@ -157,3 +158,12 @@ def return_to_root(): cwd = os.getcwd() yield os.chdir(cwd) + + +@pytest.fixture(autouse=True) +def disable_package_logger(monkeypatch): + """Various parts of PROCESS change directories and do not always change back. + This fixture ensures that, at the end of each test, the cwd is reset to what it + was at the beginning of the test. + """ + monkeypatch.setattr(main, "PACKAGE_LOGGING", False)