From afe91028f5f5aa8e6b384344d4c5805d6b174a34 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 11 Nov 2024 14:10:27 +0000 Subject: [PATCH 1/4] :sparkles: Add log to file. --- docs/inputs/toml.rst | 3 +++ src/muse/__init__.py | 28 ++++++++++++++++++++++++++++ src/muse/__main__.py | 3 ++- src/muse/examples.py | 2 ++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/inputs/toml.rst b/docs/inputs/toml.rst index 6ebaf6dad..0431aab7d 100644 --- a/docs/inputs/toml.rst +++ b/docs/inputs/toml.rst @@ -66,6 +66,9 @@ a whole. *log_level* verbosity of the output. Valid options, from the highest to the lowest level of verbosity, are: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". +*log_file* + Path to a file where the log will be saved, in addition to printing it to the console. If not given, the log will be added to a file named `muse_CURRENT_DATETIME.log` in the current working directory. + *equilibirum_variable* whether equilibrium of `demand` or `prices` should be sought. Defaults to `demand`. diff --git a/src/muse/__init__.py b/src/muse/__init__.py index 3f6b24984..10357bf48 100644 --- a/src/muse/__init__.py +++ b/src/muse/__init__.py @@ -34,6 +34,34 @@ def _create_logger(color: bool = True): return logger +def add_file_logger(file_path: str | None) -> None: + """Adds a file logger to the main logger. + + If the file already exists, it is deleted. + + Args: + file_path (str): Path to the file where the logs will be written. + """ + import datetime + import logging + from pathlib import Path + + if not file_path: + file_path = ( + f"muse_{datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S-%f')}.log" + ) + + if (path := Path(file_path)).exists(): + path.unlink() + + file_handler = logging.FileHandler(file_path) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + logging.getLogger(name=__name__).addHandler(file_handler) + + logger = _create_logger(os.environ.get("MUSE_COLOR_LOG") != "False") """ Main logger """ diff --git a/src/muse/__main__.py b/src/muse/__main__.py index a71bc7f0a..ad718ea96 100644 --- a/src/muse/__main__.py +++ b/src/muse/__main__.py @@ -38,7 +38,7 @@ def muse_main(settings, model, copy): from logging import getLogger from pathlib import Path - from muse import examples + from muse import add_file_logger, examples from muse.mca import MCA from muse.readers.toml import read_settings @@ -53,6 +53,7 @@ def muse_main(settings, model, copy): else: settings = read_settings(settings) getLogger("muse").setLevel(settings.log_level) + add_file_logger(getattr(settings, "log_file", None)) MCA.factory(settings).run() diff --git a/src/muse/examples.py b/src/muse/examples.py index 2ccc37d59..5a5d296c6 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -55,6 +55,7 @@ def model(name: str = "default") -> MCA: """Fully constructs a given example model.""" from tempfile import TemporaryDirectory + from muse import add_file_logger from muse.readers.toml import read_settings # we could modify the settings directly, but instead we use the copy_model function. @@ -63,6 +64,7 @@ def model(name: str = "default") -> MCA: path = copy_model(name, tmpdir) settings = read_settings(path / "settings.toml") getLogger("muse").setLevel(settings.log_level) + add_file_logger(getattr(settings, "log_file", None)) return MCA.factory(settings) From 1ff392c4f1746daf1e0a74592d79b0fd31a1c1f1 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 11 Nov 2024 15:19:57 +0000 Subject: [PATCH 2/4] :sparkles: Split logger into info and warning files. --- src/muse/__init__.py | 51 ++++++++++++++++++++++++++++---------------- src/muse/__main__.py | 3 +-- src/muse/examples.py | 2 -- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/muse/__init__.py b/src/muse/__init__.py index 10357bf48..17646587d 100644 --- a/src/muse/__init__.py +++ b/src/muse/__init__.py @@ -31,35 +31,50 @@ def _create_logger(color: bool = True): logger.setLevel(logging.DEBUG) + add_file_logger() + return logger -def add_file_logger(file_path: str | None) -> None: +def add_file_logger() -> None: """Adds a file logger to the main logger. - If the file already exists, it is deleted. - - Args: - file_path (str): Path to the file where the logs will be written. + The file logger is split into two files: one for INFO and DEBUG messages, and one + for WARNING messages and above to avoid cluttering the main log file and highlight + potential issues. """ - import datetime import logging from pathlib import Path - if not file_path: - file_path = ( - f"muse_{datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S-%f')}.log" - ) - - if (path := Path(file_path)).exists(): - path.unlink() + from .defaults import DEFAULT_OUTPUT_DIRECTORY - file_handler = logging.FileHandler(file_path) - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter( - logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) - logging.getLogger(name=__name__).addHandler(file_handler) + + # Sets the warning log, for warnings and above + warning_file = Path(DEFAULT_OUTPUT_DIRECTORY) / "muse_warning.log" + if warning_file.exists(): + warning_file.unlink() + + warning_file_handler = logging.FileHandler(warning_file) + warning_file_handler.setLevel(logging.WARNING) + warning_file_handler.setFormatter(formatter) + warning_file_handler.filters = [lambda record: record.levelno > logging.INFO] + + logging.getLogger(name=__name__).addHandler(warning_file_handler) + + # Sets the info log, for debug and info only + info_file = Path(DEFAULT_OUTPUT_DIRECTORY) / "muse_info.log" + if info_file.exists(): + info_file.unlink() + + info_file_handler = logging.FileHandler(info_file) + info_file_handler.setLevel(logging.DEBUG) + info_file_handler.setFormatter(formatter) + info_file_handler.filters = [lambda record: record.levelno <= logging.INFO] + + logging.getLogger(name=__name__).addHandler(info_file_handler) logger = _create_logger(os.environ.get("MUSE_COLOR_LOG") != "False") diff --git a/src/muse/__main__.py b/src/muse/__main__.py index ad718ea96..a71bc7f0a 100644 --- a/src/muse/__main__.py +++ b/src/muse/__main__.py @@ -38,7 +38,7 @@ def muse_main(settings, model, copy): from logging import getLogger from pathlib import Path - from muse import add_file_logger, examples + from muse import examples from muse.mca import MCA from muse.readers.toml import read_settings @@ -53,7 +53,6 @@ def muse_main(settings, model, copy): else: settings = read_settings(settings) getLogger("muse").setLevel(settings.log_level) - add_file_logger(getattr(settings, "log_file", None)) MCA.factory(settings).run() diff --git a/src/muse/examples.py b/src/muse/examples.py index 5a5d296c6..2ccc37d59 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -55,7 +55,6 @@ def model(name: str = "default") -> MCA: """Fully constructs a given example model.""" from tempfile import TemporaryDirectory - from muse import add_file_logger from muse.readers.toml import read_settings # we could modify the settings directly, but instead we use the copy_model function. @@ -64,7 +63,6 @@ def model(name: str = "default") -> MCA: path = copy_model(name, tmpdir) settings = read_settings(path / "settings.toml") getLogger("muse").setLevel(settings.log_level) - add_file_logger(getattr(settings, "log_file", None)) return MCA.factory(settings) From f61207149b826545d4de9395cf4520c781d2f7a7 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 11 Nov 2024 15:51:04 +0000 Subject: [PATCH 3/4] :bug: Fix tests (and code) to not have file loggers during tests. --- docs/inputs/toml.rst | 3 --- src/muse/__init__.py | 6 ++---- src/muse/__main__.py | 3 ++- src/muse/examples.py | 17 +++++++++++++++-- tests/test_aggregoutput.py | 6 +++--- tests/test_subsector.py | 2 +- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/docs/inputs/toml.rst b/docs/inputs/toml.rst index 0431aab7d..6ebaf6dad 100644 --- a/docs/inputs/toml.rst +++ b/docs/inputs/toml.rst @@ -66,9 +66,6 @@ a whole. *log_level* verbosity of the output. Valid options, from the highest to the lowest level of verbosity, are: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". -*log_file* - Path to a file where the log will be saved, in addition to printing it to the console. If not given, the log will be added to a file named `muse_CURRENT_DATETIME.log` in the current working directory. - *equilibirum_variable* whether equilibrium of `demand` or `prices` should be sought. Defaults to `demand`. diff --git a/src/muse/__init__.py b/src/muse/__init__.py index 17646587d..459a69006 100644 --- a/src/muse/__init__.py +++ b/src/muse/__init__.py @@ -31,8 +31,6 @@ def _create_logger(color: bool = True): logger.setLevel(logging.DEBUG) - add_file_logger() - return logger @@ -62,7 +60,7 @@ def add_file_logger() -> None: warning_file_handler.setFormatter(formatter) warning_file_handler.filters = [lambda record: record.levelno > logging.INFO] - logging.getLogger(name=__name__).addHandler(warning_file_handler) + logging.getLogger("muse").addHandler(warning_file_handler) # Sets the info log, for debug and info only info_file = Path(DEFAULT_OUTPUT_DIRECTORY) / "muse_info.log" @@ -74,7 +72,7 @@ def add_file_logger() -> None: info_file_handler.setFormatter(formatter) info_file_handler.filters = [lambda record: record.levelno <= logging.INFO] - logging.getLogger(name=__name__).addHandler(info_file_handler) + logging.getLogger("muse").addHandler(info_file_handler) logger = _create_logger(os.environ.get("MUSE_COLOR_LOG") != "False") diff --git a/src/muse/__main__.py b/src/muse/__main__.py index a71bc7f0a..6108b0b8a 100644 --- a/src/muse/__main__.py +++ b/src/muse/__main__.py @@ -38,7 +38,7 @@ def muse_main(settings, model, copy): from logging import getLogger from pathlib import Path - from muse import examples + from muse import add_file_logger, examples from muse.mca import MCA from muse.readers.toml import read_settings @@ -53,6 +53,7 @@ def muse_main(settings, model, copy): else: settings = read_settings(settings) getLogger("muse").setLevel(settings.log_level) + add_file_logger() MCA.factory(settings).run() diff --git a/src/muse/examples.py b/src/muse/examples.py index 2ccc37d59..9aa4fa9b6 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -51,10 +51,21 @@ def available_examples() -> list[str]: return [d.stem for d in example_data_dir().iterdir() if d.is_dir()] -def model(name: str = "default") -> MCA: - """Fully constructs a given example model.""" +def model(name: str = "default", test: bool = False) -> MCA: + """Fully constructs a given example model. + + File logging is added if ``test`` is False. + + Args: + name: Name of the model to load. + test: If True, the logging to file is not added. + + Returns: + The MCA model. + """ from tempfile import TemporaryDirectory + from muse import add_file_logger from muse.readers.toml import read_settings # we could modify the settings directly, but instead we use the copy_model function. @@ -63,6 +74,8 @@ def model(name: str = "default") -> MCA: path = copy_model(name, tmpdir) settings = read_settings(path / "settings.toml") getLogger("muse").setLevel(settings.log_level) + if not test: + add_file_logger() return MCA.factory(settings) diff --git a/tests/test_aggregoutput.py b/tests/test_aggregoutput.py index a5578d19f..f11e8e74b 100644 --- a/tests/test_aggregoutput.py +++ b/tests/test_aggregoutput.py @@ -9,7 +9,7 @@ def test_aggregate_sector(): """ from pandas import DataFrame, concat - mca = examples.model("multiple_agents") + mca = examples.model("multiple_agents", test=True) year = [2020, 2025] sector_list = [sector for sector in mca.sectors if "preset" not in sector.name] agent_list = [list(a.agents) for a in sector_list] @@ -43,7 +43,7 @@ def test_aggregate_sectors(): from muse.outputs.mca import _aggregate_sectors - mca = examples.model("multiple_agents") + mca = examples.model("multiple_agents", test=True) year = [2020, 2025, 2030] sector_list = [sector for sector in mca.sectors if "preset" not in sector.name] agent_list = [list(a.agents) for a in sector_list] @@ -83,7 +83,7 @@ def test_aggregate_sector_manyregions(): from muse.outputs.mca import _aggregate_sectors - mca = examples.model("multiple_agents") + mca = examples.model("multiple_agents", test=True) residential = next(sector for sector in mca.sectors if sector.name == "residential") agents = list(residential.agents) agents[0].assets["region"] = "BELARUS" diff --git a/tests/test_subsector.py b/tests/test_subsector.py index 9c326f1f4..244f80bd1 100644 --- a/tests/test_subsector.py +++ b/tests/test_subsector.py @@ -34,7 +34,7 @@ def test_subsector_investing_aggregation(): sector_list = ["residential", "power", "gas"] for model in model_list: - mca = examples.model(model) + mca = examples.model(model, test=True) for sname in sector_list: agents = list(examples.sector(sname, model).agents) sector = next(sector for sector in mca.sectors if sector.name == sname) From efcf42a4b1254b15a73ea54e469b20d8f9fe3bde Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 12 Nov 2024 09:57:40 +0000 Subject: [PATCH 4/4] :bug: Create default directoryb before start logging. --- src/muse/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/muse/__init__.py b/src/muse/__init__.py index 459a69006..a0fe9856a 100644 --- a/src/muse/__init__.py +++ b/src/muse/__init__.py @@ -50,6 +50,8 @@ def add_file_logger() -> None: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) + DEFAULT_OUTPUT_DIRECTORY.mkdir(parents=True, exist_ok=True) + # Sets the warning log, for warnings and above warning_file = Path(DEFAULT_OUTPUT_DIRECTORY) / "muse_warning.log" if warning_file.exists():