From 63aaa80695234eca0ea9a7f2de19b268811880d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20SI=C4=9EIRCI?= Date: Thu, 15 Aug 2024 16:28:03 +0300 Subject: [PATCH 1/4] ruff upgraded --- pyproject.toml | 10 ++++++---- requirements-dev.lock | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 96a14ad..53120e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [] managed = true dev-dependencies = [ "pre-commit", - "ruff==0.5.7", + "ruff==0.6.0", "mypy", "scalene~=1.5.21.2", ## DOCS @@ -94,9 +94,7 @@ line-length = 88 src = ["src"] respect-gitignore = true -extend-include = [ - "*.ipynb" -] +# extend-include = [] extend-exclude = [ "docs", @@ -106,6 +104,8 @@ extend-exclude = [ # Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`. [tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401"] +# "*.ipynb" = ["E501"] # disable line-too-long in notebooks + # "path/to/file.py" = ["E402"] # 'python_template/__init__.py' = ['F405', 'F403'] @@ -137,8 +137,10 @@ select = [ "RET", # return "RUF", # Enable all ruff-specific checks "SIM", # simplify + "S307", # eval "T20", # (disallow print statements) keep debugging statements out of the codebase "W", # pycodestyle warnings + "ASYNC" # async ] ignore = [ diff --git a/requirements-dev.lock b/requirements-dev.lock index 32444cc..4c84c85 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -161,7 +161,7 @@ pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via ghp-import # via jupyter-client -pyyaml==6.0.1 +pyyaml==6.0.2 # via mkdocs # via mkdocs-get-deps # via pre-commit @@ -178,7 +178,7 @@ requests==2.31.0 # via mkdocs-material rich==13.7.1 # via scalene -ruff==0.5.7 +ruff==0.6.0 scalene==1.5.21.4 setuptools==69.5.1 # via nodeenv From c52fa152a55328387b194ffb45ab582a687df7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20SI=C4=9EIRCI?= Date: Fri, 4 Oct 2024 23:25:21 +0300 Subject: [PATCH 2/4] ruff updated --- pyproject.toml | 2 +- requirements-dev.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53120e5..c5037d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [] managed = true dev-dependencies = [ "pre-commit", - "ruff==0.6.0", + "ruff==0.6.9", "mypy", "scalene~=1.5.21.2", ## DOCS diff --git a/requirements-dev.lock b/requirements-dev.lock index 4c84c85..a5a5a6d 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -178,7 +178,7 @@ requests==2.31.0 # via mkdocs-material rich==13.7.1 # via scalene -ruff==0.6.0 +ruff==0.6.9 scalene==1.5.21.4 setuptools==69.5.1 # via nodeenv From f7c78b333420073ff8e3d3c9b43ff855a380fd24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20SI=C4=9EIRCI?= Date: Fri, 4 Oct 2024 23:26:02 +0300 Subject: [PATCH 3/4] github actions updated --- .github/workflows/doc.yaml | 6 +++--- .github/workflows/docker_push.yaml | 2 +- .github/workflows/lint.yaml | 6 +++--- .github/workflows/test.yaml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/doc.yaml b/.github/workflows/doc.yaml index d85bcde..d2ebfce 100644 --- a/.github/workflows/doc.yaml +++ b/.github/workflows/doc.yaml @@ -13,11 +13,11 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version-file: ".python-version" - name: Set up cache uses: actions/cache@v2 with: diff --git a/.github/workflows/docker_push.yaml b/.github/workflows/docker_push.yaml index 4fb9514..c3913d9 100644 --- a/.github/workflows/docker_push.yaml +++ b/.github/workflows/docker_push.yaml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # https://github.com/marketplace/actions/push-to-ghcr - name: Build and publish a Docker image for ${{ github.repository }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a6efbc9..2b6159d 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -14,11 +14,11 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version-file: ".python-version" - name: Pip Update run: | make -s update-pip diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5641b67..b899e4c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,7 +12,7 @@ jobs: # test: # runs-on: ubuntu-22.04 # steps: - # - uses: actions/checkout@v3 + # - uses: actions/checkout@v4 # - name: Disable Logger Outputs # run: | # sed -i "s/log_cli = true/log_cli = false/" pyproject.toml @@ -35,7 +35,7 @@ jobs: os: [ubuntu-22.04] # windows-latest runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Disable Logger Outputs run: | sed -i "s/log_cli = true/log_cli = false/" pyproject.toml From 0e1c1ab3f0ce0f600afc2627a8375228371e6c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0lker=20SI=C4=9EIRCI?= Date: Mon, 7 Oct 2024 23:24:04 +0300 Subject: [PATCH 4/4] custom loggers and configs added --- src/python_template/loggers/__init__.py | 1 + .../loggers/configs/__init__.py | 1 + .../loggers/configs/advanced.py | 119 ++++++++++++++++++ .../loggers/configs/default.py | 72 +++++++++++ .../loggers/configs/uvicorn.py | 53 ++++++++ src/python_template/loggers/filters.py | 105 ++++++++++++++++ src/python_template/loggers/formatters.py | 29 +++++ src/python_template/loggers/setup.py | 43 +++++++ src/python_template/utils/__init__.py | 1 + src/python_template/utils/general.py | 56 +++++++++ 10 files changed, 480 insertions(+) create mode 100644 src/python_template/loggers/__init__.py create mode 100644 src/python_template/loggers/configs/__init__.py create mode 100644 src/python_template/loggers/configs/advanced.py create mode 100644 src/python_template/loggers/configs/default.py create mode 100644 src/python_template/loggers/configs/uvicorn.py create mode 100644 src/python_template/loggers/filters.py create mode 100644 src/python_template/loggers/formatters.py create mode 100644 src/python_template/loggers/setup.py create mode 100644 src/python_template/utils/__init__.py create mode 100644 src/python_template/utils/general.py diff --git a/src/python_template/loggers/__init__.py b/src/python_template/loggers/__init__.py new file mode 100644 index 0000000..1f59b0b --- /dev/null +++ b/src/python_template/loggers/__init__.py @@ -0,0 +1 @@ +"""Custom loggers.""" diff --git a/src/python_template/loggers/configs/__init__.py b/src/python_template/loggers/configs/__init__.py new file mode 100644 index 0000000..6b29c76 --- /dev/null +++ b/src/python_template/loggers/configs/__init__.py @@ -0,0 +1 @@ +"""Loggers configs.""" diff --git a/src/python_template/loggers/configs/advanced.py b/src/python_template/loggers/configs/advanced.py new file mode 100644 index 0000000..9936fe1 --- /dev/null +++ b/src/python_template/loggers/configs/advanced.py @@ -0,0 +1,119 @@ +"""Advanced logging configuration.""" + +ADVANCED_LOGGER_CONFIG = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "standard": {"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"}, + "error": { + "format": "%(asctime)s - %(name)s - %(levelname)s %(name)s.%(funcName)s(): %(message)s" + }, + }, + "handlers": { + "info_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "standard", + "filename": "logs/info.log", + "maxBytes": 10485760, # 10MB + "backupCount": 20, + "encoding": "utf8", + }, + "warn_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "WARN", + "formatter": "standard", + "filename": "logs/warn.log", + "maxBytes": 10485760, # 10MB + "backupCount": 20, + "encoding": "utf8", + }, + "error_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "ERROR", + "formatter": "error", + "filename": "logs/errors.log", + "maxBytes": 10485760, # 10MB + "backupCount": 20, + "encoding": "utf8", + }, + "critical_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "CRITICAL", + "formatter": "standard", + "filename": "logs/critical.log", + "maxBytes": 10485760, # 10MB + "backupCount": 20, + "encoding": "utf8", + }, + "debug_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "DEBUG", + "formatter": "standard", + "filename": "logs/debug.log", + "maxBytes": 10485760, # 10MB + "backupCount": 20, + "encoding": "utf8", + }, + "root_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "DEBUG", + "formatter": "standard", + "filename": "logs/logs.log", + "maxBytes": 10485760, # 10MB + "backupCount": 20, + "encoding": "utf8", + }, + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "standard", + "stream": "ext://sys.stdout", + }, + "error_console": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "error", + }, + }, + "loggers": { + "": { # root logger + "level": "DEBUG", + "handlers": ["console", "error_console", "root_file_handler"], + "propagate": True, + }, + "main": { + "level": "DEBUG", + "handlers": [ + "info_file_handler", + "warn_file_handler", + "error_file_handler", + "critical_file_handler", + "debug_file_handler", + ], + "propagate": False, + }, + "werkzeug": { + "level": "DEBUG", + "handlers": [ + "info_file_handler", + "warn_file_handler", + "error_file_handler", + "critical_file_handler", + "debug_file_handler", + ], + "propagate": True, + }, + "api.app_server": { + "level": "DEBUG", + "handlers": [ + "info_file_handler", + "warn_file_handler", + "error_file_handler", + "critical_file_handler", + "debug_file_handler", + ], + "propagate": True, + }, + }, +} diff --git a/src/python_template/loggers/configs/default.py b/src/python_template/loggers/configs/default.py new file mode 100644 index 0000000..cdcfd3c --- /dev/null +++ b/src/python_template/loggers/configs/default.py @@ -0,0 +1,72 @@ +"""Default logging configuration.""" + +DEFAULT_LOGGER_CONFIG = { + "version": 1, + "disable_existing_loggers": False, # NOTE: Very important + "formatters": { + "simple": {"format": "%(asctime)s :: %(name)s :: %(message)s"}, + "extended": { + "format": "%(asctime)-20s :: %(levelname)-8s :: [%(process)d]%(processName)s :: %(threadName)s[%(thread)d] :: %(pathname)s:%(lineno)d - %(funcName)s :: %(message)s" + }, + "aligned": { + "format": "{asctime} :: {levelname:<8s}:: {pathname:<10s}:{lineno} :: {message}", + "style": "{", + }, + "base": { + "format": "%(asctime)-20s :: %(levelname)-8s :: %(pathname)s:%(lineno)d:: %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S", + }, + "colored": { + "()": "python_template.loggers.formatters.ColoredFormatter", + "format": "%(asctime)-20s :: %(name)-8s :: %(levelname)-8s :: %(pathname)s:%(lineno)d :: %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S", + }, + }, + "filters": { + "info": { + "()": "python_template.loggers.filters.InfoFilter", + }, + "cwd": { + "()": "python_template.loggers.filters.CwdFilter", + }, + # "path_shortener": { + # "()": "python_template.loggers.filters.PathShortenerFilter", + # } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "stream": "ext://sys.stdout", + "formatter": "colored", + "filters": ["cwd"], + }, + "file_handler": { + "class": "logging.FileHandler", + "level": "INFO", + "filename": "python_template.log", + "formatter": "base", + "filters": ["cwd"], + }, + }, + "loggers": { + # NOTE: Default root logger parameters. It is not recommended to + # modify root logger. + "": { # root logger + # "level": "NOTSET", # logs everything + "level": "WARNING", + # "handlers": ["console"], + "propagate": True, + }, + "__main__": { # if __name__ == '__main__' + "level": "DEBUG", + "handlers": ["console"], + "propagate": True, # Inherit root handlers + }, + "python_template": { + "level": "DEBUG", + "handlers": ["console"], # ,file_handler + "propagate": True, # Inherit root handlers + }, + }, +} diff --git a/src/python_template/loggers/configs/uvicorn.py b/src/python_template/loggers/configs/uvicorn.py new file mode 100644 index 0000000..f2e50e8 --- /dev/null +++ b/src/python_template/loggers/configs/uvicorn.py @@ -0,0 +1,53 @@ +"""Uvicorn logging configuration.""" + +from typing import Any + +LOGGING_CONFIG: dict[str, Any] = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "()": "uvicorn.logging.DefaultFormatter", + "fmt": "%(asctime)s - %(name)s - %(levelprefix)s %(message)s", + "use_colors": None, + }, + "access": { + "()": "uvicorn.logging.AccessFormatter", + "fmt": '%(asctime)s - %(name)s - %(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', + }, + "access_file": { + "()": "uvicorn.logging.AccessFormatter", + "fmt": '%(asctime)s - %(name)s - %(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', + "use_colors": False, + }, + }, + "handlers": { + "file_handler": { + "formatter": "access_file", + "class": "logging.handlers.RotatingFileHandler", + "filename": "./app.log", + "mode": "a+", + "maxBytes": 10 * 1024 * 1024, + "backupCount": 0, + }, + "default": { + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + }, + "access": { + "formatter": "access", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + }, + }, + "loggers": { + "uvicorn": {"handlers": ["default"], "level": "INFO", "propagate": False}, + "uvicorn.error": {"level": "INFO"}, + "uvicorn.access": { + "handlers": ["access", "file_handler"], + "level": "INFO", + "propagate": False, + }, + }, +} diff --git a/src/python_template/loggers/filters.py b/src/python_template/loggers/filters.py new file mode 100644 index 0000000..2412f9c --- /dev/null +++ b/src/python_template/loggers/filters.py @@ -0,0 +1,105 @@ +"""Filters for logging.""" + +import logging +import threading +from pathlib import Path +from typing import Optional + +from python_template.utils.general import is_module_installed + +PATHNAME_FIELD = "pathname" +PATHNAME_MAX_LENGTH = 30 +NEST_COUNT = 2 + + +class InfoFilter(logging.Filter): + """This filter only shows log entries for INFO level.""" + + def filter(self, record: logging.LogRecord) -> bool: + """Filter function.""" + return record.levelno == logging.INFO + + +class SimpleThreadFilter(logging.Filter): + """This filter only shows log entries for specified thread name. + + Args: + thread_name: Name of the thread that will be filtered. + """ + + def __init__(self, thread_name: str): + self.thread_name = thread_name + super().__init__() + + def filter(self, record: logging.LogRecord) -> bool: + """Filter function.""" + return record.threadName == self.thread_name + + +class ThreadFilter(logging.Filter): + """Only accept log records from a specific thread or thread name. + + Args: + thread_id: Id of the thread that will be filtered. + thread_name: Name of the thread that will be filtered. + + Raises: + ValueError: Occurs when `thread_id` and/or `thread_id` not given. + """ + + def __init__( + self, thread_id: Optional[int] = None, thread_name: Optional[str] = None + ) -> None: + if thread_id is None and thread_name is None: + raise ValueError("Must specify either thread_id or thread_name") + + self._thread_id = thread_id + self._thread_name = thread_name + + def filter(self, record: logging.LogRecord) -> bool: + """Filter function.""" + if self._thread_id is not None and record.thread != self._thread_id: + return False + + return not ( + self._thread_name is not None and record.threadName != self._thread_name + ) + + +class IgnoreThreadsFilter(logging.Filter): + """Only accepts log records that originated from the main thread. + + Attributes: + _main_thread_id: Id of the main thread. + """ + + def __init__(self) -> None: + self._main_thread_id = threading.main_thread().ident + + def filter(self, record: logging.LogRecord) -> bool: + """Filter function.""" + return record.thread == self._main_thread_id + + +class RemoveColorFilter(logging.Filter): + """Remove color filter.""" + + def filter(self, record: logging.LogRecord) -> bool: + """Filter function.""" + is_module_installed(module_name="click", throw_error=True) + import click + + if record and record.msg and isinstance(record.msg, str): + record.msg = click.unstyle(record.msg) + + return True + + +class CwdFilter(logging.Filter): + """Filter that removes cwd from pathname.""" + + def filter(self, record: logging.LogRecord) -> bool: + """Filter function.""" + record.pathname = record.pathname.replace(str(Path.cwd()), "") + + return True diff --git a/src/python_template/loggers/formatters.py b/src/python_template/loggers/formatters.py new file mode 100644 index 0000000..86bc40a --- /dev/null +++ b/src/python_template/loggers/formatters.py @@ -0,0 +1,29 @@ +"""Formatters for the logger.""" + +import logging +from enum import Enum + + +class _TerminalLogColor(str, Enum): + DEBUG = "94m" + INFO = "92m" + WARNING = "93m" + ERROR = "91m" + CRITICAL = "95m" + + +class ColoredFormatter(logging.Formatter): + """Formatter for colored logs. + + It uses ansi escape codes to color the logs + in the terminal. + """ + + def format(self, record: logging.LogRecord) -> str: + """Format the log record.""" + log_level = record.levelname + + log_color = _TerminalLogColor[log_level].value + formatted_record = super().format(record) + + return f"\033[{log_color}{formatted_record}\033[0m" diff --git a/src/python_template/loggers/setup.py b/src/python_template/loggers/setup.py new file mode 100644 index 0000000..df0cf02 --- /dev/null +++ b/src/python_template/loggers/setup.py @@ -0,0 +1,43 @@ +"""Logger initialization methods.""" + +import logging +import logging.config + +from python_template.loggers.configs.default import DEFAULT_LOGGER_CONFIG + +logger = logging.getLogger(__name__) + + +def setup_logging( + logging_config: dict | None = None, + default_level: int = logging.INFO, +) -> None: + """Set up the logger using default or custom configuration. + + Args: + logging_config: Custom logging configuration. + default_level: Default level of the logger. + """ + loaded_config = DEFAULT_LOGGER_CONFIG if logging_config is None else logging_config + + try: + logging.config.dictConfig(loaded_config) + except Exception as e: + message = f"Error when loading given logging configuration. Using default configs. Error: {e}" + print(message) # noqa: T201 + logging.basicConfig(level=default_level) + + +if __name__ == "__main__": + setup_logging(default_level=logging.DEBUG) + + logger.debug("This is a debug message") + logger.info("This is an info message") + logger.warning("This is a warning message") + logger.error("This is an error message") + logger.critical("This is a critical message") + + try: + print(1 / 0) # noqa: T201 + except Exception: + logger.exception("unable print!") diff --git a/src/python_template/utils/__init__.py b/src/python_template/utils/__init__.py new file mode 100644 index 0000000..f5d3dc7 --- /dev/null +++ b/src/python_template/utils/__init__.py @@ -0,0 +1 @@ +"""Utility functions.""" diff --git a/src/python_template/utils/general.py b/src/python_template/utils/general.py new file mode 100644 index 0000000..82e8604 --- /dev/null +++ b/src/python_template/utils/general.py @@ -0,0 +1,56 @@ +"""General utility functions.""" + +import importlib +import logging +import os + +logger = logging.getLogger(__name__) + + +def check_env_vars(env_vars: list[str] | None = None) -> None: + """Checks if the required environment variables are set. + + Args: + env_vars: List of environment variables to check. Defaults to None. + + Raises: + ValueError: If any of the environment variables are not set. + """ + if env_vars is None: + return + + for env_var in env_vars: + if os.getenv(env_var) is None: + raise ValueError(f"Please set {env_var} env var.") + + +def is_module_installed(module_name: str, throw_error: bool = False) -> bool: + """Check if the module is installed or not. + + Examples: + >>> is_module_installed(module_name="yaml", throw_error=False) + True + >>> is_module_installed(module_name="numpy", throw_error=False) + False + >>> is_module_installed(module_name="numpy", throw_error=True) + Traceback (most recent call last): + ImportError: Module numpy is not installed. + + Args: + module_name: Name of the module to be checked. + throw_error: If True, raises ImportError if module is not installed. + + Returns: + Returns True if module is installed, False otherwise. + + Raises: + ImportError: If throw_error is True and module is not installed. + """ + try: + importlib.import_module(module_name) + return True + except ImportError as e: + if throw_error: + message = f"Module {module_name} is not installed." + raise ImportError(message) from e + return False