From 32bef1385116b4a01cc4de5e87736e3670ce6ef1 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 8 May 2023 15:36:03 +0200 Subject: [PATCH 1/6] windows functions --- dbsync_daemon.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index b502540..2f87e87 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -8,14 +8,35 @@ import sys import time import argparse +import platform +import os +import pathlib import dbsync from version import __version__ from config import config, validate_config, ConfigError, update_config_path +def is_pyinstaller() -> bool: + if getattr(sys, 'frozen', False) and platform.system() == "Windows": + return True + return False + + +def pyinstaller_update_path() -> None: + path = pathlib.Path(__file__).parent / "lib" + os.environ["PATH"] += os.pathsep + path.as_posix() + + +def pyinstaller_path_fix() -> None: + if is_pyinstaller(): + pyinstaller_update_path() + + def main(): + pyinstaller_path_fix() + parser = argparse.ArgumentParser(prog='dbsync_deamon.py', description='Synchronization tool between Mergin Maps project and database.', epilog='www.merginmaps.com') @@ -23,6 +44,7 @@ def main(): parser.add_argument("config_file", nargs="?", default="config.yaml", help="Path to file with configuration. Default value is config.yaml in current working directory.") parser.add_argument("--skip-init", action="store_true", help="Skip DB sync init step to make the tool start faster. It is not recommend to use it unless you are really sure you can skip the initial sanity checks.") parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") + parser.add_argument("--force-init", action="store_true", help="Force removing working directory and schemas from DB to initialize from scratch.") args = parser.parse_args() @@ -41,9 +63,16 @@ def main(): print("Error: " + str(e), file=sys.stderr) sys.exit(1) + if args.force_init and args.skip_init: + print("Cannot use `--force-init` with `--skip-init` Initialization is required. ", file=sys.stderr) + sys.exit(1) + print("Logging in to Mergin...") mc = dbsync.create_mergin_client() + if args.force_init: + dbsync.dbsync_clean(mc) + if args.single_run: if not args.skip_init: From 5e1967137b798be69e1dc5dd03e5c8196acbedbe Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 8 May 2023 15:36:30 +0200 Subject: [PATCH 2/6] script to build the windows version --- scripts/build_exe.bat | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 scripts/build_exe.bat diff --git a/scripts/build_exe.bat b/scripts/build_exe.bat new file mode 100755 index 0000000..4c0082a --- /dev/null +++ b/scripts/build_exe.bat @@ -0,0 +1,13 @@ +pyinstaller ../dbsync_daemon.py ^ + -c ^ + --noconfirm ^ + --add-binary="./windows_binaries/geodiff.exe;lib" ^ + --add-binary="./windows_binaries/geodiff.dll;lib" ^ + --add-binary="./windows_binaries/libcrypto-3-x64.dll;lib" ^ + --add-binary="./windows_binaries/LIBPQ.dll;lib" ^ + --add-binary="./windows_binaries/libssl-3-x64.dll;lib" ^ + --add-binary="./windows_binaries/sqlite3.dll;lib" ^ + --hidden-import dynaconf ^ + --collect-all mergin ^ + --clean ^ + -F \ No newline at end of file From 4f6cc40be9e03b62cf3c437fda1a5638f1246637 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 8 May 2023 17:19:46 +0200 Subject: [PATCH 3/6] simple logging --- dbsync_daemon.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 2f87e87..56b161c 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -9,6 +9,8 @@ import time import argparse import platform +import logging +import datetime import os import pathlib @@ -33,6 +35,18 @@ def pyinstaller_path_fix() -> None: pyinstaller_update_path() +def get_logger(log_path, with_time=True, with_level=True) -> logging.Logger: + log = logging.getLogger(f"{log_path}") + log.setLevel(logging.DEBUG) + if not log.handlers: + log_handler = logging.FileHandler(log_path, mode="a") + format = "%(asctime)s -" if with_time else "" + format += "%(levelname)s - %(message)s" if with_level else "%(message)s" + log_handler.setFormatter(logging.Formatter(format)) + log.addHandler(log_handler) + return log + + def main(): pyinstaller_path_fix() @@ -45,11 +59,19 @@ def main(): parser.add_argument("--skip-init", action="store_true", help="Skip DB sync init step to make the tool start faster. It is not recommend to use it unless you are really sure you can skip the initial sanity checks.") parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") parser.add_argument("--force-init", action="store_true", help="Force removing working directory and schemas from DB to initialize from scratch.") + parser.add_argument("--log-file", default="", action="store", help="Store logging to file.") args = parser.parse_args() print(f"== starting mergin-db-sync daemon == version {__version__} ==") + log_file: pathlib.Path = None + logger: logging.Logger = None + + if args.log_file: + log_file = pathlib.Path(args.log_file) + logger = get_logger(log_file.as_posix()) + try: update_config_path(args.config_file) except IOError as e: @@ -118,6 +140,8 @@ def main(): mc = dbsync.create_mergin_client() except dbsync.DbSyncError as e: + if logger: + logger.error(str(e)) print("Error: " + str(e), file=sys.stderr) print("Going to sleep") From a4d5c0e94408616d0c924981541cfc0b3a5910c8 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Tue, 9 May 2023 13:49:32 +0200 Subject: [PATCH 4/6] add logging --- dbsync_daemon.py | 80 ++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 56b161c..19e419e 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -10,9 +10,9 @@ import argparse import platform import logging -import datetime import os import pathlib +import typing import dbsync from version import __version__ @@ -35,16 +35,35 @@ def pyinstaller_path_fix() -> None: pyinstaller_update_path() -def get_logger(log_path, with_time=True, with_level=True) -> logging.Logger: - log = logging.getLogger(f"{log_path}") - log.setLevel(logging.DEBUG) - if not log.handlers: +LOGGER: logging.Logger = None + + +def setup_logger(log_path, log_messages: bool, with_time=True, with_level=True) -> logging.Logger: + global LOGGER + LOGGER = logging.getLogger(f"{log_path}") + if log_messages: + LOGGER.setLevel(logging.DEBUG) + else: + LOGGER.setLevel(logging.WARNING) + if not LOGGER.handlers: log_handler = logging.FileHandler(log_path, mode="a") format = "%(asctime)s -" if with_time else "" format += "%(levelname)s - %(message)s" if with_level else "%(message)s" log_handler.setFormatter(logging.Formatter(format)) - log.addHandler(log_handler) - return log + LOGGER.addHandler(log_handler) + + +def handle_error(error: typing.Union[Exception, str]): + if LOGGER: + LOGGER.error(str(error)) + print("Error: " + str(error), file=sys.stderr) + sys.exit(1) + + +def handle_message(msg: str): + if LOGGER: + LOGGER.debug(msg) + print(msg) def main(): @@ -60,36 +79,33 @@ def main(): parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") parser.add_argument("--force-init", action="store_true", help="Force removing working directory and schemas from DB to initialize from scratch.") parser.add_argument("--log-file", default="", action="store", help="Store logging to file.") + parser.add_argument("--log-messages", action="store_true", help="Log messages, not only errors.") args = parser.parse_args() - print(f"== starting mergin-db-sync daemon == version {__version__} ==") - - log_file: pathlib.Path = None - logger: logging.Logger = None - if args.log_file: log_file = pathlib.Path(args.log_file) - logger = get_logger(log_file.as_posix()) + setup_logger(log_file.as_posix(), args.log_messages) + + handle_message(f"== starting mergin-db-sync daemon == version {__version__} ==") try: update_config_path(args.config_file) except IOError as e: - print("Error: " + str(e), file=sys.stderr) - sys.exit(1) + handle_error(e) sleep_time = config.as_int("daemon.sleep_time") try: validate_config(config) except ConfigError as e: - print("Error: " + str(e), file=sys.stderr) - sys.exit(1) + handle_error(e) if args.force_init and args.skip_init: - print("Cannot use `--force-init` with `--skip-init` Initialization is required. ", file=sys.stderr) - sys.exit(1) + msg = "Cannot use `--force-init` with `--skip-init` Initialization is required. " + handle_error(msg) + + handle_message("Logging in to Mergin...") - print("Logging in to Mergin...") mc = dbsync.create_mergin_client() if args.force_init: @@ -101,18 +117,17 @@ def main(): try: dbsync.dbsync_init(mc) except dbsync.DbSyncError as e: - print("Error: " + str(e), file=sys.stderr) - sys.exit(1) + handle_error(e) try: - print("Trying to pull") + handle_message("Trying to pull") dbsync.dbsync_pull(mc) - print("Trying to push") + handle_message("Trying to push") dbsync.dbsync_push(mc) + except dbsync.DbSyncError as e: - print("Error: " + str(e), file=sys.stderr) - sys.exit(1) + handle_error(e) else: @@ -120,18 +135,17 @@ def main(): try: dbsync.dbsync_init(mc) except dbsync.DbSyncError as e: - print("Error: " + str(e), file=sys.stderr) - sys.exit(1) + handle_error(e) while True: print(datetime.datetime.now()) try: - print("Trying to pull") + handle_message("Trying to pull") dbsync.dbsync_pull(mc) - print("Trying to push") + handle_message("Trying to push") dbsync.dbsync_push(mc) # check mergin client token expiration @@ -140,11 +154,9 @@ def main(): mc = dbsync.create_mergin_client() except dbsync.DbSyncError as e: - if logger: - logger.error(str(e)) - print("Error: " + str(e), file=sys.stderr) + handle_error(e) - print("Going to sleep") + handle_message("Going to sleep") time.sleep(sleep_time) From 7abfa0c0d1b2773cb9ba9ea3ddfdb1bce5d1cae2 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Tue, 9 May 2023 13:57:54 +0200 Subject: [PATCH 5/6] simplify --- dbsync_daemon.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 19e419e..391d211 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -101,8 +101,7 @@ def main(): handle_error(e) if args.force_init and args.skip_init: - msg = "Cannot use `--force-init` with `--skip-init` Initialization is required. " - handle_error(msg) + handle_error("Cannot use `--force-init` with `--skip-init` Initialization is required. ") handle_message("Logging in to Mergin...") From 36b2b41a5cb5004e42f3e6247f8295590a04e389 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Tue, 9 May 2023 16:46:29 +0200 Subject: [PATCH 6/6] fix verbosity --- dbsync_daemon.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 391d211..8dd0b24 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -38,11 +38,13 @@ def pyinstaller_path_fix() -> None: LOGGER: logging.Logger = None -def setup_logger(log_path, log_messages: bool, with_time=True, with_level=True) -> logging.Logger: +def setup_logger(log_path, log_verbosity: str, with_time=True, with_level=True) -> logging.Logger: global LOGGER LOGGER = logging.getLogger(f"{log_path}") - if log_messages: + if log_verbosity == "messages": LOGGER.setLevel(logging.DEBUG) + elif log_verbosity == "errors": + LOGGER.setLevel(logging.WARNING) else: LOGGER.setLevel(logging.WARNING) if not LOGGER.handlers: @@ -79,13 +81,13 @@ def main(): parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") parser.add_argument("--force-init", action="store_true", help="Force removing working directory and schemas from DB to initialize from scratch.") parser.add_argument("--log-file", default="", action="store", help="Store logging to file.") - parser.add_argument("--log-messages", action="store_true", help="Log messages, not only errors.") + parser.add_argument("--log-verbosity", choices=["errors", "messages"], default="errors", help="Log messages, not only errors.") args = parser.parse_args() if args.log_file: log_file = pathlib.Path(args.log_file) - setup_logger(log_file.as_posix(), args.log_messages) + setup_logger(log_file.as_posix(), args.log_verbosity) handle_message(f"== starting mergin-db-sync daemon == version {__version__} ==")