From d24f8f31e021127b44d20f5f33bc8b6891c479a2 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 16:32:53 +0100 Subject: [PATCH 01/11] update config with config file --- config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config.py b/config.py index dc72fb2..88d76cf 100644 --- a/config.py +++ b/config.py @@ -63,3 +63,13 @@ def validate_config(config): def get_ignored_tables(connection): return connection.skip_tables if "skip_tables" in connection else [] + + +def update_config_path(path_param: str) -> None: + config_file_path = pathlib.Path(path_param).absolute() + + if config_file_path.exists(): + print(f"== Using {path_param} config file ==") + config.settings_files = [config_file_path.absolute()] + else: + raise IOError(f"Config file {config_file_path} does not exist.") From cff5b283b317be406933c678aec79fe2da035842 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 16:34:23 +0100 Subject: [PATCH 02/11] use argparser to handle arguments, allow config file to be passed and two new flags for the tool --- dbsync_daemon.py | 71 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 16b4310..9297f35 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -7,13 +7,31 @@ import datetime import sys import time +import argparse import dbsync from version import __version__ -from config import config, validate_config, ConfigError +from config import config, validate_config, ConfigError, update_config_path def main(): + + parser = argparse.ArgumentParser(prog='dbsync_deamon.py', + description='Synchronization tool between Mergin Maps project and database.', + epilog='www.merginmaps.com') + + parser.add_argument("config_file", nargs="?", default="config.yaml", help="Path to file with configuration. Default value is config.yaml in current working directory.") + parser.add_argument("--skip-init", action="store_true", help="Skip DB Sync init step.") + parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") + + args = parser.parse_args() + + try: + update_config_path(args.config_file) + except IOError as e: + print("Error: " + str(e)) + return + print(f"== starting mergin-db-sync daemon == version {__version__} ==") sleep_time = config.as_int("daemon.sleep_time") @@ -26,12 +44,14 @@ def main(): print("Logging in to Mergin...") mc = dbsync.create_mergin_client() - # run initialization before starting the sync loop - dbsync.dbsync_init(mc) + if args.single_run: - while True: - - print(datetime.datetime.now()) + if not args.skip_init: + try: + dbsync.dbsync_init(mc) + except dbsync.DbSyncError as e: + print("Error: " + str(e)) + return try: print("Trying to pull") @@ -39,17 +59,40 @@ def main(): print("Trying to push") dbsync.dbsync_push(mc) - - # check mergin client token expiration - delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc) - if delta.total_seconds() < 3600: - mc = dbsync.create_mergin_client() - except dbsync.DbSyncError as e: print("Error: " + str(e)) + return + + else: + + if not args.skip_init: + try: + dbsync.dbsync_init(mc) + except dbsync.DbSyncError as e: + print("Error: " + str(e)) + return + + while True: + + print(datetime.datetime.now()) + + try: + print("Trying to pull") + dbsync.dbsync_pull(mc) + + print("Trying to push") + dbsync.dbsync_push(mc) + + # check mergin client token expiration + delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc) + if delta.total_seconds() < 3600: + mc = dbsync.create_mergin_client() + + except dbsync.DbSyncError as e: + print("Error: " + str(e)) - print("Going to sleep") - time.sleep(sleep_time) + print("Going to sleep") + time.sleep(sleep_time) if __name__ == '__main__': From 5b3b61ed4920e1e260e68cad83327837a6da224f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 16:42:23 +0100 Subject: [PATCH 03/11] print errors to sys.stderr --- dbsync_daemon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 9297f35..c7d0e0a 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -50,7 +50,7 @@ def main(): try: dbsync.dbsync_init(mc) except dbsync.DbSyncError as e: - print("Error: " + str(e)) + print("Error: " + str(e), file=sys.stderr) return try: @@ -60,7 +60,7 @@ def main(): print("Trying to push") dbsync.dbsync_push(mc) except dbsync.DbSyncError as e: - print("Error: " + str(e)) + print("Error: " + str(e), file=sys.stderr) return else: @@ -69,7 +69,7 @@ def main(): try: dbsync.dbsync_init(mc) except dbsync.DbSyncError as e: - print("Error: " + str(e)) + print("Error: " + str(e), file=sys.stderr) return while True: @@ -89,7 +89,7 @@ def main(): mc = dbsync.create_mergin_client() except dbsync.DbSyncError as e: - print("Error: " + str(e)) + print("Error: " + str(e), file=sys.stderr) print("Going to sleep") time.sleep(sleep_time) From ace27abe2cc6d9b95bdb1d2cdebe8278a8d15b70 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:16:16 +0100 Subject: [PATCH 04/11] fix loading specified file --- config.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 88d76cf..60e0a5e 100644 --- a/config.py +++ b/config.py @@ -14,7 +14,7 @@ config = Dynaconf( envvar_prefix=False, - settings_files=['config.yaml'], + settings_files=[], geodiff_exe="geodiff.exe" if platform.system() == "Windows" else "geodiff", working_dir=(pathlib.Path(tempfile.gettempdir()) / "dbsync").as_posix() ) @@ -45,6 +45,7 @@ def validate_config(config): if config.init_from not in ["gpkg", "db"]: raise ConfigError(f"Config error: `init_from` parameter must be either `gpkg` or `db`. Current value is `{config.init_from}`.") + i = 0 for conn in config.connections: for attr in ["driver", "conn_info", "modified", "base", "mergin_project", "sync_file"]: if not hasattr(conn, attr): @@ -57,9 +58,14 @@ def validate_config(config): raise ConfigError("Config error: Name of the Mergin Maps project should be provided in the namespace/name format.") if "skip_tables" in conn: + if conn.skip_tables is None: + pass + # config.update({'CONNECTIONS': [{"modified": "mergin_main"}]} {"skip_tables": []}) + if isinstance(conn.skip_tables, str): + conn.update({"skip_tables": [conn.skip_tables]}) if not isinstance(conn.skip_tables, list): raise ConfigError("Config error: Ignored tables parameter should be a list") - + i +=1 def get_ignored_tables(connection): return connection.skip_tables if "skip_tables" in connection else [] @@ -70,6 +76,8 @@ def update_config_path(path_param: str) -> None: if config_file_path.exists(): print(f"== Using {path_param} config file ==") - config.settings_files = [config_file_path.absolute()] + user_file_config = Dynaconf(envvar_prefix=False, + settings_files=[config_file_path.absolute()]) + config.update(user_file_config) else: raise IOError(f"Config file {config_file_path} does not exist.") From b1939fe319d1225cecdd31c7354d352abe97c231 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:36:44 +0100 Subject: [PATCH 05/11] let None, str or list pass the validation --- config.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index 60e0a5e..7290ad1 100644 --- a/config.py +++ b/config.py @@ -45,7 +45,6 @@ def validate_config(config): if config.init_from not in ["gpkg", "db"]: raise ConfigError(f"Config error: `init_from` parameter must be either `gpkg` or `db`. Current value is `{config.init_from}`.") - i = 0 for conn in config.connections: for attr in ["driver", "conn_info", "modified", "base", "mergin_project", "sync_file"]: if not hasattr(conn, attr): @@ -59,13 +58,12 @@ def validate_config(config): if "skip_tables" in conn: if conn.skip_tables is None: - pass - # config.update({'CONNECTIONS': [{"modified": "mergin_main"}]} {"skip_tables": []}) - if isinstance(conn.skip_tables, str): - conn.update({"skip_tables": [conn.skip_tables]}) - if not isinstance(conn.skip_tables, list): + continue + elif isinstance(conn.skip_tables, str): + continue + elif not isinstance(conn.skip_tables, list): raise ConfigError("Config error: Ignored tables parameter should be a list") - i +=1 + def get_ignored_tables(connection): return connection.skip_tables if "skip_tables" in connection else [] From ef79940a71417a67ddedd877569ee65fe88796c1 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:37:13 +0100 Subject: [PATCH 06/11] for various allowed inputs return correct skip table representation --- config.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index 7290ad1..d4e6d33 100644 --- a/config.py +++ b/config.py @@ -66,7 +66,15 @@ def validate_config(config): def get_ignored_tables(connection): - return connection.skip_tables if "skip_tables" in connection else [] + if "skip_tables" in connection: + if connection.skip_tables is None: + return [] + elif isinstance(connection.skip_tables, str): + return [connection.skip_tables] + elif isinstance(connection.skip_tables, list): + return connection.skip_tables + else: + return [] def update_config_path(path_param: str) -> None: From 1fb2d3cf666c20670e6b962db6f81ba431025776 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:45:26 +0100 Subject: [PATCH 07/11] add tests for skip_tables --- test/test_config.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/test_config.py b/test/test_config.py index 6c34a69..9f90441 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -8,7 +8,7 @@ import os import pytest -from config import config, ConfigError, validate_config +from config import config, ConfigError, validate_config, get_ignored_tables SERVER_URL = os.environ.get('TEST_MERGIN_URL') API_USER = os.environ.get('TEST_API_USERNAME') @@ -64,3 +64,40 @@ def test_config(): with pytest.raises(ConfigError, match="Config error: Name of the Mergin Maps project should be provided in the namespace/name format."): config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]}) validate_config(config) + + +def test_skip_tables(): + _reset_config() + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": None}]}) + validate_config(config) + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": []}]}) + validate_config(config) + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": "table"}]}) + validate_config(config) + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": ["table"]}]}) + + +def test_get_ignored_tables(): + _reset_config() + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": None}]}) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == [] + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": []}]}) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == [] + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": "table"}]}) + validate_config(config) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == ["table"] + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": ["table"]}]}) + validate_config(config) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == ["table"] From 8a01684d7a622e6e623d60f9708e96393fdb7d09 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:51:25 +0100 Subject: [PATCH 08/11] Update text Co-authored-by: Martin Dobias --- dbsync_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index c7d0e0a..ff258cc 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -21,7 +21,7 @@ def main(): epilog='www.merginmaps.com') parser.add_argument("config_file", nargs="?", default="config.yaml", help="Path to file with configuration. Default value is config.yaml in current working directory.") - parser.add_argument("--skip-init", action="store_true", help="Skip DB Sync init step.") + parser.add_argument("--skip-init", action="store_true", help="Skip DB sync init step to make the tool start faster. It is not recommend to use it unless you are really sure you can skip the initial sanity checks.") parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") args = parser.parse_args() From 80514679d41b61694e678b2e79f1e81e30d94362 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:50:32 +0100 Subject: [PATCH 09/11] no need for absolute paths here --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index d4e6d33..d334e62 100644 --- a/config.py +++ b/config.py @@ -78,12 +78,12 @@ def get_ignored_tables(connection): def update_config_path(path_param: str) -> None: - config_file_path = pathlib.Path(path_param).absolute() + config_file_path = pathlib.Path(path_param) if config_file_path.exists(): print(f"== Using {path_param} config file ==") user_file_config = Dynaconf(envvar_prefix=False, - settings_files=[config_file_path.absolute()]) + settings_files=[config_file_path]) config.update(user_file_config) else: raise IOError(f"Config file {config_file_path} does not exist.") From edc7ad3e8ab1093f6d9106d02227fbbf0d961374 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:55:25 +0100 Subject: [PATCH 10/11] unless in while loop exit with error code --- dbsync_daemon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbsync_daemon.py b/dbsync_daemon.py index ff258cc..de3e178 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -30,7 +30,7 @@ def main(): update_config_path(args.config_file) except IOError as e: print("Error: " + str(e)) - return + exit(1) print(f"== starting mergin-db-sync daemon == version {__version__} ==") @@ -39,7 +39,7 @@ def main(): validate_config(config) except ConfigError as e: print("Error: " + str(e)) - return + exit(1) print("Logging in to Mergin...") mc = dbsync.create_mergin_client() @@ -51,7 +51,7 @@ def main(): dbsync.dbsync_init(mc) except dbsync.DbSyncError as e: print("Error: " + str(e), file=sys.stderr) - return + exit(1) try: print("Trying to pull") @@ -61,7 +61,7 @@ def main(): dbsync.dbsync_push(mc) except dbsync.DbSyncError as e: print("Error: " + str(e), file=sys.stderr) - return + exit(1) else: @@ -70,7 +70,7 @@ def main(): dbsync.dbsync_init(mc) except dbsync.DbSyncError as e: print("Error: " + str(e), file=sys.stderr) - return + exit(1) while True: From abbbee69d08eeca229da412cbd6889443bb9a558 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 17:55:54 +0100 Subject: [PATCH 11/11] remove code that llows dbsync.py to be run as CLI tool --- dbsync.py | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/dbsync.py b/dbsync.py index 20a6a1d..d51fbc0 100644 --- a/dbsync.py +++ b/dbsync.py @@ -801,51 +801,3 @@ def dbsync_push(mc): def dbsync_status(mc): for conn in config.connections: status(conn, mc) - - -def show_usage(): - print("dbsync") - print("") - print(" dbsync init-from-db = will create base schema in DB + create gpkg file in working copy") - print(" dbsync init-from-gpkg = will create base and main schema in DB from gpkg file in working copy") - print(" dbsync status = will check whether there is anything to pull or push") - print(" dbsync push = will push changes from DB to Mergin Maps") - print(" dbsync pull = will pull changes from Mergin Maps to DB") - - -def main(): - if len(sys.argv) < 2: - show_usage() - return - - print(f"== Starting Mergin Maps DB Sync version {__version__} ==") - - try: - validate_config(config) - except ConfigError as e: - print("Error: " + str(e)) - return - - try: - print("Logging in to Mergin Maps...") - mc = create_mergin_client() - - if sys.argv[1] == 'init': - print("Initializing ...") - dbsync_init(mc) - elif sys.argv[1] == 'status': - dbsync_status(mc) - elif sys.argv[1] == 'push': - print("Pushing...") - dbsync_push(mc) - elif sys.argv[1] == 'pull': - print("Pulling...") - dbsync_pull(mc) - else: - show_usage() - except DbSyncError as e: - print("Error: " + str(e)) - - -if __name__ == '__main__': - main()