diff --git a/config.py b/config.py index dc72fb2..d334e62 100644 --- a/config.py +++ b/config.py @@ -14,7 +14,7 @@ config = Dynaconf( envvar_prefix=False, - settings_files=['config.yaml'], + settings_files=[], geodiff_exe="geodiff.exe" if platform.system() == "Windows" else "geodiff", working_dir=(pathlib.Path(tempfile.gettempdir()) / "dbsync").as_posix() ) @@ -57,9 +57,33 @@ def validate_config(config): raise ConfigError("Config error: Name of the Mergin Maps project should be provided in the namespace/name format.") if "skip_tables" in conn: - if not isinstance(conn.skip_tables, list): + if conn.skip_tables is None: + continue + elif isinstance(conn.skip_tables, str): + continue + elif not isinstance(conn.skip_tables, list): raise ConfigError("Config error: Ignored tables parameter should be a list") def get_ignored_tables(connection): - return connection.skip_tables if "skip_tables" in connection else [] + if "skip_tables" in connection: + if connection.skip_tables is None: + return [] + elif isinstance(connection.skip_tables, str): + return [connection.skip_tables] + elif isinstance(connection.skip_tables, list): + return connection.skip_tables + else: + return [] + + +def update_config_path(path_param: str) -> None: + config_file_path = pathlib.Path(path_param) + + if config_file_path.exists(): + print(f"== Using {path_param} config file ==") + user_file_config = Dynaconf(envvar_prefix=False, + settings_files=[config_file_path]) + config.update(user_file_config) + else: + raise IOError(f"Config file {config_file_path} does not exist.") diff --git a/dbsync.py b/dbsync.py index 20a6a1d..d51fbc0 100644 --- a/dbsync.py +++ b/dbsync.py @@ -801,51 +801,3 @@ def dbsync_push(mc): def dbsync_status(mc): for conn in config.connections: status(conn, mc) - - -def show_usage(): - print("dbsync") - print("") - print(" dbsync init-from-db = will create base schema in DB + create gpkg file in working copy") - print(" dbsync init-from-gpkg = will create base and main schema in DB from gpkg file in working copy") - print(" dbsync status = will check whether there is anything to pull or push") - print(" dbsync push = will push changes from DB to Mergin Maps") - print(" dbsync pull = will pull changes from Mergin Maps to DB") - - -def main(): - if len(sys.argv) < 2: - show_usage() - return - - print(f"== Starting Mergin Maps DB Sync version {__version__} ==") - - try: - validate_config(config) - except ConfigError as e: - print("Error: " + str(e)) - return - - try: - print("Logging in to Mergin Maps...") - mc = create_mergin_client() - - if sys.argv[1] == 'init': - print("Initializing ...") - dbsync_init(mc) - elif sys.argv[1] == 'status': - dbsync_status(mc) - elif sys.argv[1] == 'push': - print("Pushing...") - dbsync_push(mc) - elif sys.argv[1] == 'pull': - print("Pulling...") - dbsync_pull(mc) - else: - show_usage() - except DbSyncError as e: - print("Error: " + str(e)) - - -if __name__ == '__main__': - main() diff --git a/dbsync_daemon.py b/dbsync_daemon.py index 16b4310..de3e178 100644 --- a/dbsync_daemon.py +++ b/dbsync_daemon.py @@ -7,13 +7,31 @@ import datetime import sys import time +import argparse import dbsync from version import __version__ -from config import config, validate_config, ConfigError +from config import config, validate_config, ConfigError, update_config_path def main(): + + parser = argparse.ArgumentParser(prog='dbsync_deamon.py', + description='Synchronization tool between Mergin Maps project and database.', + epilog='www.merginmaps.com') + + parser.add_argument("config_file", nargs="?", default="config.yaml", help="Path to file with configuration. Default value is config.yaml in current working directory.") + parser.add_argument("--skip-init", action="store_true", help="Skip DB sync init step to make the tool start faster. It is not recommend to use it unless you are really sure you can skip the initial sanity checks.") + parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.") + + args = parser.parse_args() + + try: + update_config_path(args.config_file) + except IOError as e: + print("Error: " + str(e)) + exit(1) + print(f"== starting mergin-db-sync daemon == version {__version__} ==") sleep_time = config.as_int("daemon.sleep_time") @@ -21,17 +39,19 @@ def main(): validate_config(config) except ConfigError as e: print("Error: " + str(e)) - return + exit(1) print("Logging in to Mergin...") mc = dbsync.create_mergin_client() - # run initialization before starting the sync loop - dbsync.dbsync_init(mc) - - while True: + if args.single_run: - print(datetime.datetime.now()) + if not args.skip_init: + try: + dbsync.dbsync_init(mc) + except dbsync.DbSyncError as e: + print("Error: " + str(e), file=sys.stderr) + exit(1) try: print("Trying to pull") @@ -39,17 +59,40 @@ def main(): print("Trying to push") dbsync.dbsync_push(mc) + except dbsync.DbSyncError as e: + print("Error: " + str(e), file=sys.stderr) + exit(1) - # check mergin client token expiration - delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc) - if delta.total_seconds() < 3600: - mc = dbsync.create_mergin_client() + else: - except dbsync.DbSyncError as e: - print("Error: " + str(e)) + if not args.skip_init: + try: + dbsync.dbsync_init(mc) + except dbsync.DbSyncError as e: + print("Error: " + str(e), file=sys.stderr) + exit(1) + + while True: + + print(datetime.datetime.now()) + + try: + print("Trying to pull") + dbsync.dbsync_pull(mc) + + print("Trying to push") + dbsync.dbsync_push(mc) + + # check mergin client token expiration + delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc) + if delta.total_seconds() < 3600: + mc = dbsync.create_mergin_client() + + except dbsync.DbSyncError as e: + print("Error: " + str(e), file=sys.stderr) - print("Going to sleep") - time.sleep(sleep_time) + print("Going to sleep") + time.sleep(sleep_time) if __name__ == '__main__': diff --git a/test/test_config.py b/test/test_config.py index 6c34a69..9f90441 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -8,7 +8,7 @@ import os import pytest -from config import config, ConfigError, validate_config +from config import config, ConfigError, validate_config, get_ignored_tables SERVER_URL = os.environ.get('TEST_MERGIN_URL') API_USER = os.environ.get('TEST_API_USERNAME') @@ -64,3 +64,40 @@ def test_config(): with pytest.raises(ConfigError, match="Config error: Name of the Mergin Maps project should be provided in the namespace/name format."): config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]}) validate_config(config) + + +def test_skip_tables(): + _reset_config() + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": None}]}) + validate_config(config) + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": []}]}) + validate_config(config) + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": "table"}]}) + validate_config(config) + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": ["table"]}]}) + + +def test_get_ignored_tables(): + _reset_config() + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": None}]}) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == [] + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": []}]}) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == [] + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": "table"}]}) + validate_config(config) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == ["table"] + + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": ["table"]}]}) + validate_config(config) + ignored_tables = get_ignored_tables(config.connections[0]) + assert ignored_tables == ["table"]