Update to dbsync_deamon

config.py

@@ -14,7 +14,7 @@
 
 config = Dynaconf(
     envvar_prefix=False,
-    settings_files=['config.yaml'],
+    settings_files=[],
     geodiff_exe="geodiff.exe" if platform.system() == "Windows" else "geodiff",
     working_dir=(pathlib.Path(tempfile.gettempdir()) / "dbsync").as_posix()
 )
@@ -57,9 +57,33 @@ def validate_config(config):
             raise ConfigError("Config error: Name of the Mergin Maps project should be provided in the namespace/name format.")
 
         if "skip_tables" in conn:
-            if not isinstance(conn.skip_tables, list):
+            if conn.skip_tables is None:
+                continue
+            elif isinstance(conn.skip_tables, str):
+                continue
+            elif not isinstance(conn.skip_tables, list):
                 raise ConfigError("Config error: Ignored tables parameter should be a list")
 
 
 def get_ignored_tables(connection):
-    return connection.skip_tables if "skip_tables" in connection else []
+    if "skip_tables" in connection:
+        if connection.skip_tables is None:
+            return []
+        elif isinstance(connection.skip_tables, str):
+            return [connection.skip_tables]
+        elif isinstance(connection.skip_tables, list):
+            return connection.skip_tables
+    else:
+        return []
+
+
+def update_config_path(path_param: str) -> None:
+    config_file_path = pathlib.Path(path_param)
+
+    if config_file_path.exists():
+        print(f"== Using {path_param} config file ==")
+        user_file_config = Dynaconf(envvar_prefix=False,
+                                    settings_files=[config_file_path])
+        config.update(user_file_config)
+    else:
+        raise IOError(f"Config file {config_file_path} does not exist.")

dbsync.py

@@ -801,51 +801,3 @@ def dbsync_push(mc):
 def dbsync_status(mc):
     for conn in config.connections:
         status(conn, mc)
-
-
-def show_usage():
-    print("dbsync")
-    print("")
-    print("    dbsync init-from-db   = will create base schema in DB + create gpkg file in working copy")
-    print("    dbsync init-from-gpkg = will create base and main schema in DB from gpkg file in working copy")
-    print("    dbsync status      = will check whether there is anything to pull or push")
-    print("    dbsync push        = will push changes from DB to Mergin Maps")
-    print("    dbsync pull        = will pull changes from Mergin Maps to DB")
-
-
-def main():
-    if len(sys.argv) < 2:
-        show_usage()
-        return
-
-    print(f"== Starting Mergin Maps DB Sync version {__version__} ==")
-
-    try:
-        validate_config(config)
-    except ConfigError as e:
-        print("Error: " + str(e))
-        return
-
-    try:
-        print("Logging in to Mergin Maps...")
-        mc = create_mergin_client()
-
-        if sys.argv[1] == 'init':
-            print("Initializing ...")
-            dbsync_init(mc)
-        elif sys.argv[1] == 'status':
-            dbsync_status(mc)
-        elif sys.argv[1] == 'push':
-            print("Pushing...")
-            dbsync_push(mc)
-        elif sys.argv[1] == 'pull':
-            print("Pulling...")
-            dbsync_pull(mc)
-        else:
-            show_usage()
-    except DbSyncError as e:
-        print("Error: " + str(e))
-
-
-if __name__ == '__main__':
-    main()

dbsync_daemon.py

@@ -7,49 +7,92 @@
 import datetime
 import sys
 import time
+import argparse
 
 import dbsync
 from version import __version__
-from config import config, validate_config, ConfigError
+from config import config, validate_config, ConfigError, update_config_path
 
 
 def main():
+
+    parser = argparse.ArgumentParser(prog='dbsync_deamon.py',
+                                     description='Synchronization tool between Mergin Maps project and database.',
+                                     epilog='www.merginmaps.com')
+
+    parser.add_argument("config_file", nargs="?", default="config.yaml", help="Path to file with configuration. Default value is config.yaml in current working directory.")
+    parser.add_argument("--skip-init", action="store_true", help="Skip DB sync init step to make the tool start faster. It is not recommend to use it unless you are really sure you can skip the initial sanity checks.")
+    parser.add_argument("--single-run", action="store_true", help="Run just once performing single pull and push operation, instead of running in infinite loop.")
+
+    args = parser.parse_args()
+
+    try:
+        update_config_path(args.config_file)
+    except IOError as e:
+        print("Error: " + str(e))
+        exit(1)
+
     print(f"== starting mergin-db-sync daemon == version {__version__} ==")
 
     sleep_time = config.as_int("daemon.sleep_time")
     try:
         validate_config(config)
     except ConfigError as e:
         print("Error: " + str(e))
-        return
+        exit(1)
 
     print("Logging in to Mergin...")
     mc = dbsync.create_mergin_client()
 
-    # run initialization before starting the sync loop
-    dbsync.dbsync_init(mc)
-
-    while True:
+    if args.single_run:
 
-        print(datetime.datetime.now())
+        if not args.skip_init:
+            try:
+                dbsync.dbsync_init(mc)
+            except dbsync.DbSyncError as e:
+                print("Error: " + str(e), file=sys.stderr)
+                exit(1)
 
         try:
             print("Trying to pull")
             dbsync.dbsync_pull(mc)
 
             print("Trying to push")
             dbsync.dbsync_push(mc)
+        except dbsync.DbSyncError as e:
+            print("Error: " + str(e), file=sys.stderr)
+            exit(1)
 
-            # check mergin client token expiration
-            delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc)
-            if delta.total_seconds() < 3600:
-                mc = dbsync.create_mergin_client()
+    else:
 
-        except dbsync.DbSyncError as e:
-            print("Error: " + str(e))
+        if not args.skip_init:
+            try:
+                dbsync.dbsync_init(mc)
+            except dbsync.DbSyncError as e:
+                print("Error: " + str(e), file=sys.stderr)
+                exit(1)
+
+        while True:
+
+            print(datetime.datetime.now())
+
+            try:
+                print("Trying to pull")
+                dbsync.dbsync_pull(mc)
+
+                print("Trying to push")
+                dbsync.dbsync_push(mc)
+
+                # check mergin client token expiration
+                delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc)
+                if delta.total_seconds() < 3600:
+                    mc = dbsync.create_mergin_client()
+
+            except dbsync.DbSyncError as e:
+                print("Error: " + str(e), file=sys.stderr)
 
-        print("Going to sleep")
-        time.sleep(sleep_time)
+            print("Going to sleep")
+            time.sleep(sleep_time)
 
 
 if __name__ == '__main__':

test/test_config.py

@@ -8,7 +8,7 @@
 import os
 import pytest
 
-from config import config, ConfigError, validate_config
+from config import config, ConfigError, validate_config, get_ignored_tables
 
 SERVER_URL = os.environ.get('TEST_MERGIN_URL')
 API_USER = os.environ.get('TEST_API_USERNAME')
@@ -64,3 +64,40 @@ def test_config():
     with pytest.raises(ConfigError, match="Config error: Name of the Mergin Maps project should be provided in the namespace/name format."):
         config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]})
         validate_config(config)
+
+
+def test_skip_tables():
+    _reset_config()
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": None}]})
+    validate_config(config)
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": []}]})
+    validate_config(config)
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": "table"}]})
+    validate_config(config)
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": ["table"]}]})
+
+
+def test_get_ignored_tables():
+    _reset_config()
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": None}]})
+    ignored_tables = get_ignored_tables(config.connections[0])
+    assert ignored_tables == []
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": []}]})
+    ignored_tables = get_ignored_tables(config.connections[0])
+    assert ignored_tables == []
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": "table"}]})
+    validate_config(config)
+    ignored_tables = get_ignored_tables(config.connections[0])
+    assert ignored_tables == ["table"]
+
+    config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables": ["table"]}]})
+    validate_config(config)
+    ignored_tables = get_ignored_tables(config.connections[0])
+    assert ignored_tables == ["table"]