From 37ee2d5cd0b21b51c4b581822766d7dfe257281f Mon Sep 17 00:00:00 2001 From: Alexander Bruy Date: Fri, 8 Jul 2022 14:54:15 +0300 Subject: [PATCH 1/6] add option to exclude tables from sync (fix #62) --- README.md | 2 +- config.py | 6 +++ config.yaml.default | 1 + dbsync.py | 89 +++++++++++++++++++------------ docs/quick_start.md | 38 +++++++++++++ test/test_basic.py | 44 ++++++++++++++- test/test_config.py | 9 ++++ test/test_data/base_2tables.gpkg | Bin 0 -> 118784 bytes test/test_data/modified_all.gpkg | Bin 0 -> 118784 bytes 9 files changed, 153 insertions(+), 36 deletions(-) create mode 100644 test/test_data/base_2tables.gpkg create mode 100644 test/test_data/modified_all.gpkg diff --git a/README.md b/README.md index c9f1638..f6d752b 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The easiest way to run DB sync is with Docker. To run the container, use a comma sudo docker run -it \ -e MERGIN__USERNAME=john \ -e MERGIN__PASSWORD=myStrongPassword \ - -e CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=mergin_dbsync user=postgres password=top_secret', modified='sync_main', base='sync_base', mergin_project='john/my_project', sync_file='sync_db.gpkg'}]" \ + -e CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=mergin_dbsync user=postgres password=top_secret', modified='sync_main', base='sync_base', mergin_project='john/my_project', sync_file='sync_db.gpkg', skip_tables:[]}]" \ lutraconsulting/mergin-db-sync:latest \ python3 dbsync_daemon.py --init-from-gpkg ``` diff --git a/config.py b/config.py index a9aecb8..5871073 100644 --- a/config.py +++ b/config.py @@ -42,3 +42,9 @@ def validate_config(config): if "/" not in conn.mergin_project: raise ConfigError("Config error: Name of the Mergin Maps project should be provided in the namespace/name format.") + + if "skip_tables" in conn: + if not isinstance(conn.skip_tables, list): + raise ConfigError("Config error: Ignored tables parameter should be a list") + if len(config.connections): + raise ConfigError("Config error: Ignored tables list can not be empty") diff --git a/config.yaml.default b/config.yaml.default index 7ffac74..6ddb837 100644 --- a/config.yaml.default +++ b/config.yaml.default @@ -13,6 +13,7 @@ connections: base: mergin_base mergin_project: john/myproject sync_file: sync.gpkg + skip_tables: daemon: sleep_time: 10 diff --git a/dbsync.py b/dbsync.py index 63465c1..9434ef1 100644 --- a/dbsync.py +++ b/dbsync.py @@ -71,16 +71,25 @@ def _run_geodiff(cmd): raise DbSyncError("geodiff failed!\n" + str(cmd)) -def _geodiff_create_changeset(driver, conn_info, base, modified, changeset): - _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, base, modified, changeset]) +def _geodiff_create_changeset(driver, conn_info, base, modified, changeset, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, "--skip-tables", ignored_tables, base, modified, changeset]) + else: + _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, base, modified, changeset]) -def _geodiff_apply_changeset(driver, conn_info, base, changeset): - _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, base, changeset]) +def _geodiff_apply_changeset(driver, conn_info, base, changeset, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, "--skip-tables", ignored_tables, base, changeset]) + else: + _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, base, changeset]) -def _geodiff_rebase(driver, conn_info, base, our, base2their, conflicts): - _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, base, our, base2their, conflicts]) +def _geodiff_rebase(driver, conn_info, base, our, base2their, conflicts, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, "--skip-tables", ignored_tables, base, our, base2their, conflicts]) + else: + _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, base, our, base2their, conflicts]) def _geodiff_list_changes_details(changeset): @@ -113,20 +122,26 @@ def _geodiff_list_changes_summary(changeset): return out["geodiff_summary"] -def _geodiff_make_copy(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst): - _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst]) +def _geodiff_make_copy(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", ignored_tables, src, dst]) + else: + _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst]) -def _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, changeset): - _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst, changeset]) +def _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, changeset, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", ignored_tables, src, dst, changeset]) + else: + _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst, changeset]) -def _compare_datasets(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, summary_only=True): +def _compare_datasets(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, ignored_tables, summary_only=True): """ Compare content of two datasets (from various drivers) and return geodiff JSON summary of changes """ tmp_dir = tempfile.gettempdir() tmp_changeset = os.path.join(tmp_dir, ''.join(random.choices(string.ascii_letters, k=8))) - _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, tmp_changeset) + _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, tmp_changeset, ignored_tables) if summary_only: return _geodiff_list_changes_summary(tmp_changeset) else: @@ -206,6 +221,7 @@ def pull(conn_cfg, mc): """ Downloads any changes from Mergin Maps and applies them to the database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" project_name = conn_cfg.mergin_project.split("/")[1] work_dir = os.path.join(config.working_dir, project_name) @@ -215,6 +231,7 @@ def pull(conn_cfg, mc): _check_has_sync_file(gpkg_full_path) mp = MerginProject(work_dir) + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") project_path = mp.metadata["name"] @@ -246,7 +263,7 @@ def pull(conn_cfg, mc): tmp_base2their = os.path.join(tmp_dir, f'{project_name}-dbsync-pull-base2their') # find out our local changes in the database (base2our) - _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_base2our) + _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_base2our, ignored_tables) needs_rebase = False if os.path.getsize(tmp_base2our) != 0: @@ -263,7 +280,7 @@ def pull(conn_cfg, mc): print("Pulled new version from Mergin Maps: " + _get_project_version(work_dir)) # simple case when there are no pending local changes - just apply whatever changes are coming - _geodiff_create_changeset("sqlite", "", gpkg_basefile_old, gpkg_basefile, tmp_base2their) + _geodiff_create_changeset("sqlite", "", gpkg_basefile_old, gpkg_basefile, tmp_base2their, ignored_tables) # summarize changes summary = _geodiff_list_changes_summary(tmp_base2their) @@ -271,14 +288,14 @@ def pull(conn_cfg, mc): if not needs_rebase: print("Applying new version [no rebase]") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their, ignored_tables) else: print("Applying new version [WITH rebase]") tmp_conflicts = os.path.join(tmp_dir, f'{project_name}-dbsync-pull-conflicts') _geodiff_rebase(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, - conn_cfg.modified, tmp_base2their, tmp_conflicts) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their) + conn_cfg.modified, tmp_base2their, tmp_conflicts, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables) os.remove(gpkg_basefile_old) conn = psycopg2.connect(conn_cfg.conn_info) @@ -290,6 +307,7 @@ def status(conn_cfg, mc): """ Figure out if there are any pending changes in the database or in Mergin Maps""" print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" project_name = conn_cfg.mergin_project.split("/")[1] @@ -301,6 +319,7 @@ def status(conn_cfg, mc): # get basic information mp = MerginProject(work_dir) + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") status_push = mp.get_push_changes() @@ -342,7 +361,7 @@ def status(conn_cfg, mc): tmp_changeset_file = os.path.join(tmp_dir, f'{project_name}-dbsync-status-base2our') if os.path.exists(tmp_changeset_file): os.remove(tmp_changeset_file) - _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file) + _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file, ignored_tables) if os.path.getsize(tmp_changeset_file) == 0: print("No changes in the database.") @@ -357,6 +376,7 @@ def push(conn_cfg, mc): """ Take changes in the 'modified' schema in the database and push them to Mergin Maps""" print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" project_name = conn_cfg.mergin_project.split("/")[1] @@ -371,6 +391,7 @@ def push(conn_cfg, mc): _check_has_sync_file(gpkg_full_path) mp = MerginProject(work_dir) + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") project_path = mp.metadata["name"] @@ -400,7 +421,7 @@ def push(conn_cfg, mc): raise DbSyncError("The 'modified' schema does not exist: " + conn_cfg.modified) # get changes in the DB - _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file) + _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file, ignored_tables) if os.path.getsize(tmp_changeset_file) == 0: print("No changes in the database.") @@ -412,7 +433,7 @@ def push(conn_cfg, mc): # write changes to the local geopackage print("Writing DB changes to working dir...") - _geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file) + _geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file, ignored_tables) # write to the server try: @@ -426,7 +447,7 @@ def push(conn_cfg, mc): # update base schema in the DB print("Updating DB base schema...") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables) _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, version) @@ -434,6 +455,8 @@ def init(conn_cfg, mc, from_gpkg=True): """ Initialize the dbsync so that it is possible to do two-way sync between Mergin Maps and a database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" + project_name = conn_cfg.mergin_project.split("/")[1] # let's start with various environment checks to make sure @@ -457,7 +480,7 @@ def init(conn_cfg, mc, from_gpkg=True): raise DbSyncError("Base schema exists but missing which project it belongs to") if "error" in db_proj_info: changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base, + conn_cfg.conn_info, conn_cfg.base, ignored_tables, summary_only=False) changes = json.dumps(changes_gpkg_base, indent=2) print(f"Changeset from failed init:\n {changes}") @@ -503,9 +526,9 @@ def init(conn_cfg, mc, from_gpkg=True): if modified_schema_exists and base_schema_exists: # if db schema already exists make sure it is already synchronized with source gpkg or fail summary_modified = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.modified) + conn_cfg.conn_info, conn_cfg.modified, ignored_tables) summary_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base) + conn_cfg.conn_info, conn_cfg.base, ignored_tables) if len(summary_base): # seems someone modified base schema manually - this should never happen! print(f"Local project version at {local_version} and base schema at {db_proj_info['version']}") @@ -528,17 +551,17 @@ def init(conn_cfg, mc, from_gpkg=True): try: # COPY: gpkg -> modified _geodiff_make_copy("sqlite", "", gpkg_full_path, - conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified) + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, ignored_tables) # COPY: modified -> base _geodiff_make_copy(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base) + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, ignored_tables) # sanity check to verify that right after initialization we do not have any changes # between the 'base' schema and the geopackage in Mergin Maps project, to make sure that # copying data back and forth will keep data intact changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base, + conn_cfg.conn_info, conn_cfg.base, ignored_tables, summary_only=False) # mark project version into db schema if len(changes_gpkg_base): @@ -560,9 +583,9 @@ def init(conn_cfg, mc, from_gpkg=True): if os.path.exists(gpkg_full_path) and base_schema_exists: # make sure output gpkg is in sync with db or fail summary_modified = _compare_datasets(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - "sqlite", "", gpkg_full_path) + "sqlite", "", gpkg_full_path, ignored_tables) summary_base = _compare_datasets(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, - "sqlite", "", gpkg_full_path) + "sqlite", "", gpkg_full_path, ignored_tables) if len(summary_base): print(f"Local project version at {_get_project_version(work_dir)} and base schema at {db_proj_info['version']}") _print_changes_summary(summary_base, "Base schema changes:") @@ -586,17 +609,17 @@ def init(conn_cfg, mc, from_gpkg=True): try: # COPY: modified -> base _geodiff_make_copy(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base) + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, ignored_tables) # COPY: modified -> gpkg _geodiff_make_copy(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - "sqlite", "", gpkg_full_path) + "sqlite", "", gpkg_full_path, ignored_tables) # sanity check to verify that right after initialization we do not have any changes # between the 'base' schema and the geopackage in Mergin Maps project, to make sure that # copying data back and forth will keep data intact changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base, summary_only=False) + conn_cfg.conn_info, conn_cfg.base, ignored_tables, summary_only=False) if len(changes_gpkg_base): changes = json.dumps(changes_gpkg_base, indent=2) print(f"Changeset after internal copy (should be empty):\n {changes}") diff --git a/docs/quick_start.md b/docs/quick_start.md index 5b9f0a3..274d66c 100644 --- a/docs/quick_start.md +++ b/docs/quick_start.md @@ -52,3 +52,41 @@ In order to stop syncing simply stop docker container. --- Note: If you run your db-sync docker on the same host as your database is, you need to set up correct [networking](https://docs.docker.com/network/). For instance, on linux for testing purpose you can use default bridge, e.g IP address `172.17.0.1` + +## 4. Excluding tables from sync + +Sometimes in the database there are tables that should not be synchronised to Mergin projects. It is possible to ignore +these tables and not sync them. To do so add `skip_tables` setting to the corresponding `CONNECTIONS` entry in the config +file + +``` +working_dir: /tmp/dbsync +geodiff_exe: geodiff + +mergin: + url: https://app.merginmaps.com + username: john + password: mysecret + +connections: + - driver: postgres + conn_info: + modified: mergin_main + base: mergin_base + mergin_project: john/myproject + sync_file: sync.gpkg + skip_tables: + - table1 + - table2 + - ... + - tableN + +daemon: + sleep_time: 10 +``` + +or as environment variable + +``` +CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=db-sync user=postgres password=top_secret', modified='sync_data', base='sync_base', mergin_project='john/db-sync', sync_file='sync_db.gpkg', skip_tables=[table1,table2,...,tableN]}]" +``` diff --git a/test/test_basic.py b/test/test_basic.py index bff7ec5..fb84a30 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -48,7 +48,7 @@ def cleanup_db(conn, schema_base, schema_main): cur.execute("COMMIT") -def init_sync_from_geopackage(mc, project_name, source_gpkg_path): +def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[]): """ Initialize sync from given GeoPackage file: - (re)create Mergin Maps project with the file @@ -75,13 +75,18 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path): # prepare dbsync config # patch config to fit testing purposes + if ignored_tables: + connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg", "skip_tables":ignored_tables} + else: + connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"} + config.update({ 'GEODIFF_EXE': GEODIFF_EXE, 'WORKING_DIR': sync_project_dir, 'MERGIN__USERNAME': API_USER, 'MERGIN__PASSWORD': USER_PWD, 'MERGIN__URL': SERVER_URL, - 'CONNECTIONS': [{"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"}], + 'CONNECTIONS': [connection], }) dbsync_init(mc, from_gpkg=True) @@ -320,3 +325,38 @@ def test_basic_both(mc): print("---") dbsync_status(mc) + +def test_init_with_skip(mc): + project_name = 'test_init_skip' + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base_2tables.gpkg') + project_dir = os.path.join(TMP_DIR, project_name + '_work') + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' + + init_sync_from_geopackage(mc, project_name, source_gpkg_path, ["lines"]) + + # test that database schemas does not have ignored table + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + assert cur.fetchone()[0] == False + cur.execute(f"SELECT count(*) from {db_schema_main}.points") + assert cur.fetchone()[0] == 0 + + # run again, nothing should change + dbsync_init(mc, from_gpkg=True) + cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + assert cur.fetchone()[0] == False + cur.execute(f"SELECT count(*) from {db_schema_main}.points") + assert cur.fetchone()[0] == 0 + + # make change in GPKG and push to server to create pending changes, it should pass but not sync + shutil.copy(os.path.join(TEST_DATA_DIR, 'modified_all.gpkg'), os.path.join(project_dir, 'test_sync.gpkg')) + mc.push_project(project_dir) + + # pull server changes to db to make sure only points table is updated + dbsync_pull(mc) + cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + assert cur.fetchone()[0] == False + cur.execute(f"SELECT count(*) from {db_schema_main}.points") + assert cur.fetchone()[0] == 4 diff --git a/test/test_config.py b/test/test_config.py index 17b5910..955793c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -66,3 +66,12 @@ def test_config(): config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]}) validate_config(config) + _reset_config() + with pytest.raises(ConfigError, match="Config error: Ignored tables parameter should be a list"): + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables":""}]}) + validate_config(config) + + _reset_config() + with pytest.raises(ConfigError, match="Config error: Ignored tables list can not be empty"): + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables":[]}]}) + validate_config(config) diff --git a/test/test_data/base_2tables.gpkg b/test/test_data/base_2tables.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..69b6eafe06ed2d68251d33edddf79675f187ca0c GIT binary patch literal 118784 zcmeI5&2JlLe#eK@3n}Y`iZ>2dQ4~L>qf%T`FQlZ%a_pvzG?uxMNSQBDwBCTrkRxhf za;R`dk(3q%C^^{{XfN%phXMuKLs6hW58LL@!(R3u=&4xj0!0toL$*M>K=;r+Y@c~% zhMXaXG)Y;OReTA1IP<*zp3n1pUPt^r$;X?bB9OgurIc03A~()C9Ng~_!g1Vn`hSZ4 zH$HdhO~3I#za1Rs|5x>SWBbDmle_u$IyblT-2v*-=lO5deaiD2&#$#xb1>Y400@8p z2!H?xfB*=900@8p2!H?xyl4X7{tiuX{B3&ijE~Z5-{{|PWArk%|Dtn3dLRG-AOHd& z00JNY0w4eaAOHeg5%@mmdcfUq-z=r2ys#%qLY@?aa-otv*cWponMlUsG5*l-tKFCb z<;vwsUX-%*U2jF$6Dop~6Nr2wD?;hXZ;g-ZJ?@6%=F@sEYuSn-$YNG9GWl&S@!Ack z2Wpea+3SC`H*|-);d~}mr*<}yf1a!pke+@Z$2ji@``b^(dZ*rgoqB)RcWe z-^#+`g71@0Y<%8b48N=TY$u{yJk4#RA_&rvnA`US7i@yxU0n#N!PBv5>WMEe6lic7 za&95Cvb4MyUJWmYR#%r6?PORDg~LpSkE67Zn^{Fv4)Zi}ES^X`qM~WeWvQUM^O0oA zPU0m^;-V39zNjZf>hxtIN+d7j#8S3Mzf~z^QkICaFA$x*e6TN6%3@w!|2sW@!g+o| zFZh4}2!H?xfB*=900@8p2!H?xfB*NRZ z@a)3M?80&?w0Li6nf{v#-Cg?L_%Bu+!xK{zdc=dWC@Hm=+EY{1MY?BkcZEj1rsn1O zCFl92=hqirKx6;{AOHd&00JNY0w4eaAOHd&00JOz;RJ^IybiXL;2i3k7}9nIu>QYr z?8pEFKmY_l00ck)1V8`;KmY_l00dqt0lW3T_V@qaaGwA1eDhL;L1rKT0w4eaAOHd& z00JNY0w4eaAOHdvLBQ?o^K$I{|B$0^g2Vg&i-3$YK>!3m00ck)1V8`;KmY_l00cnb zvI(?)|Nmpo^Y5M?U$%V69|S-E1V8`;KmY_l00ck)1V8`;K){{=ak{)yx14S_d$CVX zR455biJLm9N-E9AOHd& z00JNY0w4eaAOHd&00JQJ>Jqr#@14@j{TWMs=k@-HDc$a$;RgGEuWk)OG7ta(5C8!X z009sH0T2KI5C8!X=!HP5zyIgPdr1v(KmY_l00ck)1V8`;KmY_l00ck)1g;7J_5DBA z|5v59ASVcb00@8p2!H?xfB*=900@8p2=qokJ^v5u|K8Xk76^a<2!H?xfB*=900@8p z2!H?xTonR%|9@3#3UYz~2!H?xfB*=900@8p2!H?xfI#mAM#p}|jXVCD8~fjppO5|1 z*dL5IM*d=GfA}AVpSu2V=;pxJ14sRLo!`&^TOH&v_AAFu2f+}eI|8TcqyKi`x#M^g^JP`W-Y--QaquDLTqgC3F2?`w@brM zKHwk56FiB=l6)dX;<4sT^XpGOVNjeM}llSW@M!xG6z*^)p~{KqL0i_`z<&CQ^xsPw!TUhO(+ewJp|%6DH7 z3;PP~K^h{n?n`UY_I|A*9*D=ZeI93I@e6^h$J1(oY9eW#C-$Ce@fbZ6CqlC$2TwDK zQX<=l=vE}LOFrax%?yI3HW~<;qV3UWrBcR3OYI(`8?nX|0wlpdr2SEBjZbR0X@8~& zk|LW>2LfkrUvqhvmmObfNjBC?Mi!d-an)_OZEdO{%%!o`sJYBRxgux8d~@ZLL@9IJ z1Xp!EX?D#X+l8gB&0(!53rb0TVJsV}iKR`8r;bkb3ew8kTsiVVRuQwsOhwqs$R{$H z(E|oHh6lY*mOE~zlzekrRqJEq8Rv3&S5_Q9yjgFrh7IOSu3S7UNiwVcMn`vZT{i}4 zUD-h`WxIx5OGG(25bOf9(v2@jDjM5W8&R#*m71MQHBH-^nweTk*{pqLv4eEHR~axt zEKTl^#?G0wBpr)BPSb(D-Uuz>X%kh$OV)}v1@5S2HfmEYbADZl4h?!gS!t_FwMx-7 zA``aNiEB}J_ep7ENMjB|)i(xQ-uZb)^+{bZ%Y_Q9Ev>&|0+<~Jn-$XRZK{_x9nYSS zRSqix&E-(44-V`6!$^8l8+ihe!9j0&zAY|uzIwYeJGsu@qNk(z{f;k>>SeQ(%XsK( zE|cM9RMpBII!B+ zc~njDji~)}?d4Y2U#EQzHujzc>dMhD4lvzsaC0`RN<#>3^st#rwdhgJePXa&DP$$_ z3pyMbno;Xz&0e(Olq-2URM`&Sd0|hKM3zK@IyUNEK>hpwG3N;992xu3=x_x)F2&bj9JTkd_883Q+z;{Hs1&^~?d7;pn5#%noLf(&pdkL(7!Typ(xPR zyJbeBSFcvd1_gcR&DTLhABj9n(WUk}-AALFMpXZOT3SARgH3NJO_MXa^OhnOQOJXa z=UEcc_sWXpBcWoNuLWpW&CRdn=GXRms>g5I7hT)e>7nRtA{s@nZ}GgiqTlJd=-U&6 zs=DtoZP)i{WZ|SVp1Fc(>HN*1E^Uw0B1Gv>W?RmM3SGF+xjfr- zuJ7}yb2PT|Y>D){wawD#8yefmj@b6NU24wz727a1i$>1La`#3@da7*@o=;CLX|&W+ zeJk>^CMgXA%(`g%uts&hG)o?6nVv90&|%`b;u!i%OC$#VBwuE`zxwrRCh zo7ZQT+CF!$ifWfgjb_tyTHjN@ELF7#+r@-h7rXu@@6mOQt+z9_Ju0e2+*hnewa9i3 zl=%Js6<@SNd=LNu5C8!X009sH0T2KI5C8!X=zsv${~bU=5D)+X5C8!X009sH0T2KI z5C8!XxZ(uV^}o~efb;w%z2E}^AOHd&00JNY0w4eaAOHd&00JP;Od#VN^iIuwcB?2# zf^7Waj{R;<`;jiQ=CrRU2lwXZkB*M!XpnWDT{Yh zq2{1w%Tn>=2adrBH3RMUfftd3YX9Hq`I_^5?fF@A0pS7yAOHd&00JNY0w4eaAOHd& z00JOznFMYR(47ZI+l>e3?E$)ZFw}nU0Du2~nYD<#K>!3m00ck)1V8`;KmY_l00cnb zrV%t_WxbqA36Wt zIpYX(_eXwk?O%p|#FASwCF)l1tMYeTlaIabvmyN$M*YkFNNo071$HrxIqv8{Y!<^f z6xwtiw(kA`zY&d*i#`sUWoDd)EeGsNZF{ISi^(pnO5Hk|QR@1ksJ<6f>W)srwv@VW z#^v>T9aVuHTU{^Kgi$W-bb4LDbDmkf8IAGDRDzypZ#lR+P~GvnCiw~X=@VVmG&>vG zPh2vFeGFayG*?4BL)qHJFo2uXOiRS4o|qmZT@i;xv8JYuE%^WE!p8go}NFw)J@U! zC`Q?gGht0f%CoZ7#38pHkQl$C$wnfvb?szpQ@Nu_R_b_yBvYBF%$E+76L$J_Jxn0b z3l!RtXKHc;>r!aP(4Re+W{AF;?f}tZ_lOo>AR?+7jFE~@Rg)JC%A!3+_mCh7etR>r z#;YgA$6Ko4N0H4mpVTW=Jy?a)nXW(snL}+4lpy4aAZ4O?0iyS zB>k!n?N#qBmzU=q)eu`9*iL>k2ib}sDee5c#$;VvOSB8>n~}nk%xu!&9E<9MykM4Ie_>7SEMt8m>>WGAOHd&00JNY0w4eaAOHd& za1H`k|DOXMqJjVjfB*=900@8p2!H?xfB*=9z!f2&uK!{G?}}6w!~_8l009sH0T2KI z5O}!+VE^ys@2>b; zH5cx-=ALgx)4m=r*otN&eA;!`ThcunUSM#(0?~MdCGk~{Si(<07*B}e?d9wpy zI<>l2b=`4!mzUku#g^h|1^}&s=n{MsO{CJ1O_FL|Q$CVTQtRo2r7|PtgQO%%$Ml*# NRf6Fpm-US^uD0 zzEIdq<9=1+1zoaxL5qsy>1T)_k~lvrBx4KoudHQ;q(mak^F(8xS)Aw7DIqS;|1I`$ z#{M!r;Q#>;009sH0T2KI5C8!X009sH0T8$>foV&lqs`HD%tjB(0$rxb%<(a4Da~h; zo8F$@-p-zbojn6#Z{Oh_@8N;|t^>UXzPx_nV$D}+u`)mil_-hy8R%g(NSfSI7MUE))*>19V@7-K?#p%|Rv9uL0;HQ!3m00ck)1V8`;KmY_l00cmwY65uwUo|QSf&d7B00@8p2!H?xfB*=900@A< zmL<^C{2OKm`&Uf!r`z9a{$cY|+u7|eHq1BvedB4{A2zhtzE-hWy$i@hq9ob=Pjda$ZyG2S1i3FccES^q8C1HV&CS%lTMgOxb#HIPDAZGZqR4g+b z7$%c}(r_+9B?#vm9wWiYX&PYEA0Q!a zoEr|4pwA!TI(#Q4gJBPGsd|x=NTk?f3p{a^=-_gZ$A#2FOyVdgH*=Gio##HQIeBY$kT;GXtjuiWn4kePP7)$ab%smt)fl2y_ zjE{Q^S*82MaB|m?^DAg(<$ULPAu%t}9;7I+(tW8d+T5>Pz(rw&w$GJlMtFiJBa;z1 zLphaH&sB5J;mH8q6vs!CBa5e_l9VJx zC5*W#EKy^gJ(+YSD#S|*Cn<>0l@hqJ>uRZMY{g8Lx|YfoqD)B1%vEDqR7^_TG<)*s zlrJDPzomtfS&T_SY$2NF=c1X_4C&AW-J^|lj#C3ww^K^Kw5`hJvHculvpEhPWWUy) zZ?C!y=ICr{VJRtQl;SUTtS>F=Vv+KN?NMVkYuLF+q}4^0GT*Xo;B8zNWlL3@J{bA$xnR&C%V>W>4iS zW-5`UrKR>)iULN5y3z`1^fr`Bg^p*Ak4Z~uo~E)S<_Cun?xZg=u8urzUtOIe(p?dk zFqK3yYFKpN5qISI@f3A}Ts+Z?qXRWL<_O7|}m7{1J zQ0jh>o29g>6iKL~hsm>4nI6^HCwfxpL`)Q(pu>?~GjhGG(Tg^mR60(FD%0UR&d&*= zphQtr9gFLoTmJsP-7>~l#+qMkn%Um8?FS8feS__ct-Wqv?fKf7nt!etw0?{EPQ#-z zGX-2jnE4a==DHo0-(R<+kkBRxoDJWiu2Fc)%?FLMh0RW{P2$MS3AG9Kd21qoyPaxU-GK=J5I0?4Amm$&%)Y$v(GNqm?XJQ73uib!YAEHplQV zo84DIn{iymnQik0R;pQ2v-Qk`)$H?(DX^_B;qOm)$ec+ejVM||TX ztjyTWi_A$PA-cyaQR3+k;>rNOqKxJ8SrAv1>s9x$wmQcH`zwyL{0g0(X6UGs(VnF8 z4!32OciSBM_p@s%U1eCzdl%i_T0pVLv*c;2WQO9&MQk=HF^p`5>lB&URLQfKVGfeu zqtYx-O6Lv*6Auq5Ox*3BYN?YmKBJVOwoa4b&n)rcEMIiHvmoC^o*nYzfQN>3XLsFU zbF{RuXKR%>)wZmjZz+husX97I2XW2`ai>{jBfdzO?&C|tO>o8u8FA{*`A(v2^y?u` zSp%o?I6!aG^Z`RmILQPz;*U(w$(;M_j6?0fkCf}cjcY5L?Z8dqFzLXJQa5akv_OgE zQbt*FPNeC?g+9wGtIqj#KKU6c3rO+gcUdO)i%j)4s_TOpU6MD`Yvd ztEzgcE)ZT>JuQf%Y(3SMB5%}#hRRk`Vq=Wg+Z(*Fe>GXo z+-hr`?yNsISgy@$Pl)O|_hu`qStL1{C6C0~n);2Zss+k!;GtMkJO3tcY^TcBQ61Y& zR@4G@Z?Yb>K({ME*<^o(sj)j4`um~Q#ACv6|CcQ)0&{R=ZHkE-JDZn>`&%@LlxB{N0&Fe%9OD)4ezUt3xH+a+;ch>P(3XtBihh1 z%L10YwRG{Iq4L^+{Qkejeu1%nX1{2^L@#iF00@8p2!H?xfB*=900@8p2!H?xY-s}4 zS}W6FwaTX&`NYzbJpX4KI~n>P2MB-w2!H?xfB*=900@A9k-*od*c#@WcT*>M7cu!@ z*B|-YVfyhM|Kb0>K|gP*&Wv^a@PGc4e*Wc{=Vw2spEor3n)Ui4@6pe@$AA0dMT4+ww&2wXD)^8BBn77lTM00@8p2!H?xfB*=900@8p z2!H?xd~pdh*`8x;mJVk7ON~u+r)$1zeb(}QONacAIm#T}KG*oOZGTYzu5F#EXZj5k z>g8+Ob1Z%@V1V*PzRX(hwK*IPHp?s9Rx1`Slv7LLrqznu74KO+?hkOGaFFh8Ua)bs zJ3H;NwQ?=>XHRJxf12)~9*y%0yu_;|O7nbF^^591ozX%#iSJ~X z3XO2%T$q!0p3ZH;Jw3(+NNOS8W!PC;-t{>U86PJnxKV#V4|Os)IZ-AVC!x^Z# zF)}$m(xq(deS~mn$$7lO$guOB-{p zqS3Z^gX()hG15+;d7JgCU*tgqc>{r)nZVkPy-E-D^!-)zP~MIAP~L0RLv4Jx*)oGh z!l^v*oqMukk*GVoZ=^)j5PE^=IEtT&I_QRNrjLBp5A1r>H~a8Rzsz#!W^SC&?OG6M zYW|q9G(W{OKV||iQcadcYQR(TBrtA`q2e{@E@-|Ms|KU z^iR87|NW!=ALjz-5$FNscamNj?zO9)gX<9CBNhs&t5t{e;{$*G$3J@E{qiF*&Mdt-bo<4x)O|nwz6wbOr3>gDdO$runipe9J}!U%-(tTc+yA>{zjz(; zgCsxz1V8`;KmY_l00ck)1V8`;KmY`;3xQh3YJvT~>w+4IfB*=900@8p2!H?xY-Iv@ z`+xEqHu+%J6VKkg-gPd0{ImYQ8EQzksm_e$C%^l*w|?@|lfOLvm*0Kv_-~(2U&dYY z#vc#>0T8%;1YrN~`mshzAOHd&00JNY0w4eaAOHd&00JOzqX@|J|7OcQjQ#QEUo}76 z^!oNkx4qHus_nD7_v-4cdo1_V?yT8;qjW>MHw%I6%3)h;h+3z!YgSmaKbqaV=o}xD zmePE5Hnk*5nsGgIV=Pq!)NQhuSm9F*?P(_L5{su3(NrQ$Qxzp%lrkj5g-L-b)U;RO zeJAm1Hux4&%Y3>+F=iN*e+lEJvZ|X))!e08Ez7P9nrB@#AhU_AR|u$QUAGxyFh_I6 zS>M0$tY?neS|4!KuQgOt>Uoop#U>Xkmbqpyvc&MB=6=Q5uByb9#PkBs95K(eX}MiK z*X4tml=juibybC3kn7+9TkEi+{%k}mE!9q@B2sK7Q*kUVT^}zqNXuhdHaIb+Ms4x9+$DFd9Yuh^Gw_2>GW-sj+rsd zP14!8aWk!=87;AvWmI5;QSW!nZFLpLapNp$-r1+t>)FefT<=Cp?($|d&CBr`ThT0r zU)GZDT4S#Q=V|WpaJ(CZb9oDXn}f6QI=)q^>SJDITsgN?sWj9vviibM<@IqzM?s&K zjP$R$?LH;3+~DXWH>^L%-qvGt3=gx}ZsmE_bSPeo(O-N^6})!qBf9UzIHy|=F3e(7 zjl|g@rsBMqlI;w0vQh1+pnt*_oFVsfGagk$7I0_lyKRo4AvW7yUM*SBR4Lh`wF;GF zxrz!=gV`~6c6*o2;q$TCz2#L&r Date: Sun, 10 Jul 2022 18:48:20 +0300 Subject: [PATCH 2/6] fix check for skipped tables --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 5871073..7b98fc9 100644 --- a/config.py +++ b/config.py @@ -46,5 +46,5 @@ def validate_config(config): if "skip_tables" in conn: if not isinstance(conn.skip_tables, list): raise ConfigError("Config error: Ignored tables parameter should be a list") - if len(config.connections): + if len(config.connections) <= 0: raise ConfigError("Config error: Ignored tables list can not be empty") From 7ce2798093ebbe97c1353e30a4e3a010f523982f Mon Sep 17 00:00:00 2001 From: Alexander Bruy Date: Sun, 10 Jul 2022 18:48:54 +0300 Subject: [PATCH 3/6] add dockerfile for building test container --- test.dockerfile | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test.dockerfile diff --git a/test.dockerfile b/test.dockerfile new file mode 100644 index 0000000..dcfd815 --- /dev/null +++ b/test.dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:20.04 +MAINTAINER Martin Dobias "martin.dobias@lutraconsulting.co.uk" + +# this is to do choice of timezone upfront, because when "tzdata" package gets installed, +# it comes up with interactive command line prompt when package is being set up +ENV TZ=Europe/London +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && apt-get install -y \ + cmake \ + git \ + libpq-dev \ + libsqlite3-dev \ + python3-pip \ + python3-psycopg2 \ + && rm -rf /var/lib/apt/lists/* + +# Python Mergin client +RUN python3 -m pip install --upgrade pip +RUN pip3 install dynaconf==3.1.7 +RUN pip3 install scikit-build wheel cmake + +# geodiff (version >= 1.0.0 is needed with PostgreSQL support - we have to compile it) +RUN git clone --branch skip-tables https://github.com/merginmaps/geodiff.git +RUN cd geodiff && mkdir build && cd build && \ + cmake -DWITH_POSTGRESQL=TRUE ../geodiff && \ + make + +# build pygeodiff +RUN cd geodiff && python3 setup.py build && python3 setup.py install + +# build mergin python client +RUN git clone --branch skip-tables https://github.com/merginmaps/mergin-py-client.git +RUN cd mergin-py-client && python3 setup.py build && python3 setup.py install + +# DB sync code +WORKDIR /mergin-db-sync +COPY version.py . +COPY config.py . +COPY dbsync.py . +COPY dbsync_daemon.py . + +# base DB sync config file (the rest is configured with env variables) +COPY config-dockerfile.yaml ./config.yaml From d17f49cda6ace62c4d642b0bde106126aea30ebd Mon Sep 17 00:00:00 2001 From: Alexander Bruy Date: Mon, 11 Jul 2022 13:44:09 +0300 Subject: [PATCH 4/6] add requirements.txt --- Dockerfile | 4 ++-- requirements.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index 4039aa8..781e235 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,8 @@ RUN apt-get update && apt-get install -y \ # Python Mergin client RUN python3 -m pip install --upgrade pip -RUN pip3 install mergin-client==0.7.3 -RUN pip3 install dynaconf==3.1.7 +COPY requirements.txt ./ +RUN pip3 install -r requirements.txt # geodiff (version >= 1.0.0 is needed with PostgreSQL support - we have to compile it) RUN git clone --branch 1.0.0 https://github.com/lutraconsulting/geodiff.git diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7ca4dd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +mergin-client==0.7.3 +dynaconf>=3.1 From 1057398b1fc5411b1bf80837bef17a9397cc6ac7 Mon Sep 17 00:00:00 2001 From: Alexander Bruy Date: Mon, 11 Jul 2022 14:22:27 +0300 Subject: [PATCH 5/6] move parsing of ignored tables to config --- config.py | 4 ++++ dbsync.py | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index 7b98fc9..536c31b 100644 --- a/config.py +++ b/config.py @@ -48,3 +48,7 @@ def validate_config(config): raise ConfigError("Config error: Ignored tables parameter should be a list") if len(config.connections) <= 0: raise ConfigError("Config error: Ignored tables list can not be empty") + + +def get_ignored_tables(connection): + return ";".join(connection.skip_tables) if "skip_tables" in connection else "" diff --git a/dbsync.py b/dbsync.py index 9434ef1..b435bda 100644 --- a/dbsync.py +++ b/dbsync.py @@ -21,7 +21,7 @@ from mergin import MerginClient, MerginProject, LoginError, ClientError from version import __version__ -from config import config, validate_config, ConfigError +from config import config, validate_config, get_ignored_tables, ConfigError # set high logging level for geodiff (used by geodiff executable) # so we get as much information as possible @@ -221,7 +221,7 @@ def pull(conn_cfg, mc): """ Downloads any changes from Mergin Maps and applies them to the database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") - ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" + ignored_tables = config.get_ignored_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] work_dir = os.path.join(config.working_dir, project_name) @@ -307,7 +307,7 @@ def status(conn_cfg, mc): """ Figure out if there are any pending changes in the database or in Mergin Maps""" print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") - ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" + ignored_tables = config.get_ignored_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -376,7 +376,7 @@ def push(conn_cfg, mc): """ Take changes in the 'modified' schema in the database and push them to Mergin Maps""" print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") - ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" + ignored_tables = config.get_ignored_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -455,7 +455,7 @@ def init(conn_cfg, mc, from_gpkg=True): """ Initialize the dbsync so that it is possible to do two-way sync between Mergin Maps and a database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") - ignored_tables = ";".join(conn_cfg.skip_tables) if "skip_tables" in conn_cfg else "" + ignored_tables = config.get_ignored_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] From 7aec290dbac2751ff494a3323b399e6c75bdc1de Mon Sep 17 00:00:00 2001 From: Alexander Bruy Date: Tue, 12 Jul 2022 15:19:34 +0300 Subject: [PATCH 6/6] adapt to API change in geodiff --- config.py | 2 +- dbsync.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index 536c31b..93ac2c0 100644 --- a/config.py +++ b/config.py @@ -51,4 +51,4 @@ def validate_config(config): def get_ignored_tables(connection): - return ";".join(connection.skip_tables) if "skip_tables" in connection else "" + return connection.skip_tables if "skip_tables" in connection else [] diff --git a/dbsync.py b/dbsync.py index b435bda..56eab83 100644 --- a/dbsync.py +++ b/dbsync.py @@ -32,6 +32,10 @@ class DbSyncError(Exception): pass +def _tables_list_to_string(tables): + return ";".join(tables) + + def _check_has_working_dir(work_path): if not os.path.exists(work_path): raise DbSyncError("The project working directory does not exist: " + work_path) @@ -73,21 +77,21 @@ def _run_geodiff(cmd): def _geodiff_create_changeset(driver, conn_info, base, modified, changeset, ignored_tables): if ignored_tables: - _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, "--skip-tables", ignored_tables, base, modified, changeset]) + _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, "--skip-tables", _tables_to_string(ignored_tables), base, modified, changeset]) else: _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, base, modified, changeset]) def _geodiff_apply_changeset(driver, conn_info, base, changeset, ignored_tables): if ignored_tables: - _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, "--skip-tables", ignored_tables, base, changeset]) + _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, "--skip-tables", _tables_to_string(ignored_tables), base, changeset]) else: _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, base, changeset]) def _geodiff_rebase(driver, conn_info, base, our, base2their, conflicts, ignored_tables): if ignored_tables: - _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, "--skip-tables", ignored_tables, base, our, base2their, conflicts]) + _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, "--skip-tables", _tables_to_string(ignored_tables), base, our, base2their, conflicts]) else: _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, base, our, base2their, conflicts]) @@ -124,14 +128,14 @@ def _geodiff_list_changes_summary(changeset): def _geodiff_make_copy(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, ignored_tables): if ignored_tables: - _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", ignored_tables, src, dst]) + _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", _tables_to_string(ignored_tables), src, dst]) else: _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst]) def _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, changeset, ignored_tables): if ignored_tables: - _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", ignored_tables, src, dst, changeset]) + _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", _tables_to_string(ignored_tables), src, dst, changeset]) else: _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst, changeset]) @@ -221,7 +225,7 @@ def pull(conn_cfg, mc): """ Downloads any changes from Mergin Maps and applies them to the database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") - ignored_tables = config.get_ignored_tables(conn_cfg) + ignored_tables = config.get_ignored_tables() project_name = conn_cfg.mergin_project.split("/")[1] work_dir = os.path.join(config.working_dir, project_name)