diff --git a/Dockerfile b/Dockerfile index 4039aa8..781e235 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,8 @@ RUN apt-get update && apt-get install -y \ # Python Mergin client RUN python3 -m pip install --upgrade pip -RUN pip3 install mergin-client==0.7.3 -RUN pip3 install dynaconf==3.1.7 +COPY requirements.txt ./ +RUN pip3 install -r requirements.txt # geodiff (version >= 1.0.0 is needed with PostgreSQL support - we have to compile it) RUN git clone --branch 1.0.0 https://github.com/lutraconsulting/geodiff.git diff --git a/README.md b/README.md index c9f1638..f6d752b 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The easiest way to run DB sync is with Docker. To run the container, use a comma sudo docker run -it \ -e MERGIN__USERNAME=john \ -e MERGIN__PASSWORD=myStrongPassword \ - -e CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=mergin_dbsync user=postgres password=top_secret', modified='sync_main', base='sync_base', mergin_project='john/my_project', sync_file='sync_db.gpkg'}]" \ + -e CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=mergin_dbsync user=postgres password=top_secret', modified='sync_main', base='sync_base', mergin_project='john/my_project', sync_file='sync_db.gpkg', skip_tables:[]}]" \ lutraconsulting/mergin-db-sync:latest \ python3 dbsync_daemon.py --init-from-gpkg ``` diff --git a/config.py b/config.py index a9aecb8..93ac2c0 100644 --- a/config.py +++ b/config.py @@ -42,3 +42,13 @@ def validate_config(config): if "/" not in conn.mergin_project: raise ConfigError("Config error: Name of the Mergin Maps project should be provided in the namespace/name format.") + + if "skip_tables" in conn: + if not isinstance(conn.skip_tables, list): + raise ConfigError("Config error: Ignored tables parameter should be a list") + if len(config.connections) <= 0: + raise ConfigError("Config error: Ignored tables list can not be empty") + + +def get_ignored_tables(connection): + return connection.skip_tables if "skip_tables" in connection else [] diff --git a/config.yaml.default b/config.yaml.default index 7ffac74..6ddb837 100644 --- a/config.yaml.default +++ b/config.yaml.default @@ -13,6 +13,7 @@ connections: base: mergin_base mergin_project: john/myproject sync_file: sync.gpkg + skip_tables: daemon: sleep_time: 10 diff --git a/dbsync.py b/dbsync.py index 63465c1..56eab83 100644 --- a/dbsync.py +++ b/dbsync.py @@ -21,7 +21,7 @@ from mergin import MerginClient, MerginProject, LoginError, ClientError from version import __version__ -from config import config, validate_config, ConfigError +from config import config, validate_config, get_ignored_tables, ConfigError # set high logging level for geodiff (used by geodiff executable) # so we get as much information as possible @@ -32,6 +32,10 @@ class DbSyncError(Exception): pass +def _tables_list_to_string(tables): + return ";".join(tables) + + def _check_has_working_dir(work_path): if not os.path.exists(work_path): raise DbSyncError("The project working directory does not exist: " + work_path) @@ -71,16 +75,25 @@ def _run_geodiff(cmd): raise DbSyncError("geodiff failed!\n" + str(cmd)) -def _geodiff_create_changeset(driver, conn_info, base, modified, changeset): - _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, base, modified, changeset]) +def _geodiff_create_changeset(driver, conn_info, base, modified, changeset, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, "--skip-tables", _tables_to_string(ignored_tables), base, modified, changeset]) + else: + _run_geodiff([config.geodiff_exe, "diff", "--driver", driver, conn_info, base, modified, changeset]) -def _geodiff_apply_changeset(driver, conn_info, base, changeset): - _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, base, changeset]) +def _geodiff_apply_changeset(driver, conn_info, base, changeset, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, "--skip-tables", _tables_to_string(ignored_tables), base, changeset]) + else: + _run_geodiff([config.geodiff_exe, "apply", "--driver", driver, conn_info, base, changeset]) -def _geodiff_rebase(driver, conn_info, base, our, base2their, conflicts): - _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, base, our, base2their, conflicts]) +def _geodiff_rebase(driver, conn_info, base, our, base2their, conflicts, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, "--skip-tables", _tables_to_string(ignored_tables), base, our, base2their, conflicts]) + else: + _run_geodiff([config.geodiff_exe, "rebase-db", "--driver", driver, conn_info, base, our, base2their, conflicts]) def _geodiff_list_changes_details(changeset): @@ -113,20 +126,26 @@ def _geodiff_list_changes_summary(changeset): return out["geodiff_summary"] -def _geodiff_make_copy(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst): - _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst]) +def _geodiff_make_copy(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", _tables_to_string(ignored_tables), src, dst]) + else: + _run_geodiff([config.geodiff_exe, "copy", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst]) -def _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, changeset): - _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst, changeset]) +def _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, changeset, ignored_tables): + if ignored_tables: + _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, "--skip-tables", _tables_to_string(ignored_tables), src, dst, changeset]) + else: + _run_geodiff([config.geodiff_exe, "diff", "--driver-1", src_driver, src_conn_info, "--driver-2", dst_driver, dst_conn_info, src, dst, changeset]) -def _compare_datasets(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, summary_only=True): +def _compare_datasets(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, ignored_tables, summary_only=True): """ Compare content of two datasets (from various drivers) and return geodiff JSON summary of changes """ tmp_dir = tempfile.gettempdir() tmp_changeset = os.path.join(tmp_dir, ''.join(random.choices(string.ascii_letters, k=8))) - _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, tmp_changeset) + _geodiff_create_changeset_dr(src_driver, src_conn_info, src, dst_driver, dst_conn_info, dst, tmp_changeset, ignored_tables) if summary_only: return _geodiff_list_changes_summary(tmp_changeset) else: @@ -206,6 +225,7 @@ def pull(conn_cfg, mc): """ Downloads any changes from Mergin Maps and applies them to the database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = config.get_ignored_tables() project_name = conn_cfg.mergin_project.split("/")[1] work_dir = os.path.join(config.working_dir, project_name) @@ -215,6 +235,7 @@ def pull(conn_cfg, mc): _check_has_sync_file(gpkg_full_path) mp = MerginProject(work_dir) + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") project_path = mp.metadata["name"] @@ -246,7 +267,7 @@ def pull(conn_cfg, mc): tmp_base2their = os.path.join(tmp_dir, f'{project_name}-dbsync-pull-base2their') # find out our local changes in the database (base2our) - _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_base2our) + _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_base2our, ignored_tables) needs_rebase = False if os.path.getsize(tmp_base2our) != 0: @@ -263,7 +284,7 @@ def pull(conn_cfg, mc): print("Pulled new version from Mergin Maps: " + _get_project_version(work_dir)) # simple case when there are no pending local changes - just apply whatever changes are coming - _geodiff_create_changeset("sqlite", "", gpkg_basefile_old, gpkg_basefile, tmp_base2their) + _geodiff_create_changeset("sqlite", "", gpkg_basefile_old, gpkg_basefile, tmp_base2their, ignored_tables) # summarize changes summary = _geodiff_list_changes_summary(tmp_base2their) @@ -271,14 +292,14 @@ def pull(conn_cfg, mc): if not needs_rebase: print("Applying new version [no rebase]") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their, ignored_tables) else: print("Applying new version [WITH rebase]") tmp_conflicts = os.path.join(tmp_dir, f'{project_name}-dbsync-pull-conflicts') _geodiff_rebase(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, - conn_cfg.modified, tmp_base2their, tmp_conflicts) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their) + conn_cfg.modified, tmp_base2their, tmp_conflicts, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables) os.remove(gpkg_basefile_old) conn = psycopg2.connect(conn_cfg.conn_info) @@ -290,6 +311,7 @@ def status(conn_cfg, mc): """ Figure out if there are any pending changes in the database or in Mergin Maps""" print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = config.get_ignored_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -301,6 +323,7 @@ def status(conn_cfg, mc): # get basic information mp = MerginProject(work_dir) + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") status_push = mp.get_push_changes() @@ -342,7 +365,7 @@ def status(conn_cfg, mc): tmp_changeset_file = os.path.join(tmp_dir, f'{project_name}-dbsync-status-base2our') if os.path.exists(tmp_changeset_file): os.remove(tmp_changeset_file) - _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file) + _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file, ignored_tables) if os.path.getsize(tmp_changeset_file) == 0: print("No changes in the database.") @@ -357,6 +380,7 @@ def push(conn_cfg, mc): """ Take changes in the 'modified' schema in the database and push them to Mergin Maps""" print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = config.get_ignored_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -371,6 +395,7 @@ def push(conn_cfg, mc): _check_has_sync_file(gpkg_full_path) mp = MerginProject(work_dir) + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") project_path = mp.metadata["name"] @@ -400,7 +425,7 @@ def push(conn_cfg, mc): raise DbSyncError("The 'modified' schema does not exist: " + conn_cfg.modified) # get changes in the DB - _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file) + _geodiff_create_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, conn_cfg.modified, tmp_changeset_file, ignored_tables) if os.path.getsize(tmp_changeset_file) == 0: print("No changes in the database.") @@ -412,7 +437,7 @@ def push(conn_cfg, mc): # write changes to the local geopackage print("Writing DB changes to working dir...") - _geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file) + _geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file, ignored_tables) # write to the server try: @@ -426,7 +451,7 @@ def push(conn_cfg, mc): # update base schema in the DB print("Updating DB base schema...") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables) _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, version) @@ -434,6 +459,8 @@ def init(conn_cfg, mc, from_gpkg=True): """ Initialize the dbsync so that it is possible to do two-way sync between Mergin Maps and a database """ print(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") + ignored_tables = config.get_ignored_tables(conn_cfg) + project_name = conn_cfg.mergin_project.split("/")[1] # let's start with various environment checks to make sure @@ -457,7 +484,7 @@ def init(conn_cfg, mc, from_gpkg=True): raise DbSyncError("Base schema exists but missing which project it belongs to") if "error" in db_proj_info: changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base, + conn_cfg.conn_info, conn_cfg.base, ignored_tables, summary_only=False) changes = json.dumps(changes_gpkg_base, indent=2) print(f"Changeset from failed init:\n {changes}") @@ -503,9 +530,9 @@ def init(conn_cfg, mc, from_gpkg=True): if modified_schema_exists and base_schema_exists: # if db schema already exists make sure it is already synchronized with source gpkg or fail summary_modified = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.modified) + conn_cfg.conn_info, conn_cfg.modified, ignored_tables) summary_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base) + conn_cfg.conn_info, conn_cfg.base, ignored_tables) if len(summary_base): # seems someone modified base schema manually - this should never happen! print(f"Local project version at {local_version} and base schema at {db_proj_info['version']}") @@ -528,17 +555,17 @@ def init(conn_cfg, mc, from_gpkg=True): try: # COPY: gpkg -> modified _geodiff_make_copy("sqlite", "", gpkg_full_path, - conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified) + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, ignored_tables) # COPY: modified -> base _geodiff_make_copy(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base) + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, ignored_tables) # sanity check to verify that right after initialization we do not have any changes # between the 'base' schema and the geopackage in Mergin Maps project, to make sure that # copying data back and forth will keep data intact changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base, + conn_cfg.conn_info, conn_cfg.base, ignored_tables, summary_only=False) # mark project version into db schema if len(changes_gpkg_base): @@ -560,9 +587,9 @@ def init(conn_cfg, mc, from_gpkg=True): if os.path.exists(gpkg_full_path) and base_schema_exists: # make sure output gpkg is in sync with db or fail summary_modified = _compare_datasets(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - "sqlite", "", gpkg_full_path) + "sqlite", "", gpkg_full_path, ignored_tables) summary_base = _compare_datasets(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, - "sqlite", "", gpkg_full_path) + "sqlite", "", gpkg_full_path, ignored_tables) if len(summary_base): print(f"Local project version at {_get_project_version(work_dir)} and base schema at {db_proj_info['version']}") _print_changes_summary(summary_base, "Base schema changes:") @@ -586,17 +613,17 @@ def init(conn_cfg, mc, from_gpkg=True): try: # COPY: modified -> base _geodiff_make_copy(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base) + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, ignored_tables) # COPY: modified -> gpkg _geodiff_make_copy(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - "sqlite", "", gpkg_full_path) + "sqlite", "", gpkg_full_path, ignored_tables) # sanity check to verify that right after initialization we do not have any changes # between the 'base' schema and the geopackage in Mergin Maps project, to make sure that # copying data back and forth will keep data intact changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, - conn_cfg.conn_info, conn_cfg.base, summary_only=False) + conn_cfg.conn_info, conn_cfg.base, ignored_tables, summary_only=False) if len(changes_gpkg_base): changes = json.dumps(changes_gpkg_base, indent=2) print(f"Changeset after internal copy (should be empty):\n {changes}") diff --git a/docs/quick_start.md b/docs/quick_start.md index 5b9f0a3..274d66c 100644 --- a/docs/quick_start.md +++ b/docs/quick_start.md @@ -52,3 +52,41 @@ In order to stop syncing simply stop docker container. --- Note: If you run your db-sync docker on the same host as your database is, you need to set up correct [networking](https://docs.docker.com/network/). For instance, on linux for testing purpose you can use default bridge, e.g IP address `172.17.0.1` + +## 4. Excluding tables from sync + +Sometimes in the database there are tables that should not be synchronised to Mergin projects. It is possible to ignore +these tables and not sync them. To do so add `skip_tables` setting to the corresponding `CONNECTIONS` entry in the config +file + +``` +working_dir: /tmp/dbsync +geodiff_exe: geodiff + +mergin: + url: https://app.merginmaps.com + username: john + password: mysecret + +connections: + - driver: postgres + conn_info: + modified: mergin_main + base: mergin_base + mergin_project: john/myproject + sync_file: sync.gpkg + skip_tables: + - table1 + - table2 + - ... + - tableN + +daemon: + sleep_time: 10 +``` + +or as environment variable + +``` +CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=db-sync user=postgres password=top_secret', modified='sync_data', base='sync_base', mergin_project='john/db-sync', sync_file='sync_db.gpkg', skip_tables=[table1,table2,...,tableN]}]" +``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7ca4dd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +mergin-client==0.7.3 +dynaconf>=3.1 diff --git a/test.dockerfile b/test.dockerfile new file mode 100644 index 0000000..dcfd815 --- /dev/null +++ b/test.dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:20.04 +MAINTAINER Martin Dobias "martin.dobias@lutraconsulting.co.uk" + +# this is to do choice of timezone upfront, because when "tzdata" package gets installed, +# it comes up with interactive command line prompt when package is being set up +ENV TZ=Europe/London +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && apt-get install -y \ + cmake \ + git \ + libpq-dev \ + libsqlite3-dev \ + python3-pip \ + python3-psycopg2 \ + && rm -rf /var/lib/apt/lists/* + +# Python Mergin client +RUN python3 -m pip install --upgrade pip +RUN pip3 install dynaconf==3.1.7 +RUN pip3 install scikit-build wheel cmake + +# geodiff (version >= 1.0.0 is needed with PostgreSQL support - we have to compile it) +RUN git clone --branch skip-tables https://github.com/merginmaps/geodiff.git +RUN cd geodiff && mkdir build && cd build && \ + cmake -DWITH_POSTGRESQL=TRUE ../geodiff && \ + make + +# build pygeodiff +RUN cd geodiff && python3 setup.py build && python3 setup.py install + +# build mergin python client +RUN git clone --branch skip-tables https://github.com/merginmaps/mergin-py-client.git +RUN cd mergin-py-client && python3 setup.py build && python3 setup.py install + +# DB sync code +WORKDIR /mergin-db-sync +COPY version.py . +COPY config.py . +COPY dbsync.py . +COPY dbsync_daemon.py . + +# base DB sync config file (the rest is configured with env variables) +COPY config-dockerfile.yaml ./config.yaml diff --git a/test/test_basic.py b/test/test_basic.py index bff7ec5..fb84a30 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -48,7 +48,7 @@ def cleanup_db(conn, schema_base, schema_main): cur.execute("COMMIT") -def init_sync_from_geopackage(mc, project_name, source_gpkg_path): +def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[]): """ Initialize sync from given GeoPackage file: - (re)create Mergin Maps project with the file @@ -75,13 +75,18 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path): # prepare dbsync config # patch config to fit testing purposes + if ignored_tables: + connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg", "skip_tables":ignored_tables} + else: + connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"} + config.update({ 'GEODIFF_EXE': GEODIFF_EXE, 'WORKING_DIR': sync_project_dir, 'MERGIN__USERNAME': API_USER, 'MERGIN__PASSWORD': USER_PWD, 'MERGIN__URL': SERVER_URL, - 'CONNECTIONS': [{"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"}], + 'CONNECTIONS': [connection], }) dbsync_init(mc, from_gpkg=True) @@ -320,3 +325,38 @@ def test_basic_both(mc): print("---") dbsync_status(mc) + +def test_init_with_skip(mc): + project_name = 'test_init_skip' + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base_2tables.gpkg') + project_dir = os.path.join(TMP_DIR, project_name + '_work') + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' + + init_sync_from_geopackage(mc, project_name, source_gpkg_path, ["lines"]) + + # test that database schemas does not have ignored table + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + assert cur.fetchone()[0] == False + cur.execute(f"SELECT count(*) from {db_schema_main}.points") + assert cur.fetchone()[0] == 0 + + # run again, nothing should change + dbsync_init(mc, from_gpkg=True) + cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + assert cur.fetchone()[0] == False + cur.execute(f"SELECT count(*) from {db_schema_main}.points") + assert cur.fetchone()[0] == 0 + + # make change in GPKG and push to server to create pending changes, it should pass but not sync + shutil.copy(os.path.join(TEST_DATA_DIR, 'modified_all.gpkg'), os.path.join(project_dir, 'test_sync.gpkg')) + mc.push_project(project_dir) + + # pull server changes to db to make sure only points table is updated + dbsync_pull(mc) + cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + assert cur.fetchone()[0] == False + cur.execute(f"SELECT count(*) from {db_schema_main}.points") + assert cur.fetchone()[0] == 4 diff --git a/test/test_config.py b/test/test_config.py index 17b5910..955793c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -66,3 +66,12 @@ def test_config(): config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]}) validate_config(config) + _reset_config() + with pytest.raises(ConfigError, match="Config error: Ignored tables parameter should be a list"): + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables":""}]}) + validate_config(config) + + _reset_config() + with pytest.raises(ConfigError, match="Config error: Ignored tables list can not be empty"): + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables":[]}]}) + validate_config(config) diff --git a/test/test_data/base_2tables.gpkg b/test/test_data/base_2tables.gpkg new file mode 100644 index 0000000..69b6eaf Binary files /dev/null and b/test/test_data/base_2tables.gpkg differ diff --git a/test/test_data/modified_all.gpkg b/test/test_data/modified_all.gpkg new file mode 100644 index 0000000..d7086d6 Binary files /dev/null and b/test/test_data/modified_all.gpkg differ