Skip tables

Dockerfile

@@ -17,8 +17,8 @@ RUN apt-get update && apt-get install -y \
 
 # Python Mergin client
 RUN python3 -m pip install --upgrade pip
-RUN pip3 install mergin-client==0.7.3
-RUN pip3 install dynaconf==3.1.7
+COPY requirements.txt ./
+RUN pip3 install -r requirements.txt
 
 # geodiff (version >= 1.0.0 is needed with PostgreSQL support - we have to compile it)
 RUN git clone --branch 1.0.0 https://github.com/lutraconsulting/geodiff.git

README.md

@@ -34,7 +34,7 @@ The easiest way to run DB sync is with Docker. To run the container, use a comma
 sudo docker run -it \
   -e MERGIN__USERNAME=john \
   -e MERGIN__PASSWORD=myStrongPassword \
-  -e CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=mergin_dbsync user=postgres password=top_secret', modified='sync_main', base='sync_base', mergin_project='john/my_project', sync_file='sync_db.gpkg'}]" \
+  -e CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=mergin_dbsync user=postgres password=top_secret', modified='sync_main', base='sync_base', mergin_project='john/my_project', sync_file='sync_db.gpkg', skip_tables:[]}]" \
   lutraconsulting/mergin-db-sync:latest \
   python3 dbsync_daemon.py --init-from-gpkg
 ```

config.py

@@ -42,3 +42,13 @@ def validate_config(config):
 
         if "/" not in conn.mergin_project:
             raise ConfigError("Config error: Name of the Mergin Maps project should be provided in the namespace/name format.")
+
+        if "skip_tables" in conn:
+            if not isinstance(conn.skip_tables, list):
+                raise ConfigError("Config error: Ignored tables parameter should be a list")
+            if len(config.connections) <= 0:
+                raise ConfigError("Config error: Ignored tables list can not be empty")
+
+
+def get_ignored_tables(connection):
+    return connection.skip_tables if "skip_tables" in connection else []

config.yaml.default

@@ -13,6 +13,7 @@ connections:
      base: mergin_base
      mergin_project: john/myproject
      sync_file: sync.gpkg
+     skip_tables:
 
 daemon:
   sleep_time: 10

docs/quick_start.md

@@ -52,3 +52,41 @@ In order to stop syncing simply stop docker container.
 ---
 Note: If you run your db-sync docker on the same host as your database is, you need to set up correct [networking](https://docs.docker.com/network/).
 For instance, on linux for testing purpose you can use default bridge, e.g IP address `172.17.0.1`
+
+## 4. Excluding tables from sync
+
+Sometimes in the database there are tables that should not be synchronised to Mergin projects. It is possible to ignore
+these tables and not sync them. To do so add `skip_tables` setting to the corresponding `CONNECTIONS` entry in the config
+file
+
+```
+working_dir: /tmp/dbsync
+geodiff_exe: geodiff
+
+mergin:
+  url: https://app.merginmaps.com
+  username: john
+  password: mysecret
+
+connections:
+   - driver: postgres
+     conn_info:
+     modified: mergin_main
+     base: mergin_base
+     mergin_project: john/myproject
+     sync_file: sync.gpkg
+     skip_tables:
+      - table1
+      - table2
+      - ...
+      - tableN
+
+daemon:
+  sleep_time: 10
+```
+
+or as environment variable
+
+```
+CONNECTIONS="[{driver='postgres', conn_info='host=myhost.com dbname=db-sync user=postgres password=top_secret', modified='sync_data', base='sync_base', mergin_project='john/db-sync', sync_file='sync_db.gpkg', skip_tables=[table1,table2,...,tableN]}]"
+```

requirements.txt

@@ -0,0 +1,2 @@
+mergin-client==0.7.3
+dynaconf>=3.1

test.dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:20.04
+MAINTAINER Martin Dobias "martin.dobias@lutraconsulting.co.uk"
+
+# this is to do choice of timezone upfront, because when "tzdata" package gets installed,
+# it comes up with interactive command line prompt when package is being set up
+ENV TZ=Europe/London
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update && apt-get install -y \
+    cmake \
+    git \
+    libpq-dev \
+    libsqlite3-dev \
+    python3-pip \
+    python3-psycopg2 \
+ && rm -rf /var/lib/apt/lists/*
+
+# Python Mergin client
+RUN python3 -m pip install --upgrade pip
+RUN pip3 install dynaconf==3.1.7
+RUN pip3 install scikit-build wheel cmake
+
+# geodiff (version >= 1.0.0 is needed with PostgreSQL support - we have to compile it)
+RUN git clone --branch skip-tables https://github.com/merginmaps/geodiff.git
+RUN cd geodiff && mkdir build && cd build && \
+    cmake -DWITH_POSTGRESQL=TRUE ../geodiff && \
+    make
+
+# build pygeodiff
+RUN cd geodiff && python3 setup.py build && python3 setup.py install
+
+# build mergin python client
+RUN git clone --branch skip-tables https://github.com/merginmaps/mergin-py-client.git
+RUN cd mergin-py-client && python3 setup.py build && python3 setup.py install
+
+# DB sync code
+WORKDIR /mergin-db-sync
+COPY version.py .
+COPY config.py .
+COPY dbsync.py .
+COPY dbsync_daemon.py .
+
+# base DB sync config file (the rest is configured with env variables)
+COPY config-dockerfile.yaml ./config.yaml

test/test_basic.py

@@ -48,7 +48,7 @@ def cleanup_db(conn, schema_base, schema_main):
     cur.execute("COMMIT")
 
 
-def init_sync_from_geopackage(mc, project_name, source_gpkg_path):
+def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[]):
     """
     Initialize sync from given GeoPackage file:
     - (re)create Mergin Maps project with the file
@@ -75,13 +75,18 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path):
 
     # prepare dbsync config
     # patch config to fit testing purposes
+    if ignored_tables:
+        connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg", "skip_tables":ignored_tables}
+    else:
+        connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"}
+
     config.update({
         'GEODIFF_EXE': GEODIFF_EXE,
         'WORKING_DIR': sync_project_dir,
         'MERGIN__USERNAME': API_USER,
         'MERGIN__PASSWORD': USER_PWD,
         'MERGIN__URL': SERVER_URL,
-        'CONNECTIONS': [{"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"}],
+        'CONNECTIONS': [connection],
     })
 
     dbsync_init(mc, from_gpkg=True)
@@ -320,3 +325,38 @@ def test_basic_both(mc):
 
     print("---")
     dbsync_status(mc)
+
+def test_init_with_skip(mc):
+    project_name = 'test_init_skip'
+    source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base_2tables.gpkg')
+    project_dir = os.path.join(TMP_DIR, project_name + '_work')
+    db_schema_main = project_name + '_main'
+    db_schema_base = project_name + '_base'
+
+    init_sync_from_geopackage(mc, project_name, source_gpkg_path, ["lines"])
+
+    # test that database schemas does not have ignored table
+    conn = psycopg2.connect(DB_CONNINFO)
+    cur = conn.cursor()
+    cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE  schemaname = '{db_schema_main}' AND tablename = 'lines');")
+    assert cur.fetchone()[0] == False
+    cur.execute(f"SELECT count(*) from {db_schema_main}.points")
+    assert cur.fetchone()[0] == 0
+
+    # run again, nothing should change
+    dbsync_init(mc, from_gpkg=True)
+    cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE  schemaname = '{db_schema_main}' AND tablename = 'lines');")
+    assert cur.fetchone()[0] == False
+    cur.execute(f"SELECT count(*) from {db_schema_main}.points")
+    assert cur.fetchone()[0] == 0
+
+    # make change in GPKG and push to server to create pending changes, it should pass but not sync
+    shutil.copy(os.path.join(TEST_DATA_DIR, 'modified_all.gpkg'), os.path.join(project_dir, 'test_sync.gpkg'))
+    mc.push_project(project_dir)
+
+    # pull server changes to db to make sure only points table is updated
+    dbsync_pull(mc)
+    cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE  schemaname = '{db_schema_main}' AND tablename = 'lines');")
+    assert cur.fetchone()[0] == False
+    cur.execute(f"SELECT count(*) from {db_schema_main}.points")
+    assert cur.fetchone()[0] == 4

test/test_config.py

@@ -66,3 +66,12 @@ def test_config():
         config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]})
         validate_config(config)
 
+    _reset_config()
+    with pytest.raises(ConfigError, match="Config error: Ignored tables parameter should be a list"):
+        config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables":""}]})
+        validate_config(config)
+
+    _reset_config()
+    with pytest.raises(ConfigError, match="Config error: Ignored tables list can not be empty"):
+        config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "skip_tables":[]}]})
+        validate_config(config)