From d5170c7dae08ee78f7fa4c7e1b556408ac467f94 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 17 Mar 2023 13:31:36 +0100 Subject: [PATCH 01/28] function that adds quotes to schema name if the schema name contains upper case letter --- dbsync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbsync.py b/dbsync.py index 51ccedf..35585b3 100644 --- a/dbsync.py +++ b/dbsync.py @@ -34,6 +34,12 @@ class DbSyncError(Exception): pass +def _add_quotes_to_schema_name(schema: str) -> str: + if any(ele.isupper() for ele in schema): + schema = f'"{schema}"' + return schema + + def _tables_list_to_string(tables): return ";".join(tables) From 257ee6036494a39cdf504496f8053422e2eb3c90 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 17 Mar 2023 13:41:03 +0100 Subject: [PATCH 02/28] needs string with " if upper case letters, psycopg2 takes care of adding ' --- dbsync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbsync.py b/dbsync.py index 35585b3..58dd76c 100644 --- a/dbsync.py +++ b/dbsync.py @@ -236,6 +236,7 @@ def _set_db_project_comment(conn, schema, project_name, version, project_id=None def _get_db_project_comment(conn, schema): """ Get Mergin Maps project name and its current version in db schema""" cur = conn.cursor() + schema = _add_quotes_to_schema_name(schema) cur.execute("SELECT obj_description(%s::regnamespace, 'pg_namespace')", (schema, )) res = cur.fetchone()[0] try: From 2fc60765a2c951db53cd59657a91969bf5830b3e Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 17 Mar 2023 13:45:52 +0100 Subject: [PATCH 03/28] add workspace --- test/test_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_basic.py b/test/test_basic.py index fe08890..5126067 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -17,6 +17,7 @@ SERVER_URL = os.environ.get('TEST_MERGIN_URL') API_USER = os.environ.get('TEST_API_USERNAME') USER_PWD = os.environ.get('TEST_API_PASSWORD') +WORKSPACE = os.environ.get('TEST_API_WORKSPACE') TMP_DIR = tempfile.gettempdir() TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') From 009eae9ed3b7920b028a8b66a5364ecbcf362c4d Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 17 Mar 2023 13:47:14 +0100 Subject: [PATCH 04/28] parametrize tests, so that they run on two project names - adding problematic project name with upper case letter --- test/test_basic.py | 96 ++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/test/test_basic.py b/test/test_basic.py index 5126067..4255a20 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -10,7 +10,7 @@ from mergin import MerginClient, ClientError from dbsync import dbsync_init, dbsync_pull, dbsync_push, dbsync_status, config, DbSyncError, _geodiff_make_copy, \ - _get_db_project_comment, _get_mergin_project, _get_project_id, _validate_local_project_id, config + _get_db_project_comment, _get_mergin_project, _get_project_id, _validate_local_project_id, config, _add_quotes_to_schema_name GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE') DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO') @@ -45,8 +45,8 @@ def cleanup(mc, project, dirs): def cleanup_db(conn, schema_base, schema_main): """ Removes test schemas from previous tests """ cur = conn.cursor() - cur.execute("DROP SCHEMA IF EXISTS {} CASCADE".format(schema_base)) - cur.execute("DROP SCHEMA IF EXISTS {} CASCADE".format(schema_main)) + cur.execute("DROP SCHEMA IF EXISTS {} CASCADE".format(_add_quotes_to_schema_name(schema_base))) + cur.execute("DROP SCHEMA IF EXISTS {} CASCADE".format(_add_quotes_to_schema_name(schema_main))) cur.execute("COMMIT") @@ -57,7 +57,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables - (re)create local project working directory and sync directory - configure DB sync and let it do the init (make copies to the database) """ - full_project_name = API_USER + "/" + project_name + full_project_name = WORKSPACE + "/" + project_name project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync') # used by dbsync db_schema_main = project_name + '_main' @@ -69,7 +69,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables cleanup_db(conn, db_schema_base, db_schema_main) # prepare a new Mergin Maps project - mc.create_project(project_name) + mc.create_project(project_name, namespace=WORKSPACE) mc.download_project(full_project_name, project_dir) shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg')) for extra_filepath in extra_init_files: @@ -97,12 +97,12 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables dbsync_init(mc, from_gpkg=True) -def test_init_from_gpkg(mc): - project_name = 'test_init' +@pytest.mark.parametrize("project_name", ['test_init', 'Test_init']) +def test_init_from_gpkg(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') - db_schema_main = project_name + '_main' - db_schema_base = project_name + '_base' + db_schema_main = _add_quotes_to_schema_name(project_name + '_main') + db_schema_base = _add_quotes_to_schema_name(project_name + '_base') init_sync_from_geopackage(mc, project_name, source_gpkg_path) @@ -115,7 +115,7 @@ def test_init_from_gpkg(mc): dbsync_init(mc, from_gpkg=True) cur.execute(f"SELECT count(*) from {db_schema_main}.simple") assert cur.fetchone()[0] == 3 - db_proj_info = _get_db_project_comment(conn, db_schema_base) + db_proj_info = _get_db_project_comment(conn, project_name + '_base') assert db_proj_info["name"] == config.connections[0].mergin_project assert db_proj_info["version"] == 'v1' @@ -137,7 +137,7 @@ def test_init_from_gpkg(mc): dbsync_init(mc, from_gpkg=True) cur.execute(f"SELECT count(*) from {db_schema_main}.simple") assert cur.fetchone()[0] == 3 - db_proj_info = _get_db_project_comment(conn, db_schema_base) + db_proj_info = _get_db_project_comment(conn, project_name + '_base') assert db_proj_info["version"] == 'v1' # let's remove local working dir and download different version from server to mimic versions mismatch @@ -145,14 +145,14 @@ def test_init_from_gpkg(mc): mc.download_project(config.connections[0].mergin_project, config.working_dir, 'v2') # run init again, it should handle local working dir properly (e.g. download correct version) and pass but not sync dbsync_init(mc, from_gpkg=True) - db_proj_info = _get_db_project_comment(conn, db_schema_base) + db_proj_info = _get_db_project_comment(conn, project_name + "_base") assert db_proj_info["version"] == 'v1' # pull server changes to db to make sure we can sync again dbsync_pull(mc) cur.execute(f"SELECT count(*) from {db_schema_main}.simple") assert cur.fetchone()[0] == 4 - db_proj_info = _get_db_project_comment(conn, db_schema_base) + db_proj_info = _get_db_project_comment(conn, project_name + "_base") assert db_proj_info["version"] == 'v2' # update some feature from 'modified' db to create mismatch with src geopackage, it should pass but not sync @@ -174,7 +174,7 @@ def test_init_from_gpkg(mc): mc.pull_project(project_dir) gpkg_cur.execute(f"SELECT * FROM simple WHERE fid={fid}") assert gpkg_cur.fetchone()[3] == 100 - db_proj_info = _get_db_project_comment(conn, db_schema_base) + db_proj_info = _get_db_project_comment(conn, project_name + "_base") assert db_proj_info["version"] == 'v3' # update some feature from 'base' db to create mismatch with src geopackage and modified @@ -196,8 +196,8 @@ def test_init_from_gpkg(mc): assert "There are pending changes in the local directory - that should never happen" in str(err.value) -def test_init_from_gpkg_with_incomplete_dir(mc): - project_name = 'test_init' +@pytest.mark.parametrize("project_name", ['test_init', 'Test_init']) +def test_init_from_gpkg_with_incomplete_dir(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') init_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync', project_name) init_sync_from_geopackage(mc, project_name, source_gpkg_path) @@ -209,15 +209,17 @@ def test_init_from_gpkg_with_incomplete_dir(mc): assert os.listdir(init_project_dir) == ['test_sync.gpkg', '.mergin'] -def test_basic_pull(mc): +@pytest.mark.parametrize("project_name", ['test_sync_pull', 'Test_Sync_Pull']) +def test_basic_pull(mc: MerginClient, project_name: str): """ Test initialization and one pull from Mergin Maps to DB 1. create a Mergin Maps project using py-client with a testing gpkg 2. run init, check that everything is fine 3. make change in gpkg (copy new version), check everything is fine """ + project_name_main = _add_quotes_to_schema_name(project_name + "_main") + project_name_base = _add_quotes_to_schema_name(project_name + "_base") - project_name = 'test_sync_pull' source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory @@ -227,7 +229,7 @@ def test_basic_pull(mc): # test that database schemas are created + tables are populated cur = conn.cursor() - cur.execute("SELECT count(*) from test_sync_pull_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 3 # make change in GPKG and push @@ -239,19 +241,20 @@ def test_basic_pull(mc): # check that a feature has been inserted cur = conn.cursor() - cur.execute("SELECT count(*) from test_sync_pull_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 4 - db_proj_info = _get_db_project_comment(conn, 'test_sync_pull_base') + db_proj_info = _get_db_project_comment(conn, project_name + "_base") assert db_proj_info["version"] == 'v2' print("---") dbsync_status(mc) -def test_basic_push(mc): +@pytest.mark.parametrize("project_name", ['test_sync_push', 'Test_Sync_Push']) +def test_basic_push(mc: MerginClient, project_name: str): """ Initialize a project and test push of a new row from PostgreSQL to Mergin Maps""" + project_name_main = _add_quotes_to_schema_name(project_name + "_main") - project_name = 'test_sync_push' source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory @@ -261,19 +264,19 @@ def test_basic_push(mc): # test that database schemas are created + tables are populated cur = conn.cursor() - cur.execute("SELECT count(*) from test_sync_push_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 3 # make a change in PostgreSQL cur = conn.cursor() - cur.execute("INSERT INTO test_sync_push_main.simple (name, rating) VALUES ('insert in postgres', 123)") + cur.execute(f"INSERT INTO {project_name_main}.simple (name, rating) VALUES ('insert in postgres', 123)") cur.execute("COMMIT") - cur.execute("SELECT count(*) from test_sync_push_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 4 # push the change from DB to PostgreSQL dbsync_push(mc) - db_proj_info = _get_db_project_comment(conn, 'test_sync_push_base') + db_proj_info = _get_db_project_comment(conn, project_name + "_base") assert db_proj_info["version"] == 'v2' # pull new version of the project to the work project directory @@ -289,13 +292,14 @@ def test_basic_push(mc): dbsync_status(mc) -def test_basic_both(mc): +@pytest.mark.parametrize("project_name", ['test_sync_both', 'Test_Sync_Both']) +def test_basic_both(mc: MerginClient, project_name: str): """ Initializes a sync project and does both a change in Mergin Maps and in the database, and lets DB sync handle it: changes in PostgreSQL need to be rebased on top of changes in Mergin Maps server. """ + project_name_main = _add_quotes_to_schema_name(project_name + "_main") - project_name = 'test_sync_both' source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory @@ -305,7 +309,7 @@ def test_basic_both(mc): # test that database schemas are created + tables are populated cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name}_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 3 # make change in GPKG and push @@ -314,17 +318,17 @@ def test_basic_both(mc): # make a change in PostgreSQL cur = conn.cursor() - cur.execute(f"INSERT INTO {project_name}_main.simple (name, rating) VALUES ('insert in postgres', 123)") + cur.execute(f"INSERT INTO {project_name_main}.simple (name, rating) VALUES ('insert in postgres', 123)") cur.execute("COMMIT") - cur.execute(f"SELECT count(*) from {project_name}_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 4 # first pull changes from Mergin Maps to DB (+rebase changes in DB) and then push the changes from DB to Mergin Maps dbsync_pull(mc) - db_proj_info = _get_db_project_comment(conn, 'test_sync_both_base') + db_proj_info = _get_db_project_comment(conn, project_name + '_base') assert db_proj_info["version"] == 'v2' dbsync_push(mc) - db_proj_info = _get_db_project_comment(conn, 'test_sync_both_base') + db_proj_info = _get_db_project_comment(conn, project_name + '_base') assert db_proj_info["version"] == 'v3' # pull new version of the project to the work project directory @@ -338,19 +342,20 @@ def test_basic_both(mc): # check that the insert has been applied to the DB cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name}_main.simple") + cur.execute(f"SELECT count(*) from {project_name_main}.simple") assert cur.fetchone()[0] == 5 print("---") dbsync_status(mc) -def test_init_with_skip(mc): - project_name = 'test_init_skip' +@pytest.mark.parametrize("project_name", ['test_init_skip', 'Test_Init_Skip']) +def test_init_with_skip(mc: MerginClient, project_name: str): + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base_2tables.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') - db_schema_main = project_name + '_main' - db_schema_base = project_name + '_base' + db_schema_main = _add_quotes_to_schema_name(project_name + '_main') + db_schema_base = _add_quotes_to_schema_name(project_name + '_base') init_sync_from_geopackage(mc, project_name, source_gpkg_path, ["lines"]) @@ -381,8 +386,9 @@ def test_init_with_skip(mc): assert cur.fetchone()[0] == 4 -def test_with_local_changes(mc): - project_name = 'test_local_changes' +@pytest.mark.parametrize("project_name", ['test_local_changes', 'Test_Local_Changes']) +def test_with_local_changes(mc: MerginClient, project_name: str): + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') extra_files = [os.path.join(TEST_DATA_DIR, f) for f in ["note_1.txt", "note_3.txt", "modified_all.gpkg"]] dbsync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync', @@ -412,17 +418,17 @@ def test_with_local_changes(mc): assert any(local_changes.values()) is False dbsync_status(mc) +@pytest.mark.parametrize("project_name", ['test_recreated_project_ids', 'Test_Recreated_Project_Ids']) +def test_recreated_project_ids(mc: MerginClient, project_name: str): -def test_recreated_project_ids(mc): - project_name = 'test_recreated_project_ids' source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory - full_project_name = API_USER + "/" + project_name + full_project_name = WORKSPACE + "/" + project_name init_sync_from_geopackage(mc, project_name, source_gpkg_path) # delete remote project mc.delete_project(full_project_name) # recreate project with the same name - mc.create_project(project_name) + mc.create_project(project_name, namespace=WORKSPACE) # comparing project IDs after recreating it with the same name mp = _get_mergin_project(project_dir) local_project_id = _get_project_id(mp) From 9b415dbc1e05467df711b433ad52b2108fe06232 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 17 Mar 2023 16:00:16 +0100 Subject: [PATCH 05/28] add test info about workspace --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c9f1638..bd440bc 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ To run automatic tests: export TEST_MERGIN_URL= # testing server export TEST_API_USERNAME= export TEST_API_PASSWORD= + export TEST_API_WORKSPACE= pytest-3 test/ From bacdf835e926ea00e53f7fafcae74bd091666c21 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 08:45:36 +0100 Subject: [PATCH 06/28] update test name --- test/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_basic.py b/test/test_basic.py index 4255a20..bc70728 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -97,7 +97,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables dbsync_init(mc, from_gpkg=True) -@pytest.mark.parametrize("project_name", ['test_init', 'Test_init']) +@pytest.mark.parametrize("project_name", ['test_init', 'Test_Init']) def test_init_from_gpkg(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') From 0e17527c759d31c046f42d4d0683e4ec60921d03 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 10:17:19 +0100 Subject: [PATCH 07/28] fix tests to properly escape schema names --- test/test_basic.py | 92 +++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/test/test_basic.py b/test/test_basic.py index bc70728..1f5b69b 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -7,6 +7,7 @@ import tempfile import psycopg2 +from psycopg2 import sql from mergin import MerginClient, ClientError from dbsync import dbsync_init, dbsync_pull, dbsync_push, dbsync_status, config, DbSyncError, _geodiff_make_copy, \ @@ -45,8 +46,8 @@ def cleanup(mc, project, dirs): def cleanup_db(conn, schema_base, schema_main): """ Removes test schemas from previous tests """ cur = conn.cursor() - cur.execute("DROP SCHEMA IF EXISTS {} CASCADE".format(_add_quotes_to_schema_name(schema_base))) - cur.execute("DROP SCHEMA IF EXISTS {} CASCADE".format(_add_quotes_to_schema_name(schema_main))) + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base))) + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main))) cur.execute("COMMIT") @@ -96,37 +97,36 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables dbsync_init(mc, from_gpkg=True) - @pytest.mark.parametrize("project_name", ['test_init', 'Test_Init']) def test_init_from_gpkg(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') - db_schema_main = _add_quotes_to_schema_name(project_name + '_main') - db_schema_base = _add_quotes_to_schema_name(project_name + '_base') + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' init_sync_from_geopackage(mc, project_name, source_gpkg_path) # test that database schemas are created + tables are populated conn = psycopg2.connect(DB_CONNINFO) cur = conn.cursor() - cur.execute(f"SELECT count(*) from {db_schema_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(conn)) assert cur.fetchone()[0] == 3 # run again, nothing should change dbsync_init(mc, from_gpkg=True) - cur.execute(f"SELECT count(*) from {db_schema_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(conn)) assert cur.fetchone()[0] == 3 - db_proj_info = _get_db_project_comment(conn, project_name + '_base') + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["name"] == config.connections[0].mergin_project assert db_proj_info["version"] == 'v1' # rename base schema to mimic some mismatch - cur.execute(f"ALTER SCHEMA {db_schema_base} RENAME TO schema_tmp") + cur.execute(sql.SQL("ALTER SCHEMA {} RENAME TO schema_tmp").format(sql.Identifier(db_schema_base)).as_string(conn)) conn.commit() with pytest.raises(DbSyncError) as err: dbsync_init(mc, from_gpkg=True) assert "The 'modified' schema exists but the base schema is missing" in str(err.value) # and revert back - cur.execute(f"ALTER SCHEMA schema_tmp RENAME TO {db_schema_base}") + cur.execute(sql.SQL("ALTER SCHEMA schema_tmp RENAME TO {}").format(sql.Identifier(db_schema_base)).as_string(conn)) conn.commit() # make change in GPKG and push to server to create pending changes, it should pass but not sync @@ -135,9 +135,9 @@ def test_init_from_gpkg(mc: MerginClient, project_name: str): # remove local copy of project (to mimic loss at docker restart) shutil.rmtree(config.working_dir) dbsync_init(mc, from_gpkg=True) - cur.execute(f"SELECT count(*) from {db_schema_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(conn)) assert cur.fetchone()[0] == 3 - db_proj_info = _get_db_project_comment(conn, project_name + '_base') + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v1' # let's remove local working dir and download different version from server to mimic versions mismatch @@ -145,23 +145,23 @@ def test_init_from_gpkg(mc: MerginClient, project_name: str): mc.download_project(config.connections[0].mergin_project, config.working_dir, 'v2') # run init again, it should handle local working dir properly (e.g. download correct version) and pass but not sync dbsync_init(mc, from_gpkg=True) - db_proj_info = _get_db_project_comment(conn, project_name + "_base") + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v1' # pull server changes to db to make sure we can sync again dbsync_pull(mc) - cur.execute(f"SELECT count(*) from {db_schema_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(conn)) assert cur.fetchone()[0] == 4 - db_proj_info = _get_db_project_comment(conn, project_name + "_base") + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v2' # update some feature from 'modified' db to create mismatch with src geopackage, it should pass but not sync fid = 1 - cur.execute(f"SELECT * from {db_schema_main}.simple WHERE fid={fid}") + cur.execute(sql.SQL("SELECT * from {}.simple WHERE fid=%s").format(sql.Identifier(db_schema_main)), (fid,)) old_value = cur.fetchone()[3] - cur.execute(f"UPDATE {db_schema_main}.simple SET rating=100 WHERE fid={fid}") + cur.execute(sql.SQL("UPDATE {}.simple SET rating=100 WHERE fid=%s").format(sql.Identifier(db_schema_main)), (fid,)) conn.commit() - cur.execute(f"SELECT * from {db_schema_main}.simple WHERE fid={fid}") + cur.execute(sql.SQL("SELECT * from {}.simple WHERE fid=%s").format(sql.Identifier(db_schema_main)), (fid,)) assert cur.fetchone()[3] == 100 dbsync_init(mc, from_gpkg=True) # check geopackage has not been modified - after init we are not synced! @@ -172,18 +172,18 @@ def test_init_from_gpkg(mc: MerginClient, project_name: str): # push db changes to server (and download new version to local working dir) to make sure we can sync again dbsync_push(mc) mc.pull_project(project_dir) - gpkg_cur.execute(f"SELECT * FROM simple WHERE fid={fid}") + gpkg_cur.execute(f"SELECT * FROM simple WHERE fid ={fid}") assert gpkg_cur.fetchone()[3] == 100 - db_proj_info = _get_db_project_comment(conn, project_name + "_base") + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v3' # update some feature from 'base' db to create mismatch with src geopackage and modified - cur.execute(f"SELECT * from {db_schema_base}.simple") + cur.execute(sql.SQL("SELECT * from {}.simple").format(sql.Identifier(db_schema_base))) fid = cur.fetchone()[0] old_value = cur.fetchone()[3] - cur.execute(f"UPDATE {db_schema_base}.simple SET rating=100 WHERE fid={fid}") + cur.execute(sql.SQL("UPDATE {}.simple SET rating=100 WHERE fid=%s").format(sql.Identifier(db_schema_base)), (fid,)) conn.commit() - cur.execute(f"SELECT * from {db_schema_base}.simple WHERE fid={fid}") + cur.execute(sql.SQL("SELECT * from {}.simple WHERE fid=%s").format(sql.Identifier(db_schema_base)), (fid,)) assert cur.fetchone()[3] == 100 with pytest.raises(DbSyncError) as err: dbsync_init(mc, from_gpkg=True) @@ -217,8 +217,7 @@ def test_basic_pull(mc: MerginClient, project_name: str): 2. run init, check that everything is fine 3. make change in gpkg (copy new version), check everything is fine """ - project_name_main = _add_quotes_to_schema_name(project_name + "_main") - project_name_base = _add_quotes_to_schema_name(project_name + "_base") + db_schema_main = project_name + "_main" source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory @@ -229,7 +228,7 @@ def test_basic_pull(mc: MerginClient, project_name: str): # test that database schemas are created + tables are populated cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 3 # make change in GPKG and push @@ -241,7 +240,7 @@ def test_basic_pull(mc: MerginClient, project_name: str): # check that a feature has been inserted cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format((sql.Identifier(db_schema_main)))) assert cur.fetchone()[0] == 4 db_proj_info = _get_db_project_comment(conn, project_name + "_base") assert db_proj_info["version"] == 'v2' @@ -253,7 +252,7 @@ def test_basic_pull(mc: MerginClient, project_name: str): @pytest.mark.parametrize("project_name", ['test_sync_push', 'Test_Sync_Push']) def test_basic_push(mc: MerginClient, project_name: str): """ Initialize a project and test push of a new row from PostgreSQL to Mergin Maps""" - project_name_main = _add_quotes_to_schema_name(project_name + "_main") + db_schema_main = project_name + "_main" source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory @@ -264,14 +263,14 @@ def test_basic_push(mc: MerginClient, project_name: str): # test that database schemas are created + tables are populated cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 3 # make a change in PostgreSQL cur = conn.cursor() - cur.execute(f"INSERT INTO {project_name_main}.simple (name, rating) VALUES ('insert in postgres', 123)") + cur.execute(sql.SQL("INSERT INTO {}.simple (name, rating) VALUES ('insert in postgres', 123)").format(sql.Identifier(db_schema_main))) cur.execute("COMMIT") - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 4 # push the change from DB to PostgreSQL @@ -298,7 +297,8 @@ def test_basic_both(mc: MerginClient, project_name: str): and lets DB sync handle it: changes in PostgreSQL need to be rebased on top of changes in Mergin Maps server. """ - project_name_main = _add_quotes_to_schema_name(project_name + "_main") + db_schema_main = project_name + "_main" + db_schema_base = project_name + "_base" source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory @@ -309,7 +309,7 @@ def test_basic_both(mc: MerginClient, project_name: str): # test that database schemas are created + tables are populated cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 3 # make change in GPKG and push @@ -318,17 +318,17 @@ def test_basic_both(mc: MerginClient, project_name: str): # make a change in PostgreSQL cur = conn.cursor() - cur.execute(f"INSERT INTO {project_name_main}.simple (name, rating) VALUES ('insert in postgres', 123)") + cur.execute(sql.SQL("INSERT INTO {}.simple (name, rating) VALUES ('insert in postgres', 123)").format(sql.Identifier(db_schema_main))) cur.execute("COMMIT") - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 4 # first pull changes from Mergin Maps to DB (+rebase changes in DB) and then push the changes from DB to Mergin Maps dbsync_pull(mc) - db_proj_info = _get_db_project_comment(conn, project_name + '_base') + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v2' dbsync_push(mc) - db_proj_info = _get_db_project_comment(conn, project_name + '_base') + db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v3' # pull new version of the project to the work project directory @@ -342,7 +342,7 @@ def test_basic_both(mc: MerginClient, project_name: str): # check that the insert has been applied to the DB cur = conn.cursor() - cur.execute(f"SELECT count(*) from {project_name_main}.simple") + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 5 print("---") @@ -354,24 +354,24 @@ def test_init_with_skip(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base_2tables.gpkg') project_dir = os.path.join(TMP_DIR, project_name + '_work') - db_schema_main = _add_quotes_to_schema_name(project_name + '_main') - db_schema_base = _add_quotes_to_schema_name(project_name + '_base') + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' init_sync_from_geopackage(mc, project_name, source_gpkg_path, ["lines"]) # test that database schemas does not have ignored table conn = psycopg2.connect(DB_CONNINFO) cur = conn.cursor() - cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + cur.execute(sql.SQL("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{}' AND tablename = 'lines');").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == False - cur.execute(f"SELECT count(*) from {db_schema_main}.points") + cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 0 # run again, nothing should change dbsync_init(mc, from_gpkg=True) - cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + cur.execute(sql.SQL("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{}' AND tablename = 'lines');").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == False - cur.execute(f"SELECT count(*) from {db_schema_main}.points") + cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 0 # make change in GPKG and push to server to create pending changes, it should pass but not sync @@ -380,9 +380,9 @@ def test_init_with_skip(mc: MerginClient, project_name: str): # pull server changes to db to make sure only points table is updated dbsync_pull(mc) - cur.execute(f"SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{db_schema_main}' AND tablename = 'lines');") + cur.execute(sql.SQL("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{}' AND tablename = 'lines');").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == False - cur.execute(f"SELECT count(*) from {db_schema_main}.points") + cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 4 From dca2b59661da5e64631504a050b9360e6cb6e5d9 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 10:18:37 +0100 Subject: [PATCH 08/28] add to avoid accidental commits --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index afcc796..b01f141 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ config.ini mergin .coverage htmlcov +.vscode +.env \ No newline at end of file From 42d78c2671f9e05269a707863a94d18f44bd34ad Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 11:30:01 +0100 Subject: [PATCH 09/28] _drop_schema function --- dbsync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbsync.py b/dbsync.py index 58dd76c..e90bb10 100644 --- a/dbsync.py +++ b/dbsync.py @@ -60,6 +60,12 @@ def _check_has_sync_file(file_path): raise DbSyncError("The output GPKG file does not exist: " + file_path) +def _drop_schema(conn, schema_name: str) -> None: + cur = conn.cursor() + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema_name))) + conn.commit() + + def _check_schema_exists(conn, schema_name): cur = conn.cursor() cur.execute("SELECT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = %s)", (schema_name,)) From d224d640f89eb58f249f00ea02d5fef0667634ab Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 11:30:22 +0100 Subject: [PATCH 10/28] drop schema if its problematic --- dbsync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbsync.py b/dbsync.py index e90bb10..fb88a8f 100644 --- a/dbsync.py +++ b/dbsync.py @@ -598,6 +598,7 @@ def init(conn_cfg, mc, from_gpkg=True): # this is not a first run of db-sync init db_proj_info = _get_db_project_comment(conn, conn_cfg.base) if not db_proj_info: + _drop_schema(conn, conn_cfg.base) raise DbSyncError("Base schema exists but missing which project it belongs to") if "error" in db_proj_info: changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, @@ -674,8 +675,10 @@ def init(conn_cfg, mc, from_gpkg=True): print("The GPKG file, base and modified schemas are already initialized and in sync") return # nothing to do elif modified_schema_exists: + _drop_schema(conn, conn_cfg.modified) raise DbSyncError(f"The 'modified' schema exists but the base schema is missing: {conn_cfg.base}") elif base_schema_exists: + _drop_schema(conn, conn_cfg.base) raise DbSyncError(f"The base schema exists but the modified schema is missing: {conn_cfg.modified}") # initialize: we have an existing GeoPackage in our Mergin Maps project and we want to initialize database From ac2a08eab821e2751bb85ac9c08ba9305ec2eaaf Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 11:30:58 +0100 Subject: [PATCH 11/28] separate tests for problems with schema --- test/test_basic.py | 54 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/test/test_basic.py b/test/test_basic.py index 1f5b69b..53adbf4 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -119,16 +119,6 @@ def test_init_from_gpkg(mc: MerginClient, project_name: str): assert db_proj_info["name"] == config.connections[0].mergin_project assert db_proj_info["version"] == 'v1' - # rename base schema to mimic some mismatch - cur.execute(sql.SQL("ALTER SCHEMA {} RENAME TO schema_tmp").format(sql.Identifier(db_schema_base)).as_string(conn)) - conn.commit() - with pytest.raises(DbSyncError) as err: - dbsync_init(mc, from_gpkg=True) - assert "The 'modified' schema exists but the base schema is missing" in str(err.value) - # and revert back - cur.execute(sql.SQL("ALTER SCHEMA schema_tmp RENAME TO {}").format(sql.Identifier(db_schema_base)).as_string(conn)) - conn.commit() - # make change in GPKG and push to server to create pending changes, it should pass but not sync shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), os.path.join(project_dir, 'test_sync.gpkg')) mc.push_project(project_dir) @@ -196,6 +186,50 @@ def test_init_from_gpkg(mc: MerginClient, project_name: str): assert "There are pending changes in the local directory - that should never happen" in str(err.value) +@pytest.mark.parametrize("project_name", ['test_init_missing_schema', 'Test_Init_Missing_Schema']) +def test_init_from_gpkg_missing_schema(mc: MerginClient, project_name: str): + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') + + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + # drop base schema to mimic some mismatch + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_base"))) + conn.commit() + with pytest.raises(DbSyncError) as err: + dbsync_init(mc, from_gpkg=True) + assert "The 'modified' schema exists but the base schema is missing" in str(err.value) + + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + + # drop main schema to mimic some mismatch + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_main"))) + conn.commit() + with pytest.raises(DbSyncError) as err: + dbsync_init(mc, from_gpkg=True) + assert "The base schema exists but the modified schema is missing" in str(err.value) + + +@pytest.mark.parametrize("project_name", ['test_init_missing_comment', 'Test_Init_Missing_Comment']) +def test_init_from_gpkg_missing_comment(mc: MerginClient, project_name: str): + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') + + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + # drop base schema to mimic some mismatch + query = sql.SQL("COMMENT ON SCHEMA {} IS %s").format(sql.Identifier((project_name + "_base"))) + cur.execute(query.as_string(conn), ("",)) + conn.commit() + + with pytest.raises(DbSyncError) as err: + dbsync_init(mc, from_gpkg=True) + assert "Base schema exists but missing which project it belongs to" in str(err.value) + @pytest.mark.parametrize("project_name", ['test_init', 'Test_init']) def test_init_from_gpkg_with_incomplete_dir(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') From 3142ecd0347f6bb3bd1a4aafb3325d895aacf2d6 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 11:50:31 +0100 Subject: [PATCH 12/28] tests should explicitly check for schema presence/absence --- test/test_basic.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/test/test_basic.py b/test/test_basic.py index 53adbf4..7cc2821 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -190,11 +190,21 @@ def test_init_from_gpkg(mc: MerginClient, project_name: str): def test_init_from_gpkg_missing_schema(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') + db_schema_base = project_name + "_base" + db_schema_main = project_name + "_main" + init_sync_from_geopackage(mc, project_name, source_gpkg_path) conn = psycopg2.connect(DB_CONNINFO) cur = conn.cursor() + # sql query for schema + sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_main}'" + + # check that schema exists + cur.execute(sql_cmd) + cur.fetchone()[0] == db_schema_main + # drop base schema to mimic some mismatch cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_base"))) conn.commit() @@ -202,8 +212,19 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient, project_name: str): dbsync_init(mc, from_gpkg=True) assert "The 'modified' schema exists but the base schema is missing" in str(err.value) + # check that schema does not exists anymore + cur.execute(sql_cmd) + cur.fetchone() is None + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + # sql query for schema + sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_base}'" + + # check that schema exists + cur.execute(sql_cmd) + cur.fetchone()[0] == db_schema_base + # drop main schema to mimic some mismatch cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_main"))) conn.commit() @@ -211,18 +232,29 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient, project_name: str): dbsync_init(mc, from_gpkg=True) assert "The base schema exists but the modified schema is missing" in str(err.value) + # check that schema does not exists anymore + cur.execute(sql_cmd) + cur.fetchone() is None @pytest.mark.parametrize("project_name", ['test_init_missing_comment', 'Test_Init_Missing_Comment']) def test_init_from_gpkg_missing_comment(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') + schema_name = project_name + "_base" init_sync_from_geopackage(mc, project_name, source_gpkg_path) conn = psycopg2.connect(DB_CONNINFO) cur = conn.cursor() + # sql query for schema + sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{schema_name}'" + + # check that schema exists + cur.execute(sql_cmd) + cur.fetchone()[0] == schema_name + # drop base schema to mimic some mismatch - query = sql.SQL("COMMENT ON SCHEMA {} IS %s").format(sql.Identifier((project_name + "_base"))) + query = sql.SQL("COMMENT ON SCHEMA {} IS %s").format(sql.Identifier(schema_name)) cur.execute(query.as_string(conn), ("",)) conn.commit() @@ -230,6 +262,10 @@ def test_init_from_gpkg_missing_comment(mc: MerginClient, project_name: str): dbsync_init(mc, from_gpkg=True) assert "Base schema exists but missing which project it belongs to" in str(err.value) + # check that schema does not exists anymore + cur.execute(sql_cmd) + cur.fetchone() is None + @pytest.mark.parametrize("project_name", ['test_init', 'Test_init']) def test_init_from_gpkg_with_incomplete_dir(mc: MerginClient, project_name: str): source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') From 332354cbda968fc2bfce6055ecf0583a00f8650c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 22 Mar 2023 14:48:39 +0100 Subject: [PATCH 13/28] bring back test for dropping schemata --- test/test_basic.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/test/test_basic.py b/test/test_basic.py index f54349e..9ab620d 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -477,3 +477,84 @@ def test_project_names(mc: MerginClient, project_name: str): assert gpkg_cur.fetchone()[3] == 100 db_proj_info = _get_db_project_comment(conn, db_schema_base) assert db_proj_info["version"] == 'v3' + + +def test_init_from_gpkg_missing_schema(mc: MerginClient): + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') + project_name = "test_init_missing_schema" + db_schema_base = project_name + "_base" + db_schema_main = project_name + "_main" + + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + # sql query for schema + sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_main}'" + + # check that schema exists + cur.execute(sql_cmd) + cur.fetchone()[0] == db_schema_main + + # drop base schema to mimic some mismatch + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_base"))) + conn.commit() + with pytest.raises(DbSyncError) as err: + dbsync_init(mc, from_gpkg=True) + assert "The 'modified' schema exists but the base schema is missing" in str(err.value) + + # check that schema does not exists anymore + cur.execute(sql_cmd) + cur.fetchone() is None + + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + + # sql query for schema + sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_base}'" + + # check that schema exists + cur.execute(sql_cmd) + cur.fetchone()[0] == db_schema_base + + # drop main schema to mimic some mismatch + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_main"))) + conn.commit() + with pytest.raises(DbSyncError) as err: + dbsync_init(mc, from_gpkg=True) + assert "The base schema exists but the modified schema is missing" in str(err.value) + + # check that schema does not exists anymore + cur.execute(sql_cmd) + cur.fetchone() is None + + +def test_init_from_gpkg_missing_comment(mc: MerginClient, project_name: str): + project_name = "test_init_missing_comment" + source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') + schema_name = project_name + "_base" + + init_sync_from_geopackage(mc, project_name, source_gpkg_path) + + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + # sql query for schema + sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{schema_name}'" + + # check that schema exists + cur.execute(sql_cmd) + cur.fetchone()[0] == schema_name + + # drop base schema to mimic some mismatch + query = sql.SQL("COMMENT ON SCHEMA {} IS %s").format(sql.Identifier(schema_name)) + cur.execute(query.as_string(conn), ("",)) + conn.commit() + + with pytest.raises(DbSyncError) as err: + dbsync_init(mc, from_gpkg=True) + assert "Base schema exists but missing which project it belongs to" in str(err.value) + + # check that schema does not exists anymore + cur.execute(sql_cmd) + cur.fetchone() is None \ No newline at end of file From 190df0efd643516f01e2c166df79138fda133a4f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 09:08:07 +0100 Subject: [PATCH 14/28] update messages --- dbsync.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/dbsync.py b/dbsync.py index 69983ea..e596c52 100644 --- a/dbsync.py +++ b/dbsync.py @@ -601,8 +601,9 @@ def init(conn_cfg, mc, from_gpkg=True): # this is not a first run of db-sync init db_proj_info = _get_db_project_comment(conn, conn_cfg.base) if not db_proj_info: - _drop_schema(conn, conn_cfg.base) - raise DbSyncError("Base schema exists but missing which project it belongs to") + raise DbSyncError("Base schema exists but missing which project it belongs to. " + f"Both schemata `{conn_cfg.base}` and `{conn_cfg.modified}` should be deleted " + "so that DB Sync can set up sync properly") if "error" in db_proj_info: changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, ignored_tables, @@ -678,11 +679,13 @@ def init(conn_cfg, mc, from_gpkg=True): print("The GPKG file, base and modified schemas are already initialized and in sync") return # nothing to do elif modified_schema_exists: - _drop_schema(conn, conn_cfg.modified) - raise DbSyncError(f"The 'modified' schema exists but the base schema is missing: {conn_cfg.base}") + raise DbSyncError(f"The 'modified' schema exists but the base schema is missing: {conn_cfg.base}. " + f"Schema `{conn_cfg.modified}` should be removed or renamed " + "so that DB Sync can set up sync properly.") elif base_schema_exists: - _drop_schema(conn, conn_cfg.base) - raise DbSyncError(f"The base schema exists but the modified schema is missing: {conn_cfg.modified}") + raise DbSyncError(f"The base schema exists but the modified schema is missing: {conn_cfg.modified}. " + f"Schema `{conn_cfg.base}` should be removed or renamed " + "so that DB Sync can set up sync properly.") # initialize: we have an existing GeoPackage in our Mergin Maps project and we want to initialize database print("The base and modified schemas do not exist yet, going to initialize them ...") @@ -721,9 +724,9 @@ def init(conn_cfg, mc, from_gpkg=True): if os.path.exists(gpkg_full_path) and base_schema_exists: # make sure output gpkg is in sync with db or fail summary_modified = _compare_datasets(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, - "sqlite", "", gpkg_full_path, ignored_tables) + "sqlite", "", gpkg_full_path, ignored_tables) summary_base = _compare_datasets(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, - "sqlite", "", gpkg_full_path, ignored_tables) + "sqlite", "", gpkg_full_path, ignored_tables) if len(summary_base): print(f"Local project version at {_get_project_version(work_dir)} and base schema at {db_proj_info['version']}") _print_changes_summary(summary_base, "Base schema changes:") From 9ec3609b2c2124352eef2acc3f1ff2e4801d297c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 10:15:45 +0100 Subject: [PATCH 15/28] update messages --- dbsync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbsync.py b/dbsync.py index e596c52..b58f8f8 100644 --- a/dbsync.py +++ b/dbsync.py @@ -602,7 +602,7 @@ def init(conn_cfg, mc, from_gpkg=True): db_proj_info = _get_db_project_comment(conn, conn_cfg.base) if not db_proj_info: raise DbSyncError("Base schema exists but missing which project it belongs to. " - f"Both schemata `{conn_cfg.base}` and `{conn_cfg.modified}` should be deleted " + f"Both schemata `{conn_cfg.base}` and `{conn_cfg.modified}` should be deleted in the database " "so that DB Sync can set up sync properly") if "error" in db_proj_info: changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, @@ -680,11 +680,11 @@ def init(conn_cfg, mc, from_gpkg=True): return # nothing to do elif modified_schema_exists: raise DbSyncError(f"The 'modified' schema exists but the base schema is missing: {conn_cfg.base}. " - f"Schema `{conn_cfg.modified}` should be removed or renamed " + f"Schema `{conn_cfg.modified}` should be removed or renamed in the database " "so that DB Sync can set up sync properly.") elif base_schema_exists: raise DbSyncError(f"The base schema exists but the modified schema is missing: {conn_cfg.modified}. " - f"Schema `{conn_cfg.base}` should be removed or renamed " + f"Schema `{conn_cfg.base}` should be removed or renamed in the database " "so that DB Sync can set up sync properly.") # initialize: we have an existing GeoPackage in our Mergin Maps project and we want to initialize database From 57afc69bc12802c094b22b293fb76d644f181c14 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 10:17:57 +0100 Subject: [PATCH 16/28] drop both schema if the init didnt go well --- dbsync.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbsync.py b/dbsync.py index b58f8f8..91de929 100644 --- a/dbsync.py +++ b/dbsync.py @@ -711,9 +711,8 @@ def init(conn_cfg, mc, from_gpkg=True): raise DbSyncError('Initialization of db-sync failed due to a bug in geodiff.\n ' 'Please report this problem to mergin-db-sync developers') except DbSyncError: - # add comment to base schema before throwing exception - _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, local_version, - error='Initialization of db-sync failed due to a bug in geodiff') + _drop_schema(conn_cfg.base) + _drop_schema(conn_cfg.modified) raise _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, local_version) @@ -767,8 +766,8 @@ def init(conn_cfg, mc, from_gpkg=True): raise DbSyncError('Initialization of db-sync failed due to a bug in geodiff.\n ' 'Please report this problem to mergin-db-sync developers') except DbSyncError: - _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, local_version, - error='Initialization of db-sync failed due to a bug in geodiff') + _drop_schema(conn_cfg.base) + _drop_schema(conn_cfg.modified) raise # upload gpkg to Mergin Maps (client takes care of storing metadata) From 59ecee972843933cb132a9176700f5ff685779ed Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 10:18:25 +0100 Subject: [PATCH 17/28] avoid issues, only drop schema if it exist --- dbsync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync.py b/dbsync.py index 91de929..b719288 100644 --- a/dbsync.py +++ b/dbsync.py @@ -65,7 +65,7 @@ def _check_has_sync_file(file_path): def _drop_schema(conn, schema_name: str) -> None: cur = conn.cursor() - cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema_name))) + cur.execute(sql.SQL("DROP SCHEMA IF EXIST {} CASCADE").format(sql.Identifier(schema_name))) conn.commit() From a362330c8ead71a3de3b7190cf7d2e5ecc347d5f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 10:49:20 +0100 Subject: [PATCH 18/28] fix test --- test/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_basic.py b/test/test_basic.py index ca0051b..b4ef10e 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -529,7 +529,7 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient): cur.fetchone() is None -def test_init_from_gpkg_missing_comment(mc: MerginClient, project_name: str): +def test_init_from_gpkg_missing_comment(mc: MerginClient): project_name = "test_init_missing_comment" source_gpkg_path = os.path.join(TEST_DATA_DIR, 'base.gpkg') schema_name = project_name + "_base" From a111a62993078d7fe8ae837b31b29902753825ca Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 11:24:36 +0100 Subject: [PATCH 19/28] simplify the sql queries --- test/test_basic.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/test/test_basic.py b/test/test_basic.py index b4ef10e..9b5c802 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -490,43 +490,33 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient): conn = psycopg2.connect(DB_CONNINFO) cur = conn.cursor() - # sql query for schema - sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_main}'" - - # check that schema exists - cur.execute(sql_cmd) - cur.fetchone()[0] == db_schema_main - # drop base schema to mimic some mismatch - cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_base"))) + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(db_schema_base))) conn.commit() + + # check that removed schema does not exists + cur.execute(f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_base}'") + cur.fetchone() is None + with pytest.raises(DbSyncError) as err: dbsync_init(mc, from_gpkg=True) assert "The 'modified' schema exists but the base schema is missing" in str(err.value) - - # check that schema does not exists anymore - cur.execute(sql_cmd) - cur.fetchone() is None + assert f"Schema `{db_schema_main}` should be removed or renamed" in str(err.value) init_sync_from_geopackage(mc, project_name, source_gpkg_path) - # sql query for schema - sql_cmd = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_base}'" - - # check that schema exists - cur.execute(sql_cmd) - cur.fetchone()[0] == db_schema_base - # drop main schema to mimic some mismatch - cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(project_name + "_main"))) + cur.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(db_schema_main))) conn.commit() + + # check that removed schema does not exists + cur.execute(f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{db_schema_main}'") + cur.fetchone() is None + with pytest.raises(DbSyncError) as err: dbsync_init(mc, from_gpkg=True) assert "The base schema exists but the modified schema is missing" in str(err.value) - - # check that schema does not exists anymore - cur.execute(sql_cmd) - cur.fetchone() is None + assert f"Schema `{db_schema_base}` should be removed or renamed" in str(err.value) def test_init_from_gpkg_missing_comment(mc: MerginClient): @@ -557,4 +547,4 @@ def test_init_from_gpkg_missing_comment(mc: MerginClient): # check that schema does not exists anymore cur.execute(sql_cmd) - cur.fetchone() is None \ No newline at end of file + cur.fetchone() is None From 86bf7ff718f3cd728367575a75c29cedd95e4cd0 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 11:34:34 +0100 Subject: [PATCH 20/28] fix error message --- dbsync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbsync.py b/dbsync.py index b719288..b645179 100644 --- a/dbsync.py +++ b/dbsync.py @@ -718,7 +718,8 @@ def init(conn_cfg, mc, from_gpkg=True): _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, local_version) else: if not modified_schema_exists: - raise DbSyncError("The 'modified' schema does not exist: " + conn_cfg.modified) + raise DbSyncError(f"The 'modified' schema does not exist: {conn_cfg.modified}. " + "This schema is necessary if initialization should be done from database (parameter `init-from-db`).") if os.path.exists(gpkg_full_path) and base_schema_exists: # make sure output gpkg is in sync with db or fail From a535081fa65d0fa08c34885b81334f809024370f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 15:52:36 +0100 Subject: [PATCH 21/28] drop only the base schema --- dbsync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbsync.py b/dbsync.py index b645179..635987c 100644 --- a/dbsync.py +++ b/dbsync.py @@ -768,7 +768,6 @@ def init(conn_cfg, mc, from_gpkg=True): 'Please report this problem to mergin-db-sync developers') except DbSyncError: _drop_schema(conn_cfg.base) - _drop_schema(conn_cfg.modified) raise # upload gpkg to Mergin Maps (client takes care of storing metadata) From 9955c039489e5352e8b3fe77bdd9256567181ea9 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 16:44:29 +0100 Subject: [PATCH 22/28] Update message Co-authored-by: Martin Dobias --- dbsync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbsync.py b/dbsync.py index 635987c..2b3c266 100644 --- a/dbsync.py +++ b/dbsync.py @@ -602,8 +602,7 @@ def init(conn_cfg, mc, from_gpkg=True): db_proj_info = _get_db_project_comment(conn, conn_cfg.base) if not db_proj_info: raise DbSyncError("Base schema exists but missing which project it belongs to. " - f"Both schemata `{conn_cfg.base}` and `{conn_cfg.modified}` should be deleted in the database " - "so that DB Sync can set up sync properly") + f"This may be a result of a previously failed attempt to initialize DB sync. You can delete both schemas `{conn_cfg.base}` and `{conn_cfg.modified}` to fix this error and restart DB sync.") if "error" in db_proj_info: changes_gpkg_base = _compare_datasets("sqlite", "", gpkg_full_path, conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, ignored_tables, From 418e3bf2def0c0c453e2b7bc39ea7ac8d7e9b09c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 16:44:43 +0100 Subject: [PATCH 23/28] Update message Co-authored-by: Martin Dobias --- dbsync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbsync.py b/dbsync.py index 2b3c266..0b8696e 100644 --- a/dbsync.py +++ b/dbsync.py @@ -679,8 +679,7 @@ def init(conn_cfg, mc, from_gpkg=True): return # nothing to do elif modified_schema_exists: raise DbSyncError(f"The 'modified' schema exists but the base schema is missing: {conn_cfg.base}. " - f"Schema `{conn_cfg.modified}` should be removed or renamed in the database " - "so that DB Sync can set up sync properly.") + f"This may be a result of a previously failed attempt to initialize DB sync. You can delete schema `{conn_cfg.modified}` in the database to fix this error and restart DB sync.") elif base_schema_exists: raise DbSyncError(f"The base schema exists but the modified schema is missing: {conn_cfg.modified}. " f"Schema `{conn_cfg.base}` should be removed or renamed in the database " From 584b5ba4c2034dd7405a29ac3807299527dba917 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 16:45:00 +0100 Subject: [PATCH 24/28] Update message Co-authored-by: Martin Dobias --- dbsync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbsync.py b/dbsync.py index 0b8696e..1ca52c5 100644 --- a/dbsync.py +++ b/dbsync.py @@ -682,8 +682,7 @@ def init(conn_cfg, mc, from_gpkg=True): f"This may be a result of a previously failed attempt to initialize DB sync. You can delete schema `{conn_cfg.modified}` in the database to fix this error and restart DB sync.") elif base_schema_exists: raise DbSyncError(f"The base schema exists but the modified schema is missing: {conn_cfg.modified}. " - f"Schema `{conn_cfg.base}` should be removed or renamed in the database " - "so that DB Sync can set up sync properly.") + f"This may be a result of a previously failed attempt to initialize DB sync. You can delete schema `{conn_cfg.base}` in the database to fix this error and restart DB sync.") # initialize: we have an existing GeoPackage in our Mergin Maps project and we want to initialize database print("The base and modified schemas do not exist yet, going to initialize them ...") From 0843f4ea8ddf0c25bbb170c62957ff747c2f670f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 16:49:08 +0100 Subject: [PATCH 25/28] add prints before dropping schemas --- dbsync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbsync.py b/dbsync.py index 1ca52c5..e3be2ee 100644 --- a/dbsync.py +++ b/dbsync.py @@ -708,6 +708,7 @@ def init(conn_cfg, mc, from_gpkg=True): raise DbSyncError('Initialization of db-sync failed due to a bug in geodiff.\n ' 'Please report this problem to mergin-db-sync developers') except DbSyncError: + print(f"Cleaning up after a failed DB sync init - dropping schemas {conn_cfg.base} and {conn_cfg.modified}.") _drop_schema(conn_cfg.base) _drop_schema(conn_cfg.modified) raise @@ -764,6 +765,7 @@ def init(conn_cfg, mc, from_gpkg=True): raise DbSyncError('Initialization of db-sync failed due to a bug in geodiff.\n ' 'Please report this problem to mergin-db-sync developers') except DbSyncError: + print(f"Cleaning up after a failed DB sync init - dropping schema {conn_cfg.base}.") _drop_schema(conn_cfg.base) raise From cf9f5ddcd06a19539ede67874c9013fa7406c133 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 23 Mar 2023 17:00:52 +0100 Subject: [PATCH 26/28] fix test --- test/test_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_basic.py b/test/test_basic.py index 9b5c802..3105c80 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -501,7 +501,7 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient): with pytest.raises(DbSyncError) as err: dbsync_init(mc, from_gpkg=True) assert "The 'modified' schema exists but the base schema is missing" in str(err.value) - assert f"Schema `{db_schema_main}` should be removed or renamed" in str(err.value) + assert "This may be a result of a previously failed attempt to initialize DB sync" in str(err.value) init_sync_from_geopackage(mc, project_name, source_gpkg_path) @@ -516,7 +516,7 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient): with pytest.raises(DbSyncError) as err: dbsync_init(mc, from_gpkg=True) assert "The base schema exists but the modified schema is missing" in str(err.value) - assert f"Schema `{db_schema_base}` should be removed or renamed" in str(err.value) + assert "This may be a result of a previously failed attempt to initialize DB sync" in str(err.value) def test_init_from_gpkg_missing_comment(mc: MerginClient): From c1f23bd1d5d39141fd4f28d1c38265456c363cff Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 15:14:17 +0100 Subject: [PATCH 27/28] remove settings that should not be used there ;-) --- test/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_config.py b/test/test_config.py index 6b9ac24..6c34a69 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -57,10 +57,10 @@ def test_config(): _reset_config() with pytest.raises(ConfigError, match="Config error: Only 'postgres' driver is currently supported."): - config.update({'CONNECTIONS': [{"driver": "oracle", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg", "init_from": "gpkg"}]}) + config.update({'CONNECTIONS': [{"driver": "oracle", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg"}]}) validate_config(config) _reset_config() with pytest.raises(ConfigError, match="Config error: Name of the Mergin Maps project should be provided in the namespace/name format."): - config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg", "init_from": "gpkg"}]}) + config.update({'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "dbsync", "sync_file": "sync.gpkg"}]}) validate_config(config) From b7c13cf8c09e6a39414545a2bb3fbb0151d785a6 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 24 Mar 2023 15:25:15 +0100 Subject: [PATCH 28/28] fix failing tests --- test/test_basic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_basic.py b/test/test_basic.py index a9a0aa5..0cf5555 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -500,7 +500,7 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient): cur.fetchone() is None with pytest.raises(DbSyncError) as err: - dbsync_init(mc, from_gpkg=True) + dbsync_init(mc) assert "The 'modified' schema exists but the base schema is missing" in str(err.value) assert "This may be a result of a previously failed attempt to initialize DB sync" in str(err.value) @@ -515,7 +515,7 @@ def test_init_from_gpkg_missing_schema(mc: MerginClient): cur.fetchone() is None with pytest.raises(DbSyncError) as err: - dbsync_init(mc, from_gpkg=True) + dbsync_init(mc) assert "The base schema exists but the modified schema is missing" in str(err.value) assert "This may be a result of a previously failed attempt to initialize DB sync" in str(err.value) @@ -543,7 +543,7 @@ def test_init_from_gpkg_missing_comment(mc: MerginClient): conn.commit() with pytest.raises(DbSyncError) as err: - dbsync_init(mc, from_gpkg=True) + dbsync_init(mc) assert "Base schema exists but missing which project it belongs to" in str(err.value) # check that schema does not exists anymore