From cf35e9f4ef6b3762f442ff338740b699266f4a48 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 3 May 2023 22:57:45 +0200 Subject: [PATCH 1/6] postgis checking functions --- dbsync.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dbsync.py b/dbsync.py index f5c8c68..e2a37d9 100644 --- a/dbsync.py +++ b/dbsync.py @@ -75,6 +75,28 @@ def _check_schema_exists(conn, schema_name): return cur.fetchone()[0] +def _check_postgis_available(conn: psycopg2.connection) -> bool: + cur = conn.cursor() + cur.execute("SELECT extname FROM pg_extension;") + try: + result = cur.fetchall() + for row in result: + if row[0].lower() == "postgis": + return True + return False + except psycopg2.ProgrammingError: + return False + + +def _try_install_postgis(conn: psycopg2.connection) -> bool: + cur = conn.cursor() + try: + cur.execute("CREATE EXTENSION postgis;") + return True + except psycopg2.ProgrammingError: + return False + + def _check_has_password(): """ Checks whether we have password for Mergin Maps user - if not, we will ask for it """ if config.mergin.password is None: From 91fd9bf578ab0adb71d8d48178a5ca41983f04b4 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 3 May 2023 22:58:42 +0200 Subject: [PATCH 2/6] on init if driver is postgres, check that postgis is available or can be installed --- dbsync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbsync.py b/dbsync.py index e2a37d9..f0d6ce4 100644 --- a/dbsync.py +++ b/dbsync.py @@ -613,6 +613,11 @@ def init(conn_cfg, mc, from_gpkg=True): except psycopg2.Error as e: raise DbSyncError("Unable to connect to the database: " + str(e)) + if conn_cfg.driver.lower() == "postgres": + if not _check_postgis_available(conn): + if not _try_install_postgis(conn): + raise DbSyncError("Cannot find or activate `postgis` extension. You may need to install it.") + base_schema_exists = _check_schema_exists(conn, conn_cfg.base) modified_schema_exists = _check_schema_exists(conn, conn_cfg.modified) From 2b8f9ae0f2eabafe7601d23b51bb6ff5c44e1faf Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 3 May 2023 23:02:55 +0200 Subject: [PATCH 3/6] move test settings, helper functions, fixtures to separate file --- test/conftest.py | 128 ++++++++++++++++++++++++++++++++++++++++++++ test/test_basic.py | 90 ++----------------------------- test/test_config.py | 16 +----- 3 files changed, 133 insertions(+), 101 deletions(-) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..6721f49 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,128 @@ +import pytest +import os +import tempfile +import shutil + +import psycopg2 +from psycopg2 import sql + +from mergin import MerginClient, ClientError + +from dbsync import dbsync_init +from config import config + +GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE') +DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO') +SERVER_URL = os.environ.get('TEST_MERGIN_URL') +API_USER = os.environ.get('TEST_API_USERNAME') +USER_PWD = os.environ.get('TEST_API_PASSWORD') +WORKSPACE = os.environ.get('TEST_API_WORKSPACE') +TMP_DIR = tempfile.gettempdir() +TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') + + +def _reset_config(project_name: str = "mergin"): + """ helper to reset config settings to ensure valid config """ + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' + full_project_name = WORKSPACE + "/" + project_name + + config.update({ + 'MERGIN__USERNAME': API_USER, + 'MERGIN__PASSWORD': USER_PWD, + 'MERGIN__URL': SERVER_URL, + 'init_from': "gpkg", + 'CONNECTIONS': [{"driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg"}] + }) + + +def cleanup(mc, project, dirs): + """ cleanup leftovers from previous test if needed such as remote project and local directories """ + try: + print("Deleting project on Mergin Maps server: " + project) + mc.delete_project(project) + except ClientError as e: + print("Deleting project error: " + str(e)) + pass + for d in dirs: + if os.path.exists(d): + shutil.rmtree(d) + + +def cleanup_db(conn, schema_base, schema_main): + """ Removes test schemas from previous tests """ + cur = conn.cursor() + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base))) + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main))) + cur.execute("COMMIT") + + +def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files): + """ + Initialize sync from given GeoPackage file: + - (re)create Mergin Maps project with the file + - (re)create local project working directory and sync directory + - configure DB sync and let it do the init (make copies to the database) + """ + full_project_name = WORKSPACE + "/" + project_name + project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory + sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync') # used by dbsync + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' + + conn = psycopg2.connect(DB_CONNINFO) + + cleanup(mc, full_project_name, [project_dir, sync_project_dir]) + cleanup_db(conn, db_schema_base, db_schema_main) + + # prepare a new Mergin Maps project + mc.create_project(project_name, namespace=WORKSPACE) + mc.download_project(full_project_name, project_dir) + shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg')) + for extra_filepath in extra_init_files: + extra_filename = os.path.basename(extra_filepath) + target_extra_filepath = os.path.join(project_dir, extra_filename) + shutil.copy(extra_filepath, target_extra_filepath) + mc.push_project(project_dir) + + # prepare dbsync config + # patch config to fit testing purposes + if ignored_tables: + connection = {"driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg", + "skip_tables": ignored_tables} + else: + connection = {"driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg"} + + config.update({ + 'GEODIFF_EXE': GEODIFF_EXE, + 'WORKING_DIR': sync_project_dir, + 'MERGIN__USERNAME': API_USER, + 'MERGIN__PASSWORD': USER_PWD, + 'MERGIN__URL': SERVER_URL, + 'CONNECTIONS': [connection], + 'init_from': "gpkg" + }) + + dbsync_init(mc) + + +@pytest.fixture(scope='function') +def mc(): + assert SERVER_URL and API_USER and USER_PWD + #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD + return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD) diff --git a/test/test_basic.py b/test/test_basic.py index 0cf5555..5d9ece9 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -4,99 +4,17 @@ import os import shutil import sqlite3 -import tempfile import psycopg2 from psycopg2 import sql -from mergin import MerginClient, ClientError +from mergin import MerginClient + from dbsync import dbsync_init, dbsync_pull, dbsync_push, dbsync_status, config, DbSyncError, _geodiff_make_copy, \ _get_db_project_comment, _get_mergin_project, _get_project_id, _validate_local_project_id, config, _add_quotes_to_schema_name -GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE') -DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO') -SERVER_URL = os.environ.get('TEST_MERGIN_URL') -API_USER = os.environ.get('TEST_API_USERNAME') -USER_PWD = os.environ.get('TEST_API_PASSWORD') -WORKSPACE = os.environ.get('TEST_API_WORKSPACE') -TMP_DIR = tempfile.gettempdir() -TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') - - -@pytest.fixture(scope='function') -def mc(): - assert SERVER_URL and API_USER and USER_PWD - #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD - return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD) - - -def cleanup(mc, project, dirs): - """ cleanup leftovers from previous test if needed such as remote project and local directories """ - try: - print("Deleting project on Mergin Maps server: " + project) - mc.delete_project(project) - except ClientError as e: - print("Deleting project error: " + str(e)) - pass - for d in dirs: - if os.path.exists(d): - shutil.rmtree(d) - - -def cleanup_db(conn, schema_base, schema_main): - """ Removes test schemas from previous tests """ - cur = conn.cursor() - cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base))) - cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main))) - cur.execute("COMMIT") - - -def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files): - """ - Initialize sync from given GeoPackage file: - - (re)create Mergin Maps project with the file - - (re)create local project working directory and sync directory - - configure DB sync and let it do the init (make copies to the database) - """ - full_project_name = WORKSPACE + "/" + project_name - project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory - sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync') # used by dbsync - db_schema_main = project_name + '_main' - db_schema_base = project_name + '_base' - - conn = psycopg2.connect(DB_CONNINFO) - - cleanup(mc, full_project_name, [project_dir, sync_project_dir]) - cleanup_db(conn, db_schema_base, db_schema_main) - - # prepare a new Mergin Maps project - mc.create_project(project_name, namespace=WORKSPACE) - mc.download_project(full_project_name, project_dir) - shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg')) - for extra_filepath in extra_init_files: - extra_filename = os.path.basename(extra_filepath) - target_extra_filepath = os.path.join(project_dir, extra_filename) - shutil.copy(extra_filepath, target_extra_filepath) - mc.push_project(project_dir) - - # prepare dbsync config - # patch config to fit testing purposes - if ignored_tables: - connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg", "skip_tables":ignored_tables} - else: - connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"} - - config.update({ - 'GEODIFF_EXE': GEODIFF_EXE, - 'WORKING_DIR': sync_project_dir, - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'CONNECTIONS': [connection], - 'init_from': "gpkg" - }) - - dbsync_init(mc) +from .conftest import (WORKSPACE, TMP_DIR, DB_CONNINFO, + TEST_DATA_DIR, init_sync_from_geopackage) def test_init_from_gpkg(mc: MerginClient): diff --git a/test/test_config.py b/test/test_config.py index 71ea299..732326a 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -5,25 +5,11 @@ License: MIT """ -import os import pytest from config import config, ConfigError, validate_config, get_ignored_tables -SERVER_URL = os.environ.get('TEST_MERGIN_URL') -API_USER = os.environ.get('TEST_API_USERNAME') -USER_PWD = os.environ.get('TEST_API_PASSWORD') - - -def _reset_config(): - """ helper to reset config settings to ensure valid config """ - config.update({ - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'init_from': "gpkg", - 'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg"}] - }) +from .conftest import _reset_config def test_config(): From 680ae8e05ab25a2a2357cc8bb823e12370adbd96 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 3 May 2023 23:09:59 +0200 Subject: [PATCH 4/6] add fixture --- test/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 6721f49..f3ba4f9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -126,3 +126,8 @@ def mc(): assert SERVER_URL and API_USER and USER_PWD #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD) + + +@pytest.fixture(scope='function') +def db_connection() -> psycopg2.connection: + return psycopg2.connect(DB_CONNINFO) From 45012ae046e6dcf8e652de770c15fe7d44d9e842 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 3 May 2023 23:10:33 +0200 Subject: [PATCH 5/6] test for postgis functions --- test/test_db_functions.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/test_db_functions.py diff --git a/test/test_db_functions.py b/test/test_db_functions.py new file mode 100644 index 0000000..e50322d --- /dev/null +++ b/test/test_db_functions.py @@ -0,0 +1,25 @@ +import psycopg2 + +from dbsync import _check_postgis_available, _try_install_postgis + + +def test_check_postgis_available(db_connection: psycopg2.connection): + cur = db_connection.cursor() + + assert _check_postgis_available(db_connection) + + cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;") + + assert _check_postgis_available(db_connection) is False + + +def test_try_install_postgis(db_connection: psycopg2.connection): + cur = db_connection.cursor() + + cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;") + + assert _check_postgis_available(db_connection) is False + + _try_install_postgis(db_connection) + + assert _check_postgis_available(db_connection) From 39ecf515fae67a5e0a689d8e69cb0637065a4031 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 3 May 2023 23:30:58 +0200 Subject: [PATCH 6/6] fix typing --- dbsync.py | 7 ++++--- test/conftest.py | 3 ++- test/test_db_functions.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dbsync.py b/dbsync.py index f0d6ce4..ad1250e 100644 --- a/dbsync.py +++ b/dbsync.py @@ -19,8 +19,9 @@ import re import psycopg2 -from itertools import chain +import psycopg2.extensions from psycopg2 import sql +from itertools import chain from mergin import MerginClient, MerginProject, LoginError, ClientError, InvalidProject from version import __version__ @@ -75,7 +76,7 @@ def _check_schema_exists(conn, schema_name): return cur.fetchone()[0] -def _check_postgis_available(conn: psycopg2.connection) -> bool: +def _check_postgis_available(conn: psycopg2.extensions.connection) -> bool: cur = conn.cursor() cur.execute("SELECT extname FROM pg_extension;") try: @@ -88,7 +89,7 @@ def _check_postgis_available(conn: psycopg2.connection) -> bool: return False -def _try_install_postgis(conn: psycopg2.connection) -> bool: +def _try_install_postgis(conn: psycopg2.extensions.connection) -> bool: cur = conn.cursor() try: cur.execute("CREATE EXTENSION postgis;") diff --git a/test/conftest.py b/test/conftest.py index f3ba4f9..d531e16 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,6 +4,7 @@ import shutil import psycopg2 +import psycopg2.extensions from psycopg2 import sql from mergin import MerginClient, ClientError @@ -129,5 +130,5 @@ def mc(): @pytest.fixture(scope='function') -def db_connection() -> psycopg2.connection: +def db_connection() -> psycopg2.extensions.connection: return psycopg2.connect(DB_CONNINFO) diff --git a/test/test_db_functions.py b/test/test_db_functions.py index e50322d..1c15659 100644 --- a/test/test_db_functions.py +++ b/test/test_db_functions.py @@ -1,9 +1,10 @@ import psycopg2 +import psycopg2.extensions from dbsync import _check_postgis_available, _try_install_postgis -def test_check_postgis_available(db_connection: psycopg2.connection): +def test_check_postgis_available(db_connection: psycopg2.extensions.connection): cur = db_connection.cursor() assert _check_postgis_available(db_connection) @@ -13,7 +14,7 @@ def test_check_postgis_available(db_connection: psycopg2.connection): assert _check_postgis_available(db_connection) is False -def test_try_install_postgis(db_connection: psycopg2.connection): +def test_try_install_postgis(db_connection: psycopg2.extensions.connection): cur = db_connection.cursor() cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;")