diff --git a/dbsync.py b/dbsync.py index 1e364c8..45597fa 100644 --- a/dbsync.py +++ b/dbsync.py @@ -20,8 +20,9 @@ import pathlib import psycopg2 -from itertools import chain +import psycopg2.extensions from psycopg2 import sql +from itertools import chain from mergin import MerginClient, MerginProject, LoginError, ClientError, InvalidProject from version import __version__ @@ -85,6 +86,28 @@ def _check_schema_exists(conn, schema_name): return cur.fetchone()[0] +def _check_postgis_available(conn: psycopg2.extensions.connection) -> bool: + cur = conn.cursor() + cur.execute("SELECT extname FROM pg_extension;") + try: + result = cur.fetchall() + for row in result: + if row[0].lower() == "postgis": + return True + return False + except psycopg2.ProgrammingError: + return False + + +def _try_install_postgis(conn: psycopg2.extensions.connection) -> bool: + cur = conn.cursor() + try: + cur.execute("CREATE EXTENSION postgis;") + return True + except psycopg2.ProgrammingError: + return False + + def _check_has_password(): """ Checks whether we have password for Mergin Maps user - if not, we will ask for it """ if config.mergin.password is None: @@ -601,6 +624,11 @@ def init(conn_cfg, mc, from_gpkg=True): except psycopg2.Error as e: raise DbSyncError("Unable to connect to the database: " + str(e)) + if conn_cfg.driver.lower() == "postgres": + if not _check_postgis_available(conn): + if not _try_install_postgis(conn): + raise DbSyncError("Cannot find or activate `postgis` extension. You may need to install it.") + base_schema_exists = _check_schema_exists(conn, conn_cfg.base) modified_schema_exists = _check_schema_exists(conn, conn_cfg.modified) diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..d531e16 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,134 @@ +import pytest +import os +import tempfile +import shutil + +import psycopg2 +import psycopg2.extensions +from psycopg2 import sql + +from mergin import MerginClient, ClientError + +from dbsync import dbsync_init +from config import config + +GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE') +DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO') +SERVER_URL = os.environ.get('TEST_MERGIN_URL') +API_USER = os.environ.get('TEST_API_USERNAME') +USER_PWD = os.environ.get('TEST_API_PASSWORD') +WORKSPACE = os.environ.get('TEST_API_WORKSPACE') +TMP_DIR = tempfile.gettempdir() +TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') + + +def _reset_config(project_name: str = "mergin"): + """ helper to reset config settings to ensure valid config """ + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' + full_project_name = WORKSPACE + "/" + project_name + + config.update({ + 'MERGIN__USERNAME': API_USER, + 'MERGIN__PASSWORD': USER_PWD, + 'MERGIN__URL': SERVER_URL, + 'init_from': "gpkg", + 'CONNECTIONS': [{"driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg"}] + }) + + +def cleanup(mc, project, dirs): + """ cleanup leftovers from previous test if needed such as remote project and local directories """ + try: + print("Deleting project on Mergin Maps server: " + project) + mc.delete_project(project) + except ClientError as e: + print("Deleting project error: " + str(e)) + pass + for d in dirs: + if os.path.exists(d): + shutil.rmtree(d) + + +def cleanup_db(conn, schema_base, schema_main): + """ Removes test schemas from previous tests """ + cur = conn.cursor() + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base))) + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main))) + cur.execute("COMMIT") + + +def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files): + """ + Initialize sync from given GeoPackage file: + - (re)create Mergin Maps project with the file + - (re)create local project working directory and sync directory + - configure DB sync and let it do the init (make copies to the database) + """ + full_project_name = WORKSPACE + "/" + project_name + project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory + sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync') # used by dbsync + db_schema_main = project_name + '_main' + db_schema_base = project_name + '_base' + + conn = psycopg2.connect(DB_CONNINFO) + + cleanup(mc, full_project_name, [project_dir, sync_project_dir]) + cleanup_db(conn, db_schema_base, db_schema_main) + + # prepare a new Mergin Maps project + mc.create_project(project_name, namespace=WORKSPACE) + mc.download_project(full_project_name, project_dir) + shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg')) + for extra_filepath in extra_init_files: + extra_filename = os.path.basename(extra_filepath) + target_extra_filepath = os.path.join(project_dir, extra_filename) + shutil.copy(extra_filepath, target_extra_filepath) + mc.push_project(project_dir) + + # prepare dbsync config + # patch config to fit testing purposes + if ignored_tables: + connection = {"driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg", + "skip_tables": ignored_tables} + else: + connection = {"driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg"} + + config.update({ + 'GEODIFF_EXE': GEODIFF_EXE, + 'WORKING_DIR': sync_project_dir, + 'MERGIN__USERNAME': API_USER, + 'MERGIN__PASSWORD': USER_PWD, + 'MERGIN__URL': SERVER_URL, + 'CONNECTIONS': [connection], + 'init_from': "gpkg" + }) + + dbsync_init(mc) + + +@pytest.fixture(scope='function') +def mc(): + assert SERVER_URL and API_USER and USER_PWD + #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD + return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD) + + +@pytest.fixture(scope='function') +def db_connection() -> psycopg2.extensions.connection: + return psycopg2.connect(DB_CONNINFO) diff --git a/test/test_basic.py b/test/test_basic.py index a07678d..2766d19 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -6,100 +6,18 @@ import sqlite3 import tempfile import pathlib -import sqlite3 import psycopg2 from psycopg2 import sql -from mergin import MerginClient, ClientError +from mergin import MerginClient + from dbsync import dbsync_init, dbsync_pull, dbsync_push, dbsync_status, config, DbSyncError, _geodiff_make_copy, \ _get_db_project_comment, _get_mergin_project, _get_project_id, _validate_local_project_id, config, _add_quotes_to_schema_name, \ dbsync_clean, _check_schema_exists -GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE') -DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO') -SERVER_URL = os.environ.get('TEST_MERGIN_URL') -API_USER = os.environ.get('TEST_API_USERNAME') -USER_PWD = os.environ.get('TEST_API_PASSWORD') -WORKSPACE = os.environ.get('TEST_API_WORKSPACE') -TMP_DIR = tempfile.gettempdir() -TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') - - -@pytest.fixture(scope='function') -def mc(): - assert SERVER_URL and API_USER and USER_PWD - #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD - return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD) - - -def cleanup(mc, project, dirs): - """ cleanup leftovers from previous test if needed such as remote project and local directories """ - try: - print("Deleting project on Mergin Maps server: " + project) - mc.delete_project(project) - except ClientError as e: - print("Deleting project error: " + str(e)) - pass - for d in dirs: - if os.path.exists(d): - shutil.rmtree(d) - - -def cleanup_db(conn, schema_base, schema_main): - """ Removes test schemas from previous tests """ - cur = conn.cursor() - cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base))) - cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main))) - cur.execute("COMMIT") - - -def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files): - """ - Initialize sync from given GeoPackage file: - - (re)create Mergin Maps project with the file - - (re)create local project working directory and sync directory - - configure DB sync and let it do the init (make copies to the database) - """ - full_project_name = WORKSPACE + "/" + project_name - project_dir = os.path.join(TMP_DIR, project_name + '_work') # working directory - sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync') # used by dbsync - db_schema_main = project_name + '_main' - db_schema_base = project_name + '_base' - - conn = psycopg2.connect(DB_CONNINFO) - - cleanup(mc, full_project_name, [project_dir, sync_project_dir]) - cleanup_db(conn, db_schema_base, db_schema_main) - - # prepare a new Mergin Maps project - mc.create_project(project_name, namespace=WORKSPACE) - mc.download_project(full_project_name, project_dir) - shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg')) - for extra_filepath in extra_init_files: - extra_filename = os.path.basename(extra_filepath) - target_extra_filepath = os.path.join(project_dir, extra_filename) - shutil.copy(extra_filepath, target_extra_filepath) - mc.push_project(project_dir) - - # prepare dbsync config - # patch config to fit testing purposes - if ignored_tables: - connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg", "skip_tables":ignored_tables} - else: - connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"} - - config.update({ - 'GEODIFF_EXE': GEODIFF_EXE, - 'WORKING_DIR': sync_project_dir, - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'CONNECTIONS': [connection], - 'init_from': "gpkg" - }) - - dbsync_init(mc) +from .conftest import (WORKSPACE, TMP_DIR, DB_CONNINFO, + TEST_DATA_DIR, init_sync_from_geopackage) def test_init_from_gpkg(mc: MerginClient): diff --git a/test/test_config.py b/test/test_config.py index 71ea299..732326a 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -5,25 +5,11 @@ License: MIT """ -import os import pytest from config import config, ConfigError, validate_config, get_ignored_tables -SERVER_URL = os.environ.get('TEST_MERGIN_URL') -API_USER = os.environ.get('TEST_API_USERNAME') -USER_PWD = os.environ.get('TEST_API_PASSWORD') - - -def _reset_config(): - """ helper to reset config settings to ensure valid config """ - config.update({ - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'init_from': "gpkg", - 'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg"}] - }) +from .conftest import _reset_config def test_config(): diff --git a/test/test_db_functions.py b/test/test_db_functions.py new file mode 100644 index 0000000..1c15659 --- /dev/null +++ b/test/test_db_functions.py @@ -0,0 +1,26 @@ +import psycopg2 +import psycopg2.extensions + +from dbsync import _check_postgis_available, _try_install_postgis + + +def test_check_postgis_available(db_connection: psycopg2.extensions.connection): + cur = db_connection.cursor() + + assert _check_postgis_available(db_connection) + + cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;") + + assert _check_postgis_available(db_connection) is False + + +def test_try_install_postgis(db_connection: psycopg2.extensions.connection): + cur = db_connection.cursor() + + cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;") + + assert _check_postgis_available(db_connection) is False + + _try_install_postgis(db_connection) + + assert _check_postgis_available(db_connection)