Postgis checks

dbsync.py

@@ -20,8 +20,9 @@
 import pathlib
 
 import psycopg2
-from itertools import chain
+import psycopg2.extensions
 from psycopg2 import sql
+from itertools import chain
 
 from mergin import MerginClient, MerginProject, LoginError, ClientError, InvalidProject
 from version import __version__
@@ -85,6 +86,28 @@ def _check_schema_exists(conn, schema_name):
     return cur.fetchone()[0]
 
 
+def _check_postgis_available(conn: psycopg2.extensions.connection) -> bool:
+    cur = conn.cursor()
+    cur.execute("SELECT extname FROM pg_extension;")
+    try:
+        result = cur.fetchall()
+        for row in result:
+            if row[0].lower() == "postgis":
+                return True
+        return False
+    except psycopg2.ProgrammingError:
+        return False
+
+
+def _try_install_postgis(conn: psycopg2.extensions.connection) -> bool:
+    cur = conn.cursor()
+    try:
+        cur.execute("CREATE EXTENSION postgis;")
+        return True
+    except psycopg2.ProgrammingError:
+        return False
+
+
 def _check_has_password():
     """ Checks whether we have password for Mergin Maps user - if not, we will ask for it """
     if config.mergin.password is None:
@@ -601,6 +624,11 @@ def init(conn_cfg, mc, from_gpkg=True):
     except psycopg2.Error as e:
         raise DbSyncError("Unable to connect to the database: " + str(e))
 
+    if conn_cfg.driver.lower() == "postgres":
+        if not _check_postgis_available(conn):
+            if not _try_install_postgis(conn):
+                raise DbSyncError("Cannot find or activate `postgis` extension. You may need to install it.")
+
     base_schema_exists = _check_schema_exists(conn, conn_cfg.base)
     modified_schema_exists = _check_schema_exists(conn, conn_cfg.modified)
 

test/conftest.py

@@ -0,0 +1,134 @@
+import pytest
+import os
+import tempfile
+import shutil
+
+import psycopg2
+import psycopg2.extensions
+from psycopg2 import sql
+
+from mergin import MerginClient, ClientError
+
+from dbsync import dbsync_init
+from config import config
+
+GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE')
+DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO')
+SERVER_URL = os.environ.get('TEST_MERGIN_URL')
+API_USER = os.environ.get('TEST_API_USERNAME')
+USER_PWD = os.environ.get('TEST_API_PASSWORD')
+WORKSPACE = os.environ.get('TEST_API_WORKSPACE')
+TMP_DIR = tempfile.gettempdir()
+TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data')
+
+
+def _reset_config(project_name: str = "mergin"):
+    """ helper to reset config settings to ensure valid config """
+    db_schema_main = project_name + '_main'
+    db_schema_base = project_name + '_base'
+    full_project_name = WORKSPACE + "/" + project_name
+
+    config.update({
+        'MERGIN__USERNAME': API_USER,
+        'MERGIN__PASSWORD': USER_PWD,
+        'MERGIN__URL': SERVER_URL,
+        'init_from': "gpkg",
+        'CONNECTIONS': [{"driver": "postgres",
+                         "conn_info": DB_CONNINFO,
+                         "modified": db_schema_main,
+                         "base": db_schema_base,
+                         "mergin_project": full_project_name,
+                         "sync_file": "test_sync.gpkg"}]
+    })
+
+
+def cleanup(mc, project, dirs):
+    """ cleanup leftovers from previous test if needed such as remote project and local directories """
+    try:
+        print("Deleting project on Mergin Maps server: " + project)
+        mc.delete_project(project)
+    except ClientError as e:
+        print("Deleting project error: " + str(e))
+        pass
+    for d in dirs:
+        if os.path.exists(d):
+            shutil.rmtree(d)
+
+
+def cleanup_db(conn, schema_base, schema_main):
+    """ Removes test schemas from previous tests """
+    cur = conn.cursor()
+    cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base)))
+    cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main)))
+    cur.execute("COMMIT")
+
+
+def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files):
+    """
+    Initialize sync from given GeoPackage file:
+    - (re)create Mergin Maps project with the file
+    - (re)create local project working directory and sync directory
+    - configure DB sync and let it do the init (make copies to the database)
+    """
+    full_project_name = WORKSPACE + "/" + project_name
+    project_dir = os.path.join(TMP_DIR, project_name + '_work')  # working directory
+    sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync')  # used by dbsync
+    db_schema_main = project_name + '_main'
+    db_schema_base = project_name + '_base'
+
+    conn = psycopg2.connect(DB_CONNINFO)
+
+    cleanup(mc, full_project_name, [project_dir, sync_project_dir])
+    cleanup_db(conn, db_schema_base, db_schema_main)
+
+    # prepare a new Mergin Maps project
+    mc.create_project(project_name, namespace=WORKSPACE)
+    mc.download_project(full_project_name, project_dir)
+    shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg'))
+    for extra_filepath in extra_init_files:
+        extra_filename = os.path.basename(extra_filepath)
+        target_extra_filepath = os.path.join(project_dir, extra_filename)
+        shutil.copy(extra_filepath, target_extra_filepath)
+    mc.push_project(project_dir)
+
+    # prepare dbsync config
+    # patch config to fit testing purposes
+    if ignored_tables:
+        connection = {"driver": "postgres",
+                      "conn_info": DB_CONNINFO,
+                      "modified": db_schema_main,
+                      "base": db_schema_base,
+                      "mergin_project": full_project_name,
+                      "sync_file": "test_sync.gpkg",
+                      "skip_tables": ignored_tables}
+    else:
+        connection = {"driver": "postgres",
+                      "conn_info": DB_CONNINFO,
+                      "modified": db_schema_main,
+                      "base": db_schema_base,
+                      "mergin_project": full_project_name,
+                      "sync_file": "test_sync.gpkg"}
+
+    config.update({
+        'GEODIFF_EXE': GEODIFF_EXE,
+        'WORKING_DIR': sync_project_dir,
+        'MERGIN__USERNAME': API_USER,
+        'MERGIN__PASSWORD': USER_PWD,
+        'MERGIN__URL': SERVER_URL,
+        'CONNECTIONS': [connection],
+        'init_from': "gpkg"
+    })
+
+    dbsync_init(mc)
+
+
+@pytest.fixture(scope='function')
+def mc():
+    assert SERVER_URL and API_USER and USER_PWD
+    #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD
+    return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD)
+
+
+@pytest.fixture(scope='function')
+def db_connection() -> psycopg2.extensions.connection:
+    return psycopg2.connect(DB_CONNINFO)

test/test_basic.py

@@ -6,100 +6,18 @@
 import sqlite3
 import tempfile
 import pathlib
-import sqlite3
 
 import psycopg2
 from psycopg2 import sql
 
-from mergin import MerginClient, ClientError
+from mergin import MerginClient
+
 from dbsync import dbsync_init, dbsync_pull, dbsync_push, dbsync_status, config, DbSyncError, _geodiff_make_copy, \
     _get_db_project_comment, _get_mergin_project, _get_project_id, _validate_local_project_id, config, _add_quotes_to_schema_name, \
     dbsync_clean, _check_schema_exists
 
-GEODIFF_EXE = os.environ.get('TEST_GEODIFF_EXE')
-DB_CONNINFO = os.environ.get('TEST_DB_CONNINFO')
-SERVER_URL = os.environ.get('TEST_MERGIN_URL')
-API_USER = os.environ.get('TEST_API_USERNAME')
-USER_PWD = os.environ.get('TEST_API_PASSWORD')
-WORKSPACE = os.environ.get('TEST_API_WORKSPACE')
-TMP_DIR = tempfile.gettempdir()
-TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data')
-
-
-@pytest.fixture(scope='function')
-def mc():
-    assert SERVER_URL and API_USER and USER_PWD
-    #assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD
-    return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD)
-
-
-def cleanup(mc, project, dirs):
-    """ cleanup leftovers from previous test if needed such as remote project and local directories """
-    try:
-        print("Deleting project on Mergin Maps server: " + project)
-        mc.delete_project(project)
-    except ClientError as e:
-        print("Deleting project error: " + str(e))
-        pass
-    for d in dirs:
-        if os.path.exists(d):
-            shutil.rmtree(d)
-
-
-def cleanup_db(conn, schema_base, schema_main):
-    """ Removes test schemas from previous tests """
-    cur = conn.cursor()
-    cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_base)))
-    cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema_main)))
-    cur.execute("COMMIT")
-
-
-def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files):
-    """
-    Initialize sync from given GeoPackage file:
-    - (re)create Mergin Maps project with the file
-    - (re)create local project working directory and sync directory
-    - configure DB sync and let it do the init (make copies to the database)
-    """
-    full_project_name = WORKSPACE + "/" + project_name
-    project_dir = os.path.join(TMP_DIR, project_name + '_work')  # working directory
-    sync_project_dir = os.path.join(TMP_DIR, project_name + '_dbsync')  # used by dbsync
-    db_schema_main = project_name + '_main'
-    db_schema_base = project_name + '_base'
-
-    conn = psycopg2.connect(DB_CONNINFO)
-
-    cleanup(mc, full_project_name, [project_dir, sync_project_dir])
-    cleanup_db(conn, db_schema_base, db_schema_main)
-
-    # prepare a new Mergin Maps project
-    mc.create_project(project_name, namespace=WORKSPACE)
-    mc.download_project(full_project_name, project_dir)
-    shutil.copy(source_gpkg_path, os.path.join(project_dir, 'test_sync.gpkg'))
-    for extra_filepath in extra_init_files:
-        extra_filename = os.path.basename(extra_filepath)
-        target_extra_filepath = os.path.join(project_dir, extra_filename)
-        shutil.copy(extra_filepath, target_extra_filepath)
-    mc.push_project(project_dir)
-
-    # prepare dbsync config
-    # patch config to fit testing purposes
-    if ignored_tables:
-        connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg", "skip_tables":ignored_tables}
-    else:
-        connection = {"driver": "postgres", "conn_info": DB_CONNINFO, "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, "sync_file": "test_sync.gpkg"}
-
-    config.update({
-        'GEODIFF_EXE': GEODIFF_EXE,
-        'WORKING_DIR': sync_project_dir,
-        'MERGIN__USERNAME': API_USER,
-        'MERGIN__PASSWORD': USER_PWD,
-        'MERGIN__URL': SERVER_URL,
-        'CONNECTIONS': [connection],
-        'init_from': "gpkg"
-    })
-
-    dbsync_init(mc)
+from .conftest import (WORKSPACE, TMP_DIR, DB_CONNINFO, 
+                       TEST_DATA_DIR, init_sync_from_geopackage)
 
 
 def test_init_from_gpkg(mc: MerginClient):

test/test_config.py

@@ -5,25 +5,11 @@
 
 License: MIT
 """
-import os
 import pytest
 
 from config import config, ConfigError, validate_config, get_ignored_tables
 
-SERVER_URL = os.environ.get('TEST_MERGIN_URL')
-API_USER = os.environ.get('TEST_API_USERNAME')
-USER_PWD = os.environ.get('TEST_API_PASSWORD')
-
-
-def _reset_config():
-    """ helper to reset config settings to ensure valid config """
-    config.update({
-        'MERGIN__USERNAME': API_USER,
-        'MERGIN__PASSWORD': USER_PWD,
-        'MERGIN__URL': SERVER_URL,
-        'init_from': "gpkg",
-        'CONNECTIONS': [{"driver": "postgres", "conn_info": "", "modified": "mergin_main", "base": "mergin_base", "mergin_project": "john/dbsync", "sync_file": "sync.gpkg"}]
-    })
+from .conftest import _reset_config
 
 
 def test_config():

test/test_db_functions.py

@@ -0,0 +1,26 @@
+import psycopg2
+import psycopg2.extensions
+
+from dbsync import _check_postgis_available, _try_install_postgis
+
+
+def test_check_postgis_available(db_connection: psycopg2.extensions.connection):
+    cur = db_connection.cursor()
+
+    assert _check_postgis_available(db_connection)
+
+    cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;")
+
+    assert _check_postgis_available(db_connection) is False
+
+
+def test_try_install_postgis(db_connection: psycopg2.extensions.connection):
+    cur = db_connection.cursor()
+
+    cur.execute("DROP EXTENSION IF EXISTS postgis CASCADE;")
+
+    assert _check_postgis_available(db_connection) is False
+
+    _try_install_postgis(db_connection)
+
+    assert _check_postgis_available(db_connection)