task/WG-583: add check for data being public

Makefile

@@ -25,7 +25,7 @@ stop:
 
 .PHONY: restart-workers
 restart-workers:  ## Restart workers
-	docker compose -f devops/docker-compose.local.yml --env-file .env restart workers
+	docker compose -f devops/docker-compose.local.yml --env-file .env restart workers workers-heavy
 
 .PHONY: restart-nginx
 restart-nginx:  ## Restart nginx

geoapi/celery_app.py

@@ -21,10 +21,11 @@
 # Import task modules
 app.conf.imports = (
     "geoapi.tasks.raster",
-    "geoapi.tasks.lidar",
+    "geoapi.tasks.point_cloud",
     "geoapi.tasks.streetview",
     "geoapi.tasks.projects",
     "geoapi.tasks.external_data",
+    "geoapi.tasks.file_location_check",
 )
 
 # Define the queues

geoapi/custom/designsafe/project.py

@@ -3,6 +3,11 @@
 from geoapi.settings import settings
 from geoapi.custom.designsafe.default_basemap_layers import default_layers
 from geoapi.models import User, Project
+from geoapi.custom.designsafe.utils import (
+    get_designsafe_project_data,
+    is_designsafe_project,
+    extract_project_uuid,
+)
 
 
 def on_project_creation(database_session, user: User, project: Project):
@@ -47,21 +52,43 @@ def on_project_creation(database_session, user: User, project: Project):
             f" project:{project.id} project_uuid:{project.uuid} "
         )
 
-    try:
-        # add metadata to DS projects (i.e. only when system_id starts with "project-"
-        if project.system_id.startswith("project-"):
+    if is_designsafe_project(project.system_id):
+        try:
+            logger.debug(
+                f"Determining related DS project id for user:{user.username}"
+                f" project:{project.id} project_uuid:{project.uuid} "
+            )
+
+            designsafe_project_data = get_designsafe_project_data(
+                database_session=database_session,
+                user=user,
+                system_id=project.system_id,
+            )
+            project.designsafe_project_id = designsafe_project_data["projectId"]
+
+            database_session.add(project)
+            database_session.commit()
+        except Exception:
+            logger.exception(
+                f"Problem determining related DS project id for user:{user.username}"
+                f" project:{project.id} project_uuid:{project.uuid} "
+            )
+
+        try:
             logger.debug(
                 f"Adding metadata for user:{user.username}"
                 f" project:{project.id} project_uuid:{project.uuid} "
             )
+
+            # add metadata to DS projects (i.e. only when system_id starts with "project-"
             update_designsafe_project_hazmapper_metadata(
                 user, project, add_project=True
             )
-    except Exception:
-        logger.exception(
-            f"Problem adding metadata for user:{user.username}"
-            f" project:{project.id} project_uuid:{project.uuid} "
-        )
+        except Exception:
+            logger.exception(
+                f"Problem adding metadata for user:{user.username}"
+                f" project:{project.id} project_uuid:{project.uuid} "
+            )
 
 
 def on_project_deletion(database_session, user: User, project: Project):
@@ -86,7 +113,7 @@ def on_project_deletion(database_session, user: User, project: Project):
 
     try:
         # remove metadata for DS projects (i.e. only when system_id starts with "project-"
-        if project.system_id.startswith("project-"):
+        if is_designsafe_project(project.system_id):
             logger.debug(
                 f"Removing metadata for user:{user.username}"
                 f" during deletion of project:{project.id} project_uuid:{project.uuid} "
@@ -104,7 +131,7 @@ def on_project_deletion(database_session, user: User, project: Project):
 def update_designsafe_project_hazmapper_metadata(
     user: User, project: Project, add_project: bool
 ):
-    designsafe_uuid = project.system_id[len("project-") :]
+    designsafe_uuid = extract_project_uuid(project.system_id)
 
     from geoapi.utils.external_apis import get_session
 

geoapi/custom/designsafe/project_users.py

@@ -1,30 +1,5 @@
 from geoapi.exceptions import GetUsersForProjectNotSupported
-from geoapi.log import logger
-from geoapi.settings import settings
-from geoapi.models import User
-
-
-def get_project_data(database_session, user: User, system_id: str) -> dict:
-    """
-    Get project data for a certain system
-
-    :param database_session: db session
-    :param user: user to use when querying system from DesignSafe
-    :param system_id: str
-    :return: project data
-
-    """
-    from geoapi.utils.external_apis import ApiUtils
-
-    logger.debug(f"Getting project metadata for system:{system_id}")
-
-    uuid = system_id[len("project-") :]
-    client = ApiUtils(database_session, user, settings.DESIGNSAFE_URL)
-    resp = client.get(f"/api/projects/v2/{uuid}/")
-    resp.raise_for_status()
-
-    project = resp.json()["baseProject"]["value"]
-    return project
+from geoapi.custom.designsafe.utils import get_designsafe_project_data
 
 
 def _is_designsafe_project_guest_user(user):
@@ -52,7 +27,9 @@ def get_system_users(database_session, user, system_id: str):
             f"System:{system_id} is not a project so unable to get users"
         )
 
-    project = get_project_data(database_session, user, system_id)
+    project = get_designsafe_project_data(
+        database_session=database_session, user=user, system_id=system_id
+    )
 
     users = {}
     for u in project["users"]:

geoapi/custom/designsafe/utils.py

@@ -0,0 +1,77 @@
+from geoapi.log import logger
+from geoapi.settings import settings
+from geoapi.models import User
+
+
+DESIGNSAFE_PROJECT_ID_CACHE = {}
+
+
+def extract_project_uuid(system_id: str) -> str | None:
+    if system_id.startswith("project-"):
+        return system_id.removeprefix("project-")
+    return None
+
+
+def is_designsafe_project(system_id: str) -> bool:
+    if system_id and system_id.startswith("project-"):
+        return True
+    return False
+
+
+def get_designsafe_project_data(database_session, user: User, system_id: str) -> dict:
+    """
+    Get project data for a certain system
+
+    :param database_session: db session
+    :param user: user to use when querying system from DesignSafe
+    :param system_id: str
+    :return: project data
+
+    """
+    from geoapi.utils.external_apis import ApiUtils
+
+    logger.debug(f"Getting project metadata for system:{system_id}")
+
+    uuid = system_id[len("project-") :]
+    client = ApiUtils(database_session, user, settings.DESIGNSAFE_URL)
+    resp = client.get(f"/api/projects/v2/{uuid}/")
+    resp.raise_for_status()
+
+    project = resp.json()["baseProject"]["value"]
+    return project
+
+
+def get_designsafe_project_id(
+    database_session, user: User, system_id: str
+) -> str | None:
+    """Get designsafe project id (i.e. PRJ-XXXX)
+
+    If not a DS project or error occurs we return None
+    """
+    if not is_designsafe_project(system_id):
+        logger.debug(f"System {system_id} is not a DesignSafe project, skipping")
+        return None
+
+    # Check cache first
+    if system_id in DESIGNSAFE_PROJECT_ID_CACHE:
+        return DESIGNSAFE_PROJECT_ID_CACHE[system_id]
+
+    # Not in cache - fetch from DesignSafe API
+    try:
+        designsafe_project_data = get_designsafe_project_data(
+            database_session=database_session, user=user, system_id=system_id
+        )
+        designsafe_project_id = designsafe_project_data["projectId"]
+
+        # Cache it
+        DESIGNSAFE_PROJECT_ID_CACHE[system_id] = designsafe_project_id
+
+        logger.debug(
+            f"Fetched and cached project_id for system {system_id}: {designsafe_project_id}"
+        )
+        return designsafe_project_id
+    except Exception as e:
+        logger.exception(
+            f"Failed to fetch DesignSafe project_id for system {system_id}: {e}"
+        )
+        return None

geoapi/initdb.py

@@ -3,7 +3,7 @@
 from alembic.config import Config
 from alembic import command
 from geoapi.settings import settings, UnitTestingConfig
-from geoapi.db import Base, engine, get_db_connection_string
+from geoapi.db import Base, get_db_connection_string
 
 
 def setup_local_dev_database():
@@ -48,8 +48,6 @@ def setup_unit_test_database():
     Base.metadata.create_all(test_engine)
     print("All tables created in the test database based on current models.")
 
-    return engine
-
 
 if __name__ == "__main__":
     setup_local_dev_database()

geoapi/migrations/versions/20251001_0210-60f005164287_tileserver__add_kind_internal_uuid.py

@@ -11,6 +11,7 @@
 
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import inspect
 
 
 # revision identifiers, used by Alembic.
@@ -23,9 +24,16 @@
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
 
-    # Rename UC to match naming_convention; behavior unchanged. This was autogenerated
-    op.drop_constraint(op.f("auth_user_id_key"), "auth", type_="unique")
-    op.create_unique_constraint(op.f("uq_auth_user_id"), "auth", ["user_id"])
+    conn = op.get_bind()
+    inspector = inspect(conn)
+
+    unique_constraints = inspector.get_unique_constraints("auth")
+    constraint_names = {uc["name"] for uc in unique_constraints}
+
+    if "auth_user_id_key" in constraint_names:
+        # Old DB that needs to rename to match newer naming_convention
+        op.drop_constraint(op.f("auth_user_id_key"), "auth", type_="unique")
+        op.create_unique_constraint(op.f("uq_auth_user_id"), "auth", ["user_id"])
 
     # Add TileServer fields
     op.add_column(
@@ -68,11 +76,13 @@ def downgrade():
     op.drop_column("tile_servers", "uuid")
     op.drop_column("tile_servers", "internal")
     op.drop_column("tile_servers", "kind")
-    op.drop_constraint(op.f("uq_auth_user_id"), "auth", type_="unique")
-    op.create_unique_constraint(
-        op.f("auth_user_id_key"),
-        "auth",
-        ["user_id"],
-        postgresql_nulls_not_distinct=False,
-    )
+
+    # No need to drop/rename constraints as we aren't "downgrading" to older DB version.
+    # op.drop_constraint(op.f("uq_auth_user_id"), "auth", type_="unique")
+    # op.create_unique_constraint(
+    #    op.f("auth_user_id_key"),
+    #    "auth",
+    #    ["user_id"],
+    #    postgresql_nulls_not_distinct=False,
+    # )
     # ### end Alembic commands ###

geoapi/migrations/versions/20251030_2212-f5f3cfac6089_add_original_system_to_feature_asset.py

@@ -0,0 +1,40 @@
+"""add_original_system_to_feature_asset
+
+Revision ID: f5f3cfac6089
+Revises: ba8b92fc7ead
+Create Date: 2025-10-30 22:12:13.034836
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "f5f3cfac6089"
+down_revision = "ba8b92fc7ead"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "feature_assets", sa.Column("original_system", sa.String(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_feature_assets_original_system"),
+        "feature_assets",
+        ["original_system"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_feature_assets_original_system"), table_name="feature_assets"
+    )
+    op.drop_column("feature_assets", "original_system")
+    # ### end Alembic commands ###