From d090333084f8100603758fe297dc24bac6aa119a Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 11 Nov 2025 16:51:40 +0100 Subject: [PATCH 01/35] feature(street): add image provider in schema --- apps/project/graphql/inputs/project_types/street.py | 4 ++++ apps/project/graphql/types/project_types/street.py | 4 ++++ project_types/street/project.py | 12 ++++++++++++ 3 files changed, 20 insertions(+) diff --git a/apps/project/graphql/inputs/project_types/street.py b/apps/project/graphql/inputs/project_types/street.py index 1090cfe9..94035907 100644 --- a/apps/project/graphql/inputs/project_types/street.py +++ b/apps/project/graphql/inputs/project_types/street.py @@ -7,5 +7,9 @@ class StreetMapillaryImageFiltersInput: ... +@strawberry.experimental.pydantic.input(model=street_project.StreetImageProvider, all_fields=True) +class StreetImageProviderInput: ... + + @strawberry.experimental.pydantic.input(model=street_project.StreetProjectProperty, all_fields=True) class StreetProjectPropertyInput: ... diff --git a/apps/project/graphql/types/project_types/street.py b/apps/project/graphql/types/project_types/street.py index 9b350cf7..b9e32416 100644 --- a/apps/project/graphql/types/project_types/street.py +++ b/apps/project/graphql/types/project_types/street.py @@ -7,5 +7,9 @@ class StreetMapillaryImageFilters: ... +@strawberry.experimental.pydantic.type(model=street_project.StreetImageProvider, all_fields=True) +class StreetImageProvider: ... + + @strawberry.experimental.pydantic.type(model=street_project.StreetProjectProperty, all_fields=True) class StreetProjectPropertyType: ... diff --git a/project_types/street/project.py b/project_types/street/project.py index d03cc827..db4da688 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -2,6 +2,7 @@ import logging import math import typing +from enum import Enum from django.contrib.gis.geos import GEOSGeometry from django.core.files.base import ContentFile @@ -40,10 +41,21 @@ class StreetMapillaryImageFilters(BaseModel): sampling_threshold: custom_fields.PydanticPositiveInt | None = None +class ImageProviderNameEnum(str, Enum): + MAPILLARY = "mapillary" + PANORAMAX = "panoramax" + + +class StreetImageProvider(BaseModel): + name: ImageProviderNameEnum | None = None + url: custom_fields.PydanticUrl | None = None + + class StreetProjectProperty(base_project.BaseProjectProperty): aoi_geometry: custom_fields.PydanticId custom_options: list[CustomOption] | None = None mapillary_image_filters: StreetMapillaryImageFilters + image_provider: StreetImageProvider | None = None class StreetTaskGroupProperty(base_project.BaseProjectTaskGroupProperty): ... From db6bd927c7e8fdae33947dc28caff9c8153e30a8 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 13 Nov 2025 15:40:59 +0100 Subject: [PATCH 02/35] feat(street): add panoramax as image provider Add logic from https://github.com/mapswipe/python-mapswipe-workers/compare/dev...panoramax and refactor StreetImageProvider. This introduces a type change of View Streets project tasks taskId that requires migration and changes in Firebase. --- .../graphql/inputs/project_types/street.py | 3 +- .../graphql/types/project_types/street.py | 3 +- .../0011_alter_projecttask_firebase_id.py | 18 ++++++ apps/project/models.py | 4 +- main/config.py | 1 + project_types/street/api_calls.py | 58 +++++++++++++++---- project_types/street/project.py | 18 ++---- project_types/street/tests/api_calls_test.py | 20 ++++--- schema.graphql | 17 ++++++ utils/geo/street_image_provider/__init__.py | 0 utils/geo/street_image_provider/models.py | 15 +++++ 11 files changed, 122 insertions(+), 35 deletions(-) create mode 100644 apps/project/migrations/0011_alter_projecttask_firebase_id.py create mode 100644 utils/geo/street_image_provider/__init__.py create mode 100644 utils/geo/street_image_provider/models.py diff --git a/apps/project/graphql/inputs/project_types/street.py b/apps/project/graphql/inputs/project_types/street.py index 94035907..4884d05a 100644 --- a/apps/project/graphql/inputs/project_types/street.py +++ b/apps/project/graphql/inputs/project_types/street.py @@ -1,13 +1,14 @@ import strawberry from project_types.street import project as street_project +from utils.geo.street_image_provider.models import StreetImageProvider @strawberry.experimental.pydantic.input(model=street_project.StreetMapillaryImageFilters, all_fields=True) class StreetMapillaryImageFiltersInput: ... -@strawberry.experimental.pydantic.input(model=street_project.StreetImageProvider, all_fields=True) +@strawberry.experimental.pydantic.input(model=StreetImageProvider, all_fields=True) class StreetImageProviderInput: ... diff --git a/apps/project/graphql/types/project_types/street.py b/apps/project/graphql/types/project_types/street.py index b9e32416..8ca5384d 100644 --- a/apps/project/graphql/types/project_types/street.py +++ b/apps/project/graphql/types/project_types/street.py @@ -1,13 +1,14 @@ import strawberry from project_types.street import project as street_project +from utils.geo.street_image_provider.models import StreetImageProvider as StreetImageProviderModel @strawberry.experimental.pydantic.type(model=street_project.StreetMapillaryImageFilters, all_fields=True) class StreetMapillaryImageFilters: ... -@strawberry.experimental.pydantic.type(model=street_project.StreetImageProvider, all_fields=True) +@strawberry.experimental.pydantic.type(model=StreetImageProviderModel, all_fields=True) class StreetImageProvider: ... diff --git a/apps/project/migrations/0011_alter_projecttask_firebase_id.py b/apps/project/migrations/0011_alter_projecttask_firebase_id.py new file mode 100644 index 00000000..3565ac35 --- /dev/null +++ b/apps/project/migrations/0011_alter_projecttask_firebase_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.5 on 2025-11-12 15:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('project', '0010_alter_projecttask_unique_together'), + ] + + operations = [ + migrations.AlterField( + model_name='projecttask', + name='firebase_id', + field=models.CharField(max_length=36), + ), + ] diff --git a/apps/project/models.py b/apps/project/models.py index 44aa399c..ae04ab44 100644 --- a/apps/project/models.py +++ b/apps/project/models.py @@ -671,7 +671,7 @@ class ProjectTask(FirebasePushResource): in the context of a larger mapping or data collection project. """ - firebase_id = models.CharField[str, str](max_length=30) + firebase_id = models.CharField[str, str](max_length=36) task_group: ProjectTaskGroup = models.ForeignKey[ProjectTaskGroup, ProjectTaskGroup]( # type: ignore[reportAssignmentType] ProjectTaskGroup, @@ -689,7 +689,7 @@ class ProjectTask(FirebasePushResource): project_type_specifics = models.JSONField() # Type hints - id: int + id: int | str task_group_id: int # FIXME: Quick fix involves removing uniqueness constraint diff --git a/main/config.py b/main/config.py index af2fc428..907c99a2 100644 --- a/main/config.py +++ b/main/config.py @@ -44,6 +44,7 @@ class Config: # NOTE: We get mapillary data from mapillary MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" MAPILLARY_API_KEY = typing.cast("str", settings.MAPILLARY_API_KEY) + PANORAMAX_API_LINK = "https://api.panoramax.xyz/" FIREBASE_HELPER = typing.cast("FirebaseHelper", settings.FIREBASE_HELPER) FIREBASE_EMULATOR_USE = typing.cast("str | None", settings.FIREBASE_EMULATOR_USE) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 2b1128fd..1cc095f5 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -25,6 +25,7 @@ from main.logging import log_extra_response from project_types.base.project import ValidationException from utils.common import Grouping +from utils.geo.street_image_provider.models import StreetImageProvider, StreetImageProviderNameEnum from utils.spatial_sampling import spatial_sampling logger = logging.getLogger(__name__) @@ -109,6 +110,7 @@ def coordinate_download( *, polygon: ShapelyBaseGeometry, level: int, + provider: StreetImageProvider, kwargs: dict[str, Any], ) -> pd.DataFrame: tiles = create_tiles( @@ -118,13 +120,18 @@ def coordinate_download( if tiles.empty: return pd.DataFrame() - logger.info("Images will be queried in roughly %s requests from mapillary", len(tiles.index)) + logger.info( + "Images will be queried in roughly %s requests from %s", + len(tiles.index), + getattr(provider.name, "value", "unknown provider"), + ) downloaded_metadata: list[pd.DataFrame] = [] for i, row in enumerate(tiles.to_dict(orient="records")): df = download_and_process_tile( row=row, polygon=polygon, + provider=provider, kwargs=kwargs, ) if df is not None and not df.empty: @@ -168,6 +175,7 @@ def download_and_process_tile( *, row: dict[Hashable, Any] | pd.Series, polygon: ShapelyBaseGeometry, + provider: StreetImageProvider, kwargs: dict[str, Any], attempt_limit: int = 3, ) -> pd.DataFrame | None: @@ -175,14 +183,29 @@ def download_and_process_tile( x = row["x"] y = row["y"] - url = f"{Config.MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={Config.MAPILLARY_API_KEY}" + if provider.name == StreetImageProviderNameEnum.MAPILLARY: + url = f"{provider.url or Config.MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={Config.MAPILLARY_API_KEY}" + elif provider.name == StreetImageProviderNameEnum.PANORAMAX: + url = f"{provider.url or Config.PANORAMAX_API_LINK}api/map/{z}/{x}/{y}.mvt" + else: + raise Exception(f"Unknown provider {getattr(provider.name, 'value', '')}") for _ in range(attempt_limit): try: - data = get_mapillary_data(url, x, y, z) + data = get_street_image_data(url, x, y, z) if data.isna().all() is False or data.empty is False: data = data[data["geometry"].apply(lambda point: point.within(polygon))] + if provider.name == StreetImageProviderNameEnum.PANORAMAX: + data = data.rename( + columns={ + "account_id": "creator_id", + "ts": "captured_at", + "type": "is_pano", + "first_sequence": "sequence_id", + }, + ) + data["is_pano"] = data["is_pano"].eq("equirectangular") target_columns = [ "id", "geometry", @@ -202,7 +225,8 @@ def download_and_process_tile( return None except StreetException: logger.warning( - "Error while fetching Mapillary data for tile %s/%s/%s", + "Error while fetching %s data for tile %s/%s/%s", + provider.name.value, z, x, y, @@ -210,7 +234,7 @@ def download_and_process_tile( return None -def get_mapillary_data( +def get_street_image_data( url: str, x: int, y: int, @@ -219,7 +243,8 @@ def get_mapillary_data( response = requests.get(url, timeout=100) if response.status_code != 200: logger.warning( - "Mapillary API request failed", + "API request at %s failed", + url, extra=log_extra_response(response=response), ) raise StreetException @@ -253,27 +278,27 @@ def filter_results( df = results_df.copy() if creator_id is not None: if df["creator_id"].isna().all(): - logger.info("No Mapillary Feature in the AoI has a 'creator_id' value.") + logger.info("No feature in the AoI has a 'creator_id' value.") return None df = df[df["creator_id"] == creator_id] if is_pano is not None: if df["is_pano"].isna().all(): - logger.info("No Mapillary Feature in the AoI has a 'is_pano' value.") + logger.info("No feature in the AoI has a 'is_pano' value.") return None df = df[df["is_pano"] == is_pano] if organization_id is not None: if df["organization_id"].isna().all(): logger.info( - "No Mapillary Feature in the AoI has an 'organization_id' value.", + "No feature in the AoI has an 'organization_id' value.", ) return None df = df[df["organization_id"] == organization_id] if start_time is not None: if df["captured_at"].isna().all(): - logger.info("No Mapillary Feature in the AoI has a 'captured_at' value.") + logger.info("No feature in the AoI has a 'captured_at' value.") return None df = filter_by_timerange(df, start_time, end_time) @@ -298,6 +323,7 @@ def get_image_metadata( end_time: str | None = None, randomize_order: bool = False, sampling_threshold: int | None = None, + provider: StreetImageProvider | None = None, ) -> Grouping[StreetFeature]: kwargs = { "is_pano": is_pano, @@ -307,14 +333,24 @@ def get_image_metadata( "end_time": end_time, } aoi_polygon = geojson_to_polygon(aoi_geojson) + + if provider is None: + provider = StreetImageProvider( + name=StreetImageProviderNameEnum.MAPILLARY, + url=Config.MAPILLARY_API_LINK, + ) + + level = 15 if getattr(provider.name, "value", None) == "panoramax" else 14 + downloaded_metadata = coordinate_download( polygon=aoi_polygon, level=level, + provider=provider, kwargs=kwargs, ) if downloaded_metadata.empty or downloaded_metadata.isna().all() is True: raise ValidationException( - "No Mapillary features found in the area of interest with the provided filters.", + "No features found in the area of interest with the provided filters.", ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling( diff --git a/project_types/street/project.py b/project_types/street/project.py index db4da688..98419742 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -2,7 +2,6 @@ import logging import math import typing -from enum import Enum from django.contrib.gis.geos import GEOSGeometry from django.core.files.base import ContentFile @@ -27,6 +26,7 @@ from utils.asset_types.models import AoiGeometryAssetProperty from utils.common import Grouping, create_json_dump from utils.custom_options.models import CustomOption +from utils.geo.street_image_provider.models import StreetImageProvider logger = logging.getLogger(__name__) @@ -41,16 +41,6 @@ class StreetMapillaryImageFilters(BaseModel): sampling_threshold: custom_fields.PydanticPositiveInt | None = None -class ImageProviderNameEnum(str, Enum): - MAPILLARY = "mapillary" - PANORAMAX = "panoramax" - - -class StreetImageProvider(BaseModel): - name: ImageProviderNameEnum | None = None - url: custom_fields.PydanticUrl | None = None - - class StreetProjectProperty(base_project.BaseProjectProperty): aoi_geometry: custom_fields.PydanticId custom_options: list[CustomOption] | None = None @@ -112,6 +102,8 @@ def validate(self) -> Grouping[StreetFeature]: mapillary_image_filters = self.project_type_specifics.mapillary_image_filters + provider = self.project_type_specifics.image_provider + return get_image_metadata( aoi_geojson=aoi_geojson, is_pano=mapillary_image_filters.is_pano, @@ -121,6 +113,7 @@ def validate(self) -> Grouping[StreetFeature]: end_time=mapillary_image_filters.end_time, randomize_order=mapillary_image_filters.randomize_order, sampling_threshold=mapillary_image_filters.sampling_threshold, + provider=provider, ) @typing.override @@ -244,8 +237,7 @@ def compress_tasks_on_firebase(self) -> bool: def get_task_specifics_for_firebase(self, task: ProjectTask): assert task.geometry is not None, "Task geometry must not be None" return firebase_models.FbMappingTaskStreetCreateOnlyInput( - # XXX: converting this to int for backwards compatibility - taskId=int(task.firebase_id), + taskId=task.firebase_id, groupId=task.task_group.firebase_id, ) diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index 74259ef5..78c2f9b8 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -20,6 +20,7 @@ geojson_to_polygon, get_image_metadata, ) +from utils.geo.street_image_provider.models import StreetImageProvider, StreetImageProviderNameEnum if typing.TYPE_CHECKING: from collections.abc import Hashable @@ -31,6 +32,7 @@ class TestTileGroupingFunctions(unittest.TestCase): empty_polygon: Polygon # type: ignore[reportUninitializedInstanceVariable] empty_geometry: GeometryCollection # type: ignore[reportUninitializedInstanceVariable] row: pd.Series # type: ignore[reportUninitializedInstanceVariable] + provider: StreetImageProvider # type: ignore[reportUninitializedInstanceVariable] @typing.override @classmethod @@ -50,6 +52,10 @@ def setUp(self): self.empty_polygon = Polygon() self.empty_geometry = GeometryCollection() self.row = pd.Series({"x": 1, "y": 1, "z": self.level}) + self.provider = StreetImageProvider( + name=StreetImageProviderNameEnum.MAPILLARY, + url=Config.MAPILLARY_API_LINK, + ) def test_create_tiles_with_valid_polygon(self): tiles = create_tiles(polygon=self.test_polygon, level=self.level) @@ -176,7 +182,7 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): # polygon = wkt.loads("POLYGON ((-1 -1, -1 1, 1 1, 1 -1, -1 -1))") - result = download_and_process_tile(row=row, polygon=polygon, kwargs={}) + result = download_and_process_tile(row=row, polygon=polygon, kwargs={}, provider=self.provider) assert result is not None assert len(result) == 1 assert result["geometry"][0].wkt == "POINT (0 0)" @@ -187,11 +193,11 @@ def test_download_and_process_tile_failure(self, mock_get): # type: ignore[repo mock_response.status_code = 500 mock_get.return_value = mock_response - result = download_and_process_tile(row=self.row, polygon=self.test_polygon, kwargs={}) + result = download_and_process_tile(row=self.row, polygon=self.test_polygon, kwargs={}, provider=self.provider) assert result is None - @patch("project_types.street.api_calls.get_mapillary_data") - def test_download_and_process_tile_spatial_filtering(self, mock_get_mapillary_data): # type: ignore[reportMissingParameterType] + @patch("project_types.street.api_calls.get_street_image_data") + def test_download_and_process_tile_spatial_filtering(self, mock_get_street_image_data): # type: ignore[reportMissingParameterType] inside_points = [ (0.2, 0.2), (0.5, 0.5), @@ -209,9 +215,9 @@ def test_download_and_process_tile_spatial_filtering(self, mock_get_mapillary_da for x, y in points ] - mock_get_mapillary_data.return_value = pd.DataFrame(data) + mock_get_street_image_data.return_value = pd.DataFrame(data) - metadata = download_and_process_tile(row=self.row, polygon=self.test_polygon, kwargs={}) + metadata = download_and_process_tile(row=self.row, polygon=self.test_polygon, kwargs={}, provider=self.provider) assert metadata is not None metadata = metadata.drop_duplicates() assert len(metadata) == len(inside_points) @@ -220,7 +226,7 @@ def test_download_and_process_tile_spatial_filtering(self, mock_get_mapillary_da def test_coordinate_download_with_failures(self, mock_download_and_process_tile): # type: ignore[reportMissingParameterType] mock_download_and_process_tile.return_value = pd.DataFrame() - metadata = coordinate_download(polygon=self.test_polygon, level=self.level, kwargs={}) + metadata = coordinate_download(polygon=self.test_polygon, level=self.level, kwargs={}, provider=self.provider) assert metadata.empty diff --git a/schema.graphql b/schema.graphql index c66dc63d..01d80f51 100644 --- a/schema.graphql +++ b/schema.graphql @@ -902,6 +902,11 @@ enum IconEnum { WARNING_OUTLINE } +enum ImageProviderNameEnum { + mapillary + panoramax +} + input IntComparisonFilterLookup { """Exact match. Filter will be skipped on `null` value""" exact: Int @@ -2147,6 +2152,16 @@ input StrFilterLookup { startsWith: String } +type StreetImageProvider { + name: ImageProviderNameEnum + url: String +} + +input StreetImageProviderInput { + name: ImageProviderNameEnum = null + url: String = null +} + type StreetMapillaryImageFilters { creatorId: String endTime: String @@ -2171,6 +2186,7 @@ input StreetProjectPropertyInput { """Numeric value as string""" aoiGeometry: String! customOptions: [CustomOptionInput!] = null + imageProvider: StreetImageProviderInput = null mapillaryImageFilters: StreetMapillaryImageFiltersInput! } @@ -2178,6 +2194,7 @@ type StreetProjectPropertyType { """Numeric value as string""" aoiGeometry: String! customOptions: [ProjectCustomOption!] + imageProvider: StreetImageProvider mapillaryImageFilters: StreetMapillaryImageFilters! } diff --git a/utils/geo/street_image_provider/__init__.py b/utils/geo/street_image_provider/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/geo/street_image_provider/models.py b/utils/geo/street_image_provider/models.py new file mode 100644 index 00000000..d90533b4 --- /dev/null +++ b/utils/geo/street_image_provider/models.py @@ -0,0 +1,15 @@ +from enum import Enum + +from pydantic import BaseModel + +from utils import fields as custom_fields + + +class StreetImageProviderNameEnum(str, Enum): + MAPILLARY = "mapillary" + PANORAMAX = "panoramax" + + +class StreetImageProvider(BaseModel): + name: StreetImageProviderNameEnum | None = None + url: custom_fields.PydanticUrl | None = None From a66f00609e2b76050f34969d6d14a9d6244c399d Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 18 Nov 2025 10:53:08 +0100 Subject: [PATCH 03/35] feat(street): update firebase submodule pointer --- firebase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase b/firebase index 8b0d6833..f352c008 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit 8b0d6833579d182dab7d7ea1689e6b5c83d124b7 +Subproject commit f352c00839b06254cd2b0cdfd0e40f07600219cd From 8504d29daa7aaa7f0328713f63c08d99b8b8d6e1 Mon Sep 17 00:00:00 2001 From: ofritz Date: Wed, 11 Feb 2026 17:44:54 +0100 Subject: [PATCH 04/35] feat(street): update schema for image provider selection --- schema.graphql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/schema.graphql b/schema.graphql index 01d80f51..7f5c16cb 100644 --- a/schema.graphql +++ b/schema.graphql @@ -902,11 +902,6 @@ enum IconEnum { WARNING_OUTLINE } -enum ImageProviderNameEnum { - mapillary - panoramax -} - input IntComparisonFilterLookup { """Exact match. Filter will be skipped on `null` value""" exact: Int @@ -2153,15 +2148,20 @@ input StrFilterLookup { } type StreetImageProvider { - name: ImageProviderNameEnum + name: StreetImageProviderNameEnum url: String } input StreetImageProviderInput { - name: ImageProviderNameEnum = null + name: StreetImageProviderNameEnum = null url: String = null } +enum StreetImageProviderNameEnum { + MAPILLARY + PANORAMAX +} + type StreetMapillaryImageFilters { creatorId: String endTime: String From 48ab1991fde6876d213411d5138a61b26c31047d Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 17 Feb 2026 13:15:15 +0100 Subject: [PATCH 05/35] feat(street): extend firebase_id char length to 36 to accommodate panoramax ids --- .../0003_alter_announcement_firebase_id.py | 19 +++++++++++++++ apps/common/models.py | 3 ++- ...er_contributorteam_firebase_id_and_more.py | 24 +++++++++++++++++++ ...alter_organization_firebase_id_and_more.py | 24 +++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 apps/common/migrations/0003_alter_announcement_firebase_id.py create mode 100644 apps/contributor/migrations/0003_alter_contributorteam_firebase_id_and_more.py create mode 100644 apps/project/migrations/0012_alter_organization_firebase_id_and_more.py diff --git a/apps/common/migrations/0003_alter_announcement_firebase_id.py b/apps/common/migrations/0003_alter_announcement_firebase_id.py new file mode 100644 index 00000000..69940894 --- /dev/null +++ b/apps/common/migrations/0003_alter_announcement_firebase_id.py @@ -0,0 +1,19 @@ +# Generated by Django 5.2.5 on 2026-02-17 12:03 + +import ulid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0002_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='announcement', + name='firebase_id', + field=models.CharField(default=ulid.ULID, max_length=36, unique=True), + ), + ] diff --git a/apps/common/models.py b/apps/common/models.py index ee613770..28e01367 100644 --- a/apps/common/models.py +++ b/apps/common/models.py @@ -139,7 +139,8 @@ class FirebasePushResource(Model): # NOTE: We should not directly use old_id. This is ID reference to old system old_id = models.CharField[str | None, str | None](max_length=30, db_index=True, null=True, blank=True) - firebase_id = models.CharField[str, str](max_length=30, unique=True, default=ULID) + # NOTE: Panoramax uses UUIDv4 (length 36) for identifiers, which we use as firebase_id for street project tasks + firebase_id = models.CharField[str, str](max_length=36, unique=True, default=ULID) firebase_push_status: int | None = IntegerChoicesField( # type: ignore[reportAssignmentType] choices_enum=FirebasePushStatusEnum, diff --git a/apps/contributor/migrations/0003_alter_contributorteam_firebase_id_and_more.py b/apps/contributor/migrations/0003_alter_contributorteam_firebase_id_and_more.py new file mode 100644 index 00000000..1cdc8994 --- /dev/null +++ b/apps/contributor/migrations/0003_alter_contributorteam_firebase_id_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.5 on 2026-02-17 12:03 + +import ulid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contributor', '0002_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='contributorteam', + name='firebase_id', + field=models.CharField(default=ulid.ULID, max_length=36, unique=True), + ), + migrations.AlterField( + model_name='contributorusergroup', + name='firebase_id', + field=models.CharField(default=ulid.ULID, max_length=36, unique=True), + ), + ] diff --git a/apps/project/migrations/0012_alter_organization_firebase_id_and_more.py b/apps/project/migrations/0012_alter_organization_firebase_id_and_more.py new file mode 100644 index 00000000..c1ecf950 --- /dev/null +++ b/apps/project/migrations/0012_alter_organization_firebase_id_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.5 on 2026-02-17 12:03 + +import ulid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('project', '0011_alter_projecttask_firebase_id'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='firebase_id', + field=models.CharField(default=ulid.ULID, max_length=36, unique=True), + ), + migrations.AlterField( + model_name='project', + name='firebase_id', + field=models.CharField(default=ulid.ULID, max_length=36, unique=True), + ), + ] From ce75f128630a269ef72ddc13a4c3182a0461d171 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 17 Feb 2026 14:58:07 +0100 Subject: [PATCH 06/35] feat(street): use safe image provider api url --- project_types/street/api_calls.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 1cc095f5..f9851507 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -184,9 +184,11 @@ def download_and_process_tile( y = row["y"] if provider.name == StreetImageProviderNameEnum.MAPILLARY: - url = f"{provider.url or Config.MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={Config.MAPILLARY_API_KEY}" + base = (provider.url or Config.MAPILLARY_API_LINK).rstrip("/") + url = f"{base}/{z}/{x}/{y}?access_token={Config.MAPILLARY_API_KEY}" elif provider.name == StreetImageProviderNameEnum.PANORAMAX: - url = f"{provider.url or Config.PANORAMAX_API_LINK}api/map/{z}/{x}/{y}.mvt" + base = (provider.url or Config.PANORAMAX_API_LINK).rstrip("/") + url = f"{base}/api/map/{z}/{x}/{y}.mvt" else: raise Exception(f"Unknown provider {getattr(provider.name, 'value', '')}") From 8c961dda172ef5565a0d6159555cae28f0b90e71 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 17 Feb 2026 15:00:23 +0100 Subject: [PATCH 07/35] feat(street): update submodule pointer --- firebase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase b/firebase index f352c008..fd352d42 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit f352c00839b06254cd2b0cdfd0e40f07600219cd +Subproject commit fd352d42bdd99d68b84f559cf051e1dd1a8e076a From 7175ac1d02c9c66cb59ca690f750c752dd4e61e3 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 17 Feb 2026 15:01:46 +0100 Subject: [PATCH 08/35] feat(street): update submodule pointer --- firebase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase b/firebase index fd352d42..c7feb4ff 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit fd352d42bdd99d68b84f559cf051e1dd1a8e076a +Subproject commit c7feb4ffd3911aa35b156c08153cdf549a0d76ab From db831b6868580f8f11673f54012888bd31638f01 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 17 Feb 2026 15:21:05 +0100 Subject: [PATCH 09/35] feat(street): add image provider to firebase project type specifics --- project_types/street/project.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/project_types/street/project.py b/project_types/street/project.py index 98419742..6d37b140 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -250,6 +250,7 @@ def get_group_specifics_for_firebase(self, group: ProjectTaskGroup): @typing.override def get_project_specifics_for_firebase(self): custom_opts = self.project_type_specifics.custom_options + image_provider = self.project_type_specifics.image_provider number_of_groups = ProjectTaskGroup.objects.filter(project=self.project).count() return firebase_models.FbProjectStreetCreateOnlyInput( numberOfGroups=number_of_groups, @@ -274,4 +275,8 @@ def get_project_specifics_for_firebase(self): ] if custom_opts is not None else None, + imageProvider=firebase_models.FbObjImageProvider( + name=image_provider.name, + url=image_provider.url, + ) ) From 720704214db6c51d2ac0451ed75ce3897f7cd196 Mon Sep 17 00:00:00 2001 From: ofritz Date: Wed, 18 Feb 2026 17:03:57 +0100 Subject: [PATCH 10/35] feat(street): add image provider to firebase tutorial --- project_types/street/project.py | 2 +- project_types/street/tutorial.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/project_types/street/project.py b/project_types/street/project.py index 6d37b140..791af24d 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -278,5 +278,5 @@ def get_project_specifics_for_firebase(self): imageProvider=firebase_models.FbObjImageProvider( name=image_provider.name, url=image_provider.url, - ) + ), ) diff --git a/project_types/street/tutorial.py b/project_types/street/tutorial.py index cb72a87e..caa62539 100644 --- a/project_types/street/tutorial.py +++ b/project_types/street/tutorial.py @@ -52,6 +52,7 @@ def get_group_specifics_for_firebase(self): @typing.override def get_tutorial_specifics_for_firebase(self): custom_opts = self.project_type_specifics.custom_options + image_provider = self.project_type_specifics.image_provider projectType = ProjectTypeEnum.STREET.value assert projectType == 7, "Project Street should be 7" @@ -79,4 +80,8 @@ def get_tutorial_specifics_for_firebase(self): ] if custom_opts is not None else None, + imageProvider=firebase_models.FbObjImageProvider( + name=image_provider.name, + url=image_provider.url, + ), ) From 8a776df19cfb9b350b47fc15e41013c824db9705 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 19 Feb 2026 17:57:21 +0100 Subject: [PATCH 11/35] feat(street): Correctly convert Mapillary and Panoramax timestamps, ensure correct sorting, sampling threshhold input in m instead km --- firebase | 2 +- project_types/street/api_calls.py | 19 +++++++++++-------- utils/spatial_sampling.py | 7 ++++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/firebase b/firebase index c7feb4ff..75aadc01 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit c7feb4ffd3911aa35b156c08153cdf549a0d76ab +Subproject commit 75aadc01de216216dc46dfe44d3f843869b4fb65 diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index f9851507..8210e2f4 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -208,13 +208,16 @@ def download_and_process_tile( }, ) data["is_pano"] = data["is_pano"].eq("equirectangular") + data["captured_at"] = pd.to_datetime(data["captured_at"], format="mixed", errors="coerce").dt.tz_localize(None) + if provider.name == StreetImageProviderNameEnum.MAPILLARY: + data["captured_at"] = pd.to_datetime(data["captured_at"], unit="ms") target_columns = [ "id", "geometry", "captured_at", "is_pano", "compass_angle", - "sequence", + "sequence_id", "organization_id", ] for col in target_columns: @@ -308,10 +311,10 @@ def filter_results( def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str | None = None) -> pd.DataFrame: - df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") converted_start_time = pd.to_datetime(start_time).tz_localize(None) converted_end_time = pd.to_datetime(end_time).tz_localize(None) if end_time else pd.Timestamp.now().tz_localize(None) - return df[(df["captured_at"] >= converted_start_time) & (df["captured_at"] <= converted_end_time)] + filtered_df = df[(df["captured_at"] >= converted_start_time) & (df["captured_at"] <= converted_end_time)] + return filtered_df def get_image_metadata( @@ -354,11 +357,11 @@ def get_image_metadata( raise ValidationException( "No features found in the area of interest with the provided filters.", ) - if sampling_threshold is not None: - downloaded_metadata = spatial_sampling( - df=downloaded_metadata, - interval_length=sampling_threshold, - ) + + downloaded_metadata = spatial_sampling( + df=downloaded_metadata, + interval_length=sampling_threshold, + ) if randomize_order is True: downloaded_metadata = downloaded_metadata.sample(frac=1).reset_index(drop=True) diff --git a/utils/spatial_sampling.py b/utils/spatial_sampling.py index 11173bdf..3bd2e10d 100644 --- a/utils/spatial_sampling.py +++ b/utils/spatial_sampling.py @@ -108,13 +108,13 @@ def filter_points(df: pd.DataFrame, threshold_distance: float) -> pd.DataFrame: def spatial_sampling( *, df: pd.DataFrame, - interval_length: float, + interval_length: int | None, ): """Calculate spacing between points in a GeoDataFrame. Args: df (pandas.DataFrame): DataFrame containing points with timestamps. - interval_length (float): Interval length for filtering points in kms. + interval_length (float): Interval length for filtering points in m. Returns: geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. @@ -126,6 +126,7 @@ def spatial_sampling( returns the filtered GeoDataFrame along with the total road length. """ + if len(df) == 1: return df @@ -144,7 +145,7 @@ def spatial_sampling( sequence_df = sorted_df[sorted_df["sequence_id"] == sequence] if interval_length: - sequence_df = filter_points(sequence_df, interval_length) + sequence_df = filter_points(sequence_df, interval_length / 1000) if "is_pano" in sequence_df.columns: # below line prevents FutureWarning # (https://stackoverflow.com/questions/73800841/add-series-as-a-new-row-into-dataframe-triggers-futurewarning) From 228bf9de3a1c89c4f18240a7844bc515c36d0958 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 10:47:54 +0100 Subject: [PATCH 12/35] fix(street): format and typechecks --- project_types/street/api_calls.py | 9 ++++++--- project_types/street/project.py | 8 +++++--- project_types/street/tutorial.py | 8 +++++--- utils/spatial_sampling.py | 1 - utils/tests/spatial_sampling_test.py | 4 ++-- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 8210e2f4..83ef6474 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -208,7 +208,11 @@ def download_and_process_tile( }, ) data["is_pano"] = data["is_pano"].eq("equirectangular") - data["captured_at"] = pd.to_datetime(data["captured_at"], format="mixed", errors="coerce").dt.tz_localize(None) + data["captured_at"] = pd.to_datetime( + data["captured_at"], + format="mixed", + errors="coerce", + ).dt.tz_localize(None) if provider.name == StreetImageProviderNameEnum.MAPILLARY: data["captured_at"] = pd.to_datetime(data["captured_at"], unit="ms") target_columns = [ @@ -313,8 +317,7 @@ def filter_results( def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str | None = None) -> pd.DataFrame: converted_start_time = pd.to_datetime(start_time).tz_localize(None) converted_end_time = pd.to_datetime(end_time).tz_localize(None) if end_time else pd.Timestamp.now().tz_localize(None) - filtered_df = df[(df["captured_at"] >= converted_start_time) & (df["captured_at"] <= converted_end_time)] - return filtered_df + return df[(df["captured_at"] >= converted_start_time) & (df["captured_at"] <= converted_end_time)] def get_image_metadata( diff --git a/project_types/street/project.py b/project_types/street/project.py index 791af24d..2ee39923 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -276,7 +276,9 @@ def get_project_specifics_for_firebase(self): if custom_opts is not None else None, imageProvider=firebase_models.FbObjImageProvider( - name=image_provider.name, - url=image_provider.url, - ), + name=image_provider.name.value if image_provider.name else "", + url=image_provider.url if image_provider.url else None, + ) + if image_provider + else None, ) diff --git a/project_types/street/tutorial.py b/project_types/street/tutorial.py index caa62539..af575722 100644 --- a/project_types/street/tutorial.py +++ b/project_types/street/tutorial.py @@ -81,7 +81,9 @@ def get_tutorial_specifics_for_firebase(self): if custom_opts is not None else None, imageProvider=firebase_models.FbObjImageProvider( - name=image_provider.name, - url=image_provider.url, - ), + name=image_provider.name.value if image_provider.name else "", + url=image_provider.url if image_provider.url else None, + ) + if image_provider + else None, ) diff --git a/utils/spatial_sampling.py b/utils/spatial_sampling.py index 3bd2e10d..2a7bde2c 100644 --- a/utils/spatial_sampling.py +++ b/utils/spatial_sampling.py @@ -126,7 +126,6 @@ def spatial_sampling( returns the filtered GeoDataFrame along with the total road length. """ - if len(df) == 1: return df diff --git a/utils/tests/spatial_sampling_test.py b/utils/tests/spatial_sampling_test.py index 6b522915..a884756f 100644 --- a/utils/tests/spatial_sampling_test.py +++ b/utils/tests/spatial_sampling_test.py @@ -74,13 +74,13 @@ def test_spatial_sampling_ordering(self): df = pd.DataFrame(data) df["geometry"] = df["geometry"].apply(wkt.loads) # type: ignore[reportArgumentType] - interval_length = 0.1 + interval_length = 100 filtered_gdf = spatial_sampling(df=df, interval_length=interval_length) assert filtered_gdf["captured_at"].is_monotonic_decreasing def test_spatial_sampling_with_sequence(self): - threshold_distance = 0.01 + threshold_distance = 10 filtered_df = spatial_sampling(df=self.fixture_df, interval_length=threshold_distance) assert isinstance(filtered_df, pd.DataFrame) assert len(filtered_df) < len(self.fixture_df) From 38c22769bf3b323180993124aab567cf31e71b38 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 13:42:43 +0100 Subject: [PATCH 13/35] feat(street): submodule pointer for updated test assets --- assets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets b/assets index 432bbd5a..f1901ddb 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 432bbd5a38ac5bc13bb3f8e2f30e36ccfc3cc41c +Subproject commit f1901ddb5fd83832e1b8fa4309e47a9633fd9244 From 583097f4cd9ecb6041b2c9e14ba4af2988897489 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 15:20:33 +0100 Subject: [PATCH 14/35] feat(street): fix tests --- project_types/street/api_calls.py | 15 +++++++++------ project_types/street/tests/api_calls_test.py | 12 +++++++++++- utils/spatial_sampling.py | 5 +++-- utils/tests/spatial_sampling_test.py | 6 +++--- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 83ef6474..dd19b92e 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -208,12 +208,13 @@ def download_and_process_tile( }, ) data["is_pano"] = data["is_pano"].eq("equirectangular") - data["captured_at"] = pd.to_datetime( - data["captured_at"], - format="mixed", - errors="coerce", - ).dt.tz_localize(None) - if provider.name == StreetImageProviderNameEnum.MAPILLARY: + if "captured_at" in data.columns: + data["captured_at"] = pd.to_datetime( + data["captured_at"], + format="mixed", + errors="coerce", + ).dt.tz_localize(None) + if provider.name == StreetImageProviderNameEnum.MAPILLARY and "captured_at" in data.columns: data["captured_at"] = pd.to_datetime(data["captured_at"], unit="ms") target_columns = [ "id", @@ -317,6 +318,8 @@ def filter_results( def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str | None = None) -> pd.DataFrame: converted_start_time = pd.to_datetime(start_time).tz_localize(None) converted_end_time = pd.to_datetime(end_time).tz_localize(None) if end_time else pd.Timestamp.now().tz_localize(None) + if "captured_at" not in df.columns: + return df.iloc[0:0] return df[(df["captured_at"] >= converted_start_time) & (df["captured_at"] <= converted_end_time)] diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index 78c2f9b8..8059a28a 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -43,6 +43,7 @@ def setUpClass(cls): with Path(Config.BASE_DIR, "assets/fixtures/mapillary_response.csv").open(encoding="utf-8") as file: df = pd.read_csv(file) df["geometry"] = df["geometry"].apply(wkt.loads) # type: ignore[reportArgumentType] + df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") cls.fixture_df = df @typing.override @@ -333,7 +334,14 @@ def test_get_image_metadata_size_restriction( mock_coordinate_download, # type: ignore[reportMissingParameterType] mock_filter_results, # type: ignore[reportMissingParameterType] ): - mock_df = pd.DataFrame({"id": range(1, 100002), "geometry": range(1, 100002)}) + mock_df = pd.DataFrame( + { + "id": range(1, 100002), + "geometry": range(1, 100002), + "captured_at": pd.NaT, + "sequence_id": 1, + }, + ) mock_coordinate_download.return_value = mock_df with pytest.raises(ValidationException): get_image_metadata(aoi_geojson=self.fixture_data) @@ -344,6 +352,8 @@ def test_get_image_metadata_drop_duplicates(self, mock_coordinate_download): # { "id": [1, 2, 2, 3, 4, 4, 5], "geometry": ["a", "b", "b", "c", "d", "d", "e"], + "sequence_id": [1, 1, 1, 1, 1, 1, 1], + "captured_at": pd.to_datetime(["2020-01-01"] * 7), }, ) mock_coordinate_download.return_value = test_df diff --git a/utils/spatial_sampling.py b/utils/spatial_sampling.py index 2a7bde2c..18de3af8 100644 --- a/utils/spatial_sampling.py +++ b/utils/spatial_sampling.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd from numpy.typing import NDArray +from shapely.geometry import Point def distance_on_sphere( @@ -130,10 +131,10 @@ def spatial_sampling( return df df["long"] = df["geometry"].apply( - lambda geom: geom.x if geom.geom_type == "Point" else None, + lambda geom: geom.x if isinstance(geom, Point) else None, ) df["lat"] = df["geometry"].apply( - lambda geom: geom.y if geom.geom_type == "Point" else None, + lambda geom: geom.y if isinstance(geom, Point) else None, ) sorted_df = df.sort_values(by=["captured_at"]) diff --git a/utils/tests/spatial_sampling_test.py b/utils/tests/spatial_sampling_test.py index a884756f..c3d74941 100644 --- a/utils/tests/spatial_sampling_test.py +++ b/utils/tests/spatial_sampling_test.py @@ -49,12 +49,12 @@ def test_filter_points(self): df["geometry"] = df["geometry"].apply(wkt.loads) # type: ignore[reportArgumentType] df["long"] = df["geometry"].apply( - lambda geom: geom.x if geom.geom_type == "Point" else None, + lambda geom: geom.x if isinstance(geom, Point) else None, ) df["lat"] = df["geometry"].apply( - lambda geom: geom.y if geom.geom_type == "Point" else None, + lambda geom: geom.y if isinstance(geom, Point) else None, ) - threshold_distance = 100 + threshold_distance = 200 filtered_df = filter_points(df, threshold_distance) assert isinstance(filtered_df, pd.DataFrame) From c44737868fbf3cc62d0f9c549c286d24738832b1 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 16:41:05 +0100 Subject: [PATCH 15/35] feat(street): improve cov --- project_types/street/api_calls.py | 2 - project_types/street/tests/api_calls_test.py | 68 ++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index dd19b92e..32edbe85 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -318,8 +318,6 @@ def filter_results( def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str | None = None) -> pd.DataFrame: converted_start_time = pd.to_datetime(start_time).tz_localize(None) converted_end_time = pd.to_datetime(end_time).tz_localize(None) if end_time else pd.Timestamp.now().tz_localize(None) - if "captured_at" not in df.columns: - return df.iloc[0:0] return df[(df["captured_at"] >= converted_start_time) & (df["captured_at"] <= converted_end_time)] diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index 8059a28a..d7a25c69 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -188,6 +188,52 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): # assert len(result) == 1 assert result["geometry"][0].wkt == "POINT (0 0)" + @patch("project_types.street.api_calls.get_street_image_data") + def test_download_and_process_tile_panoramax(self, mock_get_street_image_data): + mock_get_street_image_data.return_value = pd.DataFrame( + [{"geometry": wkt.loads("POINT (0 0)"), "account_id": 1, "ts": "2026-02-24T10:00:00Z", + "type": "equirectangular", "first_sequence": 42}] + ) + + row: dict[Hashable, typing.Any] = {"x": 1, "y": 1, "z": 14} + polygon = wkt.loads("POLYGON ((-1 -1, -1 1, 1 1, 1 -1, -1 -1))") + + provider = StreetImageProvider( + name=StreetImageProviderNameEnum.PANORAMAX, + url=None + ) + + result = download_and_process_tile( + row=row, + polygon=polygon, + kwargs={} + , provider=provider + ) + + assert result is not None + assert "creator_id" in result.columns + assert "captured_at" in result.columns + assert pd.api.types.is_datetime64_ns_dtype(result["captured_at"]) + assert result["captured_at"].iloc[0] == pd.to_datetime("2026-02-24T10:00:00Z").tz_localize(None) + assert "is_pano" in result.columns + assert result["is_pano"].iloc[0] + + @patch("project_types.street.api_calls.get_street_image_data") + def test_download_and_process_tile_returns_none(self, mock_get_street_image_data): + mock_get_street_image_data.return_value = pd.DataFrame() + + row: dict[Hashable, typing.Any] = {"x": 1, "y": 1, "z": 14} + polygon = wkt.loads("POLYGON ((-1 -1, -1 1, 1 1, 1 -1, -1 -1))") + + result = download_and_process_tile( + row=row, + polygon=polygon, + provider=self.provider, + kwargs={} + ) + + assert result is None + @patch("project_types.street.api_calls.requests.get") def test_download_and_process_tile_failure(self, mock_get): # type: ignore[reportMissingParameterType] mock_response = MagicMock() @@ -231,6 +277,18 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) assert metadata.empty + @patch("project_types.street.api_calls.create_tiles") + def test_coordinate_download_no_tiles(self, mock_create_tiles): + mock_create_tiles.return_value = pd.DataFrame() + + metadata = coordinate_download( + polygon=None, + level=0, + kwargs={}, + provider=None, + ) + assert metadata.empty + def test_filter_within_time_range(self): start_time = "2016-01-20 00:00:00" end_time = "2022-01-21 23:59:59" @@ -278,6 +336,16 @@ def test_filter_creator_id(self): assert filtered_df is not None assert len(filtered_df) == 3 + def test_filter_results_no_creator_id(self): + df = pd.DataFrame({ + "creator_id": [None, None], + "is_pano": [True, False], + "organization_id": [1, 2], + "captured_at": pd.to_datetime(["2026-02-24", "2026-02-25"]) + }) + filtered_df = filter_results(df, creator_id=123) + assert filtered_df is None + def test_filter_time_range(self): start_time = "2016-01-20 00:00:00" end_time = "2022-01-21 23:59:59" From 0250e187c02d1014f0ea53325f83304c95912d4c Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 16:44:17 +0100 Subject: [PATCH 16/35] feat(streets): format --- project_types/street/tests/api_calls_test.py | 33 +++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index d7a25c69..2c973a4d 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -191,8 +191,15 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): # @patch("project_types.street.api_calls.get_street_image_data") def test_download_and_process_tile_panoramax(self, mock_get_street_image_data): mock_get_street_image_data.return_value = pd.DataFrame( - [{"geometry": wkt.loads("POINT (0 0)"), "account_id": 1, "ts": "2026-02-24T10:00:00Z", - "type": "equirectangular", "first_sequence": 42}] + [ + { + "geometry": wkt.loads("POINT (0 0)"), + "account_id": 1, + "ts": "2026-02-24T10:00:00Z", + "type": "equirectangular", + "first_sequence": 42, + }, + ], ) row: dict[Hashable, typing.Any] = {"x": 1, "y": 1, "z": 14} @@ -200,14 +207,14 @@ def test_download_and_process_tile_panoramax(self, mock_get_street_image_data): provider = StreetImageProvider( name=StreetImageProviderNameEnum.PANORAMAX, - url=None + url=None, ) result = download_and_process_tile( row=row, polygon=polygon, - kwargs={} - , provider=provider + kwargs={}, + provider=provider, ) assert result is not None @@ -229,7 +236,7 @@ def test_download_and_process_tile_returns_none(self, mock_get_street_image_data row=row, polygon=polygon, provider=self.provider, - kwargs={} + kwargs={}, ) assert result is None @@ -337,12 +344,14 @@ def test_filter_creator_id(self): assert len(filtered_df) == 3 def test_filter_results_no_creator_id(self): - df = pd.DataFrame({ - "creator_id": [None, None], - "is_pano": [True, False], - "organization_id": [1, 2], - "captured_at": pd.to_datetime(["2026-02-24", "2026-02-25"]) - }) + df = pd.DataFrame( + { + "creator_id": [None, None], + "is_pano": [True, False], + "organization_id": [1, 2], + "captured_at": pd.to_datetime(["2026-02-24", "2026-02-25"]), + }, + ) filtered_df = filter_results(df, creator_id=123) assert filtered_df is None From 6718dbb73a33946b702820f3bb3de7b2e5f3b323 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 16:50:57 +0100 Subject: [PATCH 17/35] feat(street): fix test --- project_types/street/tests/api_calls_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index 2c973a4d..d1226e59 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -289,10 +289,10 @@ def test_coordinate_download_no_tiles(self, mock_create_tiles): mock_create_tiles.return_value = pd.DataFrame() metadata = coordinate_download( - polygon=None, + polygon=Polygon(), level=0, kwargs={}, - provider=None, + provider=self.provider, ) assert metadata.empty From a9fac16e2fb657acf00e2f6b4b4df97a86b1a520 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 24 Feb 2026 18:33:11 +0100 Subject: [PATCH 18/35] feat(street): update submodule pointers --- assets | 2 +- firebase | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets b/assets index f1901ddb..aafdb5c0 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit f1901ddb5fd83832e1b8fa4309e47a9633fd9244 +Subproject commit aafdb5c05421f10b11d3cdc64208d1b83ba138b2 diff --git a/firebase b/firebase index 75aadc01..91341056 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit 75aadc01de216216dc46dfe44d3f843869b4fb65 +Subproject commit 913410565fea5b03fdb6598671eed1b7878b6ddf From 947446605646ac0bd3a4bf9e5112a0c339801d10 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 19 Mar 2026 16:27:34 +0100 Subject: [PATCH 19/35] feat(street): add panoramax custom to enum, use pano_only instead is_pano --- main/graphql/enums.py | 2 ++ project_types/street/api_calls.py | 25 +++++++++++------ project_types/street/project.py | 29 +++++++++++--------- project_types/street/tests/api_calls_test.py | 6 ++-- schema.graphql | 11 ++++++-- utils/geo/street_image_provider/models.py | 21 ++++++++++---- 6 files changed, 62 insertions(+), 32 deletions(-) diff --git a/main/graphql/enums.py b/main/graphql/enums.py index a992c7ab..6f3a285e 100644 --- a/main/graphql/enums.py +++ b/main/graphql/enums.py @@ -11,10 +11,12 @@ from project_types.validate.project import ValidateObjectSourceTypeEnum from project_types.validate_image.project import ValidateImageSourceTypeEnum from utils.geo.raster_tile_server.config import RasterTileServerNameEnum +from utils.geo.street_image_provider.models import StreetImageProviderNameEnum from utils.geo.vector_tile_server.config import VectorTileServerNameEnum ENUM_TO_STRAWBERRY_ENUMS: list[type] = [ RasterTileServerNameEnum, + StreetImageProviderNameEnum, VectorTileServerNameEnum, ValidateObjectSourceTypeEnum, ValidateImageSourceTypeEnum, diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 32edbe85..07a58b2a 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -38,6 +38,10 @@ class StreetException(Exception): pass +class StreetConnectionError(StreetException): + pass + + def create_tiles( *, polygon: ShapelyBaseGeometry, @@ -186,7 +190,7 @@ def download_and_process_tile( if provider.name == StreetImageProviderNameEnum.MAPILLARY: base = (provider.url or Config.MAPILLARY_API_LINK).rstrip("/") url = f"{base}/{z}/{x}/{y}?access_token={Config.MAPILLARY_API_KEY}" - elif provider.name == StreetImageProviderNameEnum.PANORAMAX: + elif provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM): base = (provider.url or Config.PANORAMAX_API_LINK).rstrip("/") url = f"{base}/api/map/{z}/{x}/{y}.mvt" else: @@ -198,7 +202,7 @@ def download_and_process_tile( if data.isna().all() is False or data.empty is False: data = data[data["geometry"].apply(lambda point: point.within(polygon))] - if provider.name == StreetImageProviderNameEnum.PANORAMAX: + if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM): data = data.rename( columns={ "account_id": "creator_id", @@ -233,6 +237,11 @@ def download_and_process_tile( return data return None + except requests.exceptions.ConnectionError as e: + raise StreetConnectionError( + f"Could not connect to the image provider API. " + f"Please check that the URL is correct and accessible: '{url}'" + ) from e except StreetException: logger.warning( "Error while fetching %s data for tile %s/%s/%s", @@ -280,7 +289,7 @@ def get_street_image_data( def filter_results( results_df: pd.DataFrame, creator_id: int | None = None, - is_pano: bool | None = None, + pano_only: bool | None = None, organization_id: int | None = None, start_time: str | None = None, end_time: str | None = None, @@ -292,11 +301,11 @@ def filter_results( return None df = df[df["creator_id"] == creator_id] - if is_pano is not None: + if pano_only: if df["is_pano"].isna().all(): logger.info("No feature in the AoI has a 'is_pano' value.") return None - df = df[df["is_pano"] == is_pano] + df = df[df["is_pano"] == True] if organization_id is not None: if df["organization_id"].isna().all(): @@ -325,7 +334,7 @@ def get_image_metadata( *, aoi_geojson: dict[str, Any], level: int = 14, - is_pano: bool | None = None, + pano_only: bool | None = None, creator_id: str | None = None, organization_id: str | None = None, start_time: str | None = None, @@ -335,7 +344,7 @@ def get_image_metadata( provider: StreetImageProvider | None = None, ) -> Grouping[StreetFeature]: kwargs = { - "is_pano": is_pano, + "pano_only": pano_only, "creator_id": creator_id, "organization_id": organization_id, "start_time": start_time, @@ -349,7 +358,7 @@ def get_image_metadata( url=Config.MAPILLARY_API_LINK, ) - level = 15 if getattr(provider.name, "value", None) == "panoramax" else 14 + level = 15 if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM) else 14 downloaded_metadata = coordinate_download( polygon=aoi_polygon, diff --git a/project_types/street/project.py b/project_types/street/project.py index 2ee39923..169a686f 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -21,7 +21,7 @@ ) from main.bulk_managers import BulkCreateManager from project_types.base import project as base_project -from project_types.street.api_calls import StreetFeature, get_image_metadata +from project_types.street.api_calls import StreetConnectionError, StreetFeature, get_image_metadata from utils import fields as custom_fields from utils.asset_types.models import AoiGeometryAssetProperty from utils.common import Grouping, create_json_dump @@ -32,7 +32,7 @@ class StreetMapillaryImageFilters(BaseModel): - is_pano: custom_fields.PydanticBool | None = None + pano_only: custom_fields.PydanticBool | None = None creator_id: custom_fields.PydanticLongText | None = None organization_id: custom_fields.PydanticLongText | None = None start_time: custom_fields.PydanticDate | None = None @@ -104,17 +104,20 @@ def validate(self) -> Grouping[StreetFeature]: provider = self.project_type_specifics.image_provider - return get_image_metadata( - aoi_geojson=aoi_geojson, - is_pano=mapillary_image_filters.is_pano, - creator_id=mapillary_image_filters.creator_id, - organization_id=mapillary_image_filters.organization_id, - start_time=mapillary_image_filters.start_time, - end_time=mapillary_image_filters.end_time, - randomize_order=mapillary_image_filters.randomize_order, - sampling_threshold=mapillary_image_filters.sampling_threshold, - provider=provider, - ) + try: + return get_image_metadata( + aoi_geojson=aoi_geojson, + pano_only=mapillary_image_filters.pano_only, + creator_id=mapillary_image_filters.creator_id, + organization_id=mapillary_image_filters.organization_id, + start_time=mapillary_image_filters.start_time, + end_time=mapillary_image_filters.end_time, + randomize_order=mapillary_image_filters.randomize_order, + sampling_threshold=mapillary_image_filters.sampling_threshold, + provider=provider, + ) + except StreetConnectionError as e: + raise base_project.ValidationException(str(e)) from e @typing.override def create_tasks( diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index d1226e59..a309eb44 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -324,12 +324,12 @@ def test_filter_default(self): assert len(filtered_df) == len(self.fixture_df) def test_filter_pano_true(self): - filtered_df = filter_results(self.fixture_df, is_pano=True) + filtered_df = filter_results(self.fixture_df, pano_only=True) assert filtered_df is not None assert len(filtered_df) == 3 def test_filter_pano_false(self): - filtered_df = filter_results(self.fixture_df, is_pano=False) + filtered_df = filter_results(self.fixture_df, pano_only=False) assert filtered_df is not None assert len(filtered_df) == 3 @@ -367,7 +367,7 @@ def test_filter_time_range(self): assert len(filtered_df) == 3 def test_filter_no_rows_after_filter(self): - filtered_df = filter_results(self.fixture_df, is_pano="False") # type: ignore[reportArgumentType] + filtered_df = filter_results(self.fixture_df, pano_only="False") # type: ignore[reportArgumentType] assert filtered_df is not None assert filtered_df.empty diff --git a/schema.graphql b/schema.graphql index 7f5c16cb..91bc3869 100644 --- a/schema.graphql +++ b/schema.graphql @@ -37,6 +37,7 @@ type AppEnumCollection { ProjectStatusEnum: [AppEnumCollectionProjectStatusEnum!]! ProjectTypeEnum: [AppEnumCollectionProjectTypeEnum!]! RasterTileServerNameEnum: [AppEnumCollectionRasterTileServerNameEnum!]! + StreetImageProviderNameEnum: [AppEnumCollectionStreetImageProviderNameEnum!]! TutorialAssetInputTypeEnum: [AppEnumCollectionTutorialAssetInputTypeEnum!]! TutorialInformationPageBlockTypeEnum: [AppEnumCollectionTutorialInformationPageBlockTypeEnum!]! TutorialStatusEnum: [AppEnumCollectionTutorialStatusEnum!]! @@ -120,6 +121,11 @@ type AppEnumCollectionRasterTileServerNameEnum { label: String! } +type AppEnumCollectionStreetImageProviderNameEnum { + key: StreetImageProviderNameEnum! + label: String! +} + type AppEnumCollectionTutorialAssetInputTypeEnum { key: TutorialAssetInputTypeEnum! label: String! @@ -2160,13 +2166,14 @@ input StreetImageProviderInput { enum StreetImageProviderNameEnum { MAPILLARY PANORAMAX + PANORAMAX_CUSTOM } type StreetMapillaryImageFilters { creatorId: String endTime: String - isPano: Boolean organizationId: String + panoOnly: Boolean randomizeOrder: Boolean! samplingThreshold: Int startTime: String @@ -2175,8 +2182,8 @@ type StreetMapillaryImageFilters { input StreetMapillaryImageFiltersInput { creatorId: String = null endTime: String = null - isPano: Boolean = null organizationId: String = null + panoOnly: Boolean = null randomizeOrder: Boolean! = false samplingThreshold: Int = null startTime: String = null diff --git a/utils/geo/street_image_provider/models.py b/utils/geo/street_image_provider/models.py index d90533b4..89db6599 100644 --- a/utils/geo/street_image_provider/models.py +++ b/utils/geo/street_image_provider/models.py @@ -1,15 +1,24 @@ -from enum import Enum - -from pydantic import BaseModel +from django.db import models +from pydantic import BaseModel, field_validator from utils import fields as custom_fields -class StreetImageProviderNameEnum(str, Enum): - MAPILLARY = "mapillary" - PANORAMAX = "panoramax" +class StreetImageProviderNameEnum(models.TextChoices): + MAPILLARY = "mapillary", "Mapillary" + PANORAMAX = "panoramax", "Panoramax (Metacatalog API)" + PANORAMAX_CUSTOM = "panoramax_custom", "Panoramax (Custom API URL)" class StreetImageProvider(BaseModel): name: StreetImageProviderNameEnum | None = None url: custom_fields.PydanticUrl | None = None + + @field_validator("url", mode="after") + @classmethod + def url_only_for_custom_panoramax(cls, v, info): + if v is not None and info.data.get("name") != StreetImageProviderNameEnum.PANORAMAX_CUSTOM: + raise ValueError("url field is only allowed for PANORAMAX_CUSTOM provider") + if v is None and info.data.get("name") == StreetImageProviderNameEnum.PANORAMAX_CUSTOM: + raise ValueError("url field is required for PANORAMAX_CUSTOM provider") + return v From 24eed04552e61542017367704c2cf929850b66ed Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 19 Mar 2026 16:36:06 +0100 Subject: [PATCH 20/35] fix(street): formatting --- project_types/street/api_calls.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 07a58b2a..37865520 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -239,13 +239,12 @@ def download_and_process_tile( return None except requests.exceptions.ConnectionError as e: raise StreetConnectionError( - f"Could not connect to the image provider API. " - f"Please check that the URL is correct and accessible: '{url}'" + f"Could not connect to the image provider API. Please check that the URL is correct and accessible: '{url}'", ) from e except StreetException: logger.warning( "Error while fetching %s data for tile %s/%s/%s", - provider.name.value, + getattr(provider.name, "value", "unknown provider"), z, x, y, @@ -305,7 +304,7 @@ def filter_results( if df["is_pano"].isna().all(): logger.info("No feature in the AoI has a 'is_pano' value.") return None - df = df[df["is_pano"] == True] + df = df[df["is_pano"]] if organization_id is not None: if df["organization_id"].isna().all(): @@ -358,7 +357,11 @@ def get_image_metadata( url=Config.MAPILLARY_API_LINK, ) - level = 15 if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM) else 14 + level = ( + 15 + if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM) + else 14 + ) downloaded_metadata = coordinate_download( polygon=aoi_polygon, From a3ce4854f9f1dcb781b8d90199d82f5d43c1a8d5 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 19 Mar 2026 16:46:05 +0100 Subject: [PATCH 21/35] fix(street): ruff errors --- project_types/street/project.py | 2 +- project_types/street/tutorial.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project_types/street/project.py b/project_types/street/project.py index 169a686f..a87d208c 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -280,7 +280,7 @@ def get_project_specifics_for_firebase(self): else None, imageProvider=firebase_models.FbObjImageProvider( name=image_provider.name.value if image_provider.name else "", - url=image_provider.url if image_provider.url else None, + url=image_provider.url or None, ) if image_provider else None, diff --git a/project_types/street/tutorial.py b/project_types/street/tutorial.py index af575722..1b01218c 100644 --- a/project_types/street/tutorial.py +++ b/project_types/street/tutorial.py @@ -82,7 +82,7 @@ def get_tutorial_specifics_for_firebase(self): else None, imageProvider=firebase_models.FbObjImageProvider( name=image_provider.name.value if image_provider.name else "", - url=image_provider.url if image_provider.url else None, + url=image_provider.url or None, ) if image_provider else None, From 1e67c2b020a63d3310eecdf8c69cde7d5dd72816 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 19 Mar 2026 17:14:21 +0100 Subject: [PATCH 22/35] fix(street): adjust tests --- apps/project/tests/mutation_test.py | 2 +- assets | 2 +- project_types/street/tests/api_calls_test.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/project/tests/mutation_test.py b/apps/project/tests/mutation_test.py index f406ac4a..8a14e937 100644 --- a/apps/project/tests/mutation_test.py +++ b/apps/project/tests/mutation_test.py @@ -1652,7 +1652,7 @@ def test_project_street(self, mock_requests): # type: ignore[reportMissingParam ], }, "mapillaryImageFilters": { - "isPano": True, + "panoOnly": True, "creatorId": None, "organizationId": None, "startTime": None, diff --git a/assets b/assets index aafdb5c0..3cdcd916 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit aafdb5c05421f10b11d3cdc64208d1b83ba138b2 +Subproject commit 3cdcd916e5386b9b8776a120f3fc6e88ac519d2d diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index a309eb44..d5f61ed4 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -55,7 +55,6 @@ def setUp(self): self.row = pd.Series({"x": 1, "y": 1, "z": self.level}) self.provider = StreetImageProvider( name=StreetImageProviderNameEnum.MAPILLARY, - url=Config.MAPILLARY_API_LINK, ) def test_create_tiles_with_valid_polygon(self): From 0951e4d69621cc9462d3b80893471a6eff11eb35 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 19 Mar 2026 17:37:11 +0100 Subject: [PATCH 23/35] feat(street): remove url from provider if Mapillary --- project_types/street/api_calls.py | 11 ++++------- project_types/street/tests/api_calls_test.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 37865520..fbce7b83 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -300,7 +300,7 @@ def filter_results( return None df = df[df["creator_id"] == creator_id] - if pano_only: + if pano_only is True: if df["is_pano"].isna().all(): logger.info("No feature in the AoI has a 'is_pano' value.") return None @@ -354,14 +354,11 @@ def get_image_metadata( if provider is None: provider = StreetImageProvider( name=StreetImageProviderNameEnum.MAPILLARY, - url=Config.MAPILLARY_API_LINK, ) - level = ( - 15 - if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM) - else 14 - ) + level = ( + 15 if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM) else 14 + ) downloaded_metadata = coordinate_download( polygon=aoi_polygon, diff --git a/project_types/street/tests/api_calls_test.py b/project_types/street/tests/api_calls_test.py index d5f61ed4..a9078ee4 100644 --- a/project_types/street/tests/api_calls_test.py +++ b/project_types/street/tests/api_calls_test.py @@ -330,7 +330,7 @@ def test_filter_pano_true(self): def test_filter_pano_false(self): filtered_df = filter_results(self.fixture_df, pano_only=False) assert filtered_df is not None - assert len(filtered_df) == 3 + assert len(filtered_df) == len(self.fixture_df) def test_filter_organization_id(self): filtered_df = filter_results(self.fixture_df, organization_id=1) @@ -368,7 +368,7 @@ def test_filter_time_range(self): def test_filter_no_rows_after_filter(self): filtered_df = filter_results(self.fixture_df, pano_only="False") # type: ignore[reportArgumentType] assert filtered_df is not None - assert filtered_df.empty + assert len(filtered_df) == len(self.fixture_df) def test_filter_missing_columns(self): columns_to_check = [ @@ -380,10 +380,13 @@ def test_filter_missing_columns(self): df_copy = self.fixture_df.copy() df_copy[column] = None - if column == "captured_at": - column = "start_time" # noqa: PLW2901 + filter_key = column + if column == "is_pano": + filter_key = "pano_only" + elif column == "captured_at": + filter_key = "start_time" - result = filter_results(df_copy, **{column: True}) # type: ignore[reportArgumentType] + result = filter_results(df_copy, **{filter_key: True}) # type: ignore[reportArgumentType] assert result is None @patch("project_types.street.api_calls.coordinate_download") From b2b8177e00de465d85a9d7420b3f7b17e9a7c827 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 23 Mar 2026 15:31:21 +0100 Subject: [PATCH 24/35] Update utils/spatial_sampling.py Co-authored-by: Sushil Tiwari --- utils/spatial_sampling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/spatial_sampling.py b/utils/spatial_sampling.py index 18de3af8..daa448e5 100644 --- a/utils/spatial_sampling.py +++ b/utils/spatial_sampling.py @@ -115,7 +115,7 @@ def spatial_sampling( Args: df (pandas.DataFrame): DataFrame containing points with timestamps. - interval_length (float): Interval length for filtering points in m. + interval_length (int): Interval length for filtering points in m. Returns: geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. From ea49f01f54adf03c76efc8c5bd18fc71406bdbce Mon Sep 17 00:00:00 2001 From: ofritz Date: Mon, 23 Mar 2026 16:28:22 +0100 Subject: [PATCH 25/35] refactor(street): move tile level logic to StreetImageProvider and make name required --- project_types/street/api_calls.py | 7 +------ project_types/street/project.py | 14 +++++++++++--- project_types/street/tutorial.py | 2 +- utils/geo/street_image_provider/models.py | 21 +++++++++++++++++++-- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index fbce7b83..7191c637 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -332,7 +332,6 @@ def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str | None def get_image_metadata( *, aoi_geojson: dict[str, Any], - level: int = 14, pano_only: bool | None = None, creator_id: str | None = None, organization_id: str | None = None, @@ -356,13 +355,9 @@ def get_image_metadata( name=StreetImageProviderNameEnum.MAPILLARY, ) - level = ( - 15 if provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM) else 14 - ) - downloaded_metadata = coordinate_download( polygon=aoi_polygon, - level=level, + level=provider.tile_level, provider=provider, kwargs=kwargs, ) diff --git a/project_types/street/project.py b/project_types/street/project.py index a87d208c..743e5edd 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -5,7 +5,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.core.files.base import ContentFile -from pydantic import BaseModel +from pydantic import BaseModel, model_validator from pyfirebase_mapswipe import models as firebase_models from shapely import wkt from ulid import ULID @@ -26,7 +26,7 @@ from utils.asset_types.models import AoiGeometryAssetProperty from utils.common import Grouping, create_json_dump from utils.custom_options.models import CustomOption -from utils.geo.street_image_provider.models import StreetImageProvider +from utils.geo.street_image_provider.models import StreetImageProvider, StreetImageProviderNameEnum logger = logging.getLogger(__name__) @@ -47,6 +47,14 @@ class StreetProjectProperty(base_project.BaseProjectProperty): mapillary_image_filters: StreetMapillaryImageFilters image_provider: StreetImageProvider | None = None + @model_validator(mode="after") + def default_image_provider(self) -> "StreetProjectProperty": + if self.image_provider is None: + self.image_provider = StreetImageProvider( + name=StreetImageProviderNameEnum.MAPILLARY, + ) + return self + class StreetTaskGroupProperty(base_project.BaseProjectTaskGroupProperty): ... @@ -279,7 +287,7 @@ def get_project_specifics_for_firebase(self): if custom_opts is not None else None, imageProvider=firebase_models.FbObjImageProvider( - name=image_provider.name.value if image_provider.name else "", + name=image_provider.name.value, url=image_provider.url or None, ) if image_provider diff --git a/project_types/street/tutorial.py b/project_types/street/tutorial.py index 1b01218c..944a4423 100644 --- a/project_types/street/tutorial.py +++ b/project_types/street/tutorial.py @@ -81,7 +81,7 @@ def get_tutorial_specifics_for_firebase(self): if custom_opts is not None else None, imageProvider=firebase_models.FbObjImageProvider( - name=image_provider.name.value if image_provider.name else "", + name=image_provider.name.value, url=image_provider.url or None, ) if image_provider diff --git a/utils/geo/street_image_provider/models.py b/utils/geo/street_image_provider/models.py index 89db6599..b138c076 100644 --- a/utils/geo/street_image_provider/models.py +++ b/utils/geo/street_image_provider/models.py @@ -1,5 +1,7 @@ +from typing import assert_never + from django.db import models -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, model_validator from utils import fields as custom_fields @@ -11,9 +13,16 @@ class StreetImageProviderNameEnum(models.TextChoices): class StreetImageProvider(BaseModel): - name: StreetImageProviderNameEnum | None = None + name: StreetImageProviderNameEnum = StreetImageProviderNameEnum.MAPILLARY url: custom_fields.PydanticUrl | None = None + @model_validator(mode="before") + @classmethod + def default_name(cls, data): + if isinstance(data, dict) and data.get("name") is None: + data["name"] = StreetImageProviderNameEnum.MAPILLARY + return data + @field_validator("url", mode="after") @classmethod def url_only_for_custom_panoramax(cls, v, info): @@ -22,3 +31,11 @@ def url_only_for_custom_panoramax(cls, v, info): if v is None and info.data.get("name") == StreetImageProviderNameEnum.PANORAMAX_CUSTOM: raise ValueError("url field is required for PANORAMAX_CUSTOM provider") return v + + @property + def tile_level(self) -> int: + if self.name == StreetImageProviderNameEnum.MAPILLARY: + return 14 + if self.name == StreetImageProviderNameEnum.PANORAMAX or self.name == StreetImageProviderNameEnum.PANORAMAX_CUSTOM: + return 15 + assert_never(self.name) From 2fde0830d1c76800c6cf9201681eb55e56ff9b52 Mon Sep 17 00:00:00 2001 From: ofritz Date: Mon, 23 Mar 2026 16:33:38 +0100 Subject: [PATCH 26/35] feat(street): update graphql schema --- schema.graphql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema.graphql b/schema.graphql index 91bc3869..dfb6da9a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -2154,12 +2154,12 @@ input StrFilterLookup { } type StreetImageProvider { - name: StreetImageProviderNameEnum + name: StreetImageProviderNameEnum! url: String } input StreetImageProviderInput { - name: StreetImageProviderNameEnum = null + name: StreetImageProviderNameEnum! = MAPILLARY url: String = null } From 9c432d1766203466bf8c01426849d1a99453bf59 Mon Sep 17 00:00:00 2001 From: ofritz Date: Mon, 23 Mar 2026 16:50:08 +0100 Subject: [PATCH 27/35] test(street): update test assets to reflect changes (default image provider name) --- assets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets b/assets index 3cdcd916..d8f94e75 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 3cdcd916e5386b9b8776a120f3fc6e88ac519d2d +Subproject commit d8f94e75101088ad3e944cbfc1f738b1a7da898c From ef2d2e91fbcc8a3a363aa9dd8ebf54e757a98105 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 2 Apr 2026 10:41:13 +0200 Subject: [PATCH 28/35] fix(street): Use assert_never, cast firebase task id to int when provider is mapillary, correct type hint --- apps/project/models.py | 2 +- project_types/street/api_calls.py | 4 ++-- project_types/street/project.py | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/project/models.py b/apps/project/models.py index ae04ab44..479f4ae2 100644 --- a/apps/project/models.py +++ b/apps/project/models.py @@ -689,7 +689,7 @@ class ProjectTask(FirebasePushResource): project_type_specifics = models.JSONField() # Type hints - id: int | str + id: int task_group_id: int # FIXME: Quick fix involves removing uniqueness constraint diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 7191c637..715b2ad6 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -2,7 +2,7 @@ from collections.abc import Hashable from concurrent.futures import ProcessPoolExecutor from functools import partial -from typing import Any +from typing import Any, assert_never from warnings import deprecated import mercantile # type: ignore[reportMissingTypeStubs] @@ -194,7 +194,7 @@ def download_and_process_tile( base = (provider.url or Config.PANORAMAX_API_LINK).rstrip("/") url = f"{base}/api/map/{z}/{x}/{y}.mvt" else: - raise Exception(f"Unknown provider {getattr(provider.name, 'value', '')}") + assert_never(provider.name) for _ in range(attempt_limit): try: diff --git a/project_types/street/project.py b/project_types/street/project.py index 743e5edd..3e639581 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -247,8 +247,13 @@ def compress_tasks_on_firebase(self) -> bool: @typing.override def get_task_specifics_for_firebase(self, task: ProjectTask): assert task.geometry is not None, "Task geometry must not be None" + image_provider = self.project_type_specifics.image_provider + task_id: int | str = task.firebase_id + if image_provider and image_provider.name == StreetImageProviderNameEnum.MAPILLARY: + task_id = int(task.firebase_id) + return firebase_models.FbMappingTaskStreetCreateOnlyInput( - taskId=task.firebase_id, + taskId=task_id, groupId=task.task_group.firebase_id, ) From 39561ac8c9de114bd80acf1ef9c3af209892dc25 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 2 Apr 2026 11:00:44 +0200 Subject: [PATCH 29/35] fix(street): update firebase submodule pointer --- firebase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase b/firebase index 91341056..fa606a6c 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit 913410565fea5b03fdb6598671eed1b7878b6ddf +Subproject commit fa606a6c03c2fb3af8eea4e4adc065a53d017a61 From 4e771c75ff25a631312711da27cea6f43c8f9f21 Mon Sep 17 00:00:00 2001 From: ofritz Date: Thu, 2 Apr 2026 11:57:46 +0200 Subject: [PATCH 30/35] fix(street): update e2e test, firebase submodule pointer --- apps/project/tests/e2e_create_street_project_test.py | 6 ++++++ firebase | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/project/tests/e2e_create_street_project_test.py b/apps/project/tests/e2e_create_street_project_test.py index 65d93957..0d32451d 100644 --- a/apps/project/tests/e2e_create_street_project_test.py +++ b/apps/project/tests/e2e_create_street_project_test.py @@ -572,6 +572,12 @@ def _test_project(self, filename: str): sanitized_tasks_actual = project_tasks_fb_data sanitized_tasks_expected = test_data["expected_project_tasks_data"] + for tasks in sanitized_tasks_expected.values(): + for task in tasks: + task_id = task.get("taskId") + if isinstance(task_id, str) and task_id.isdigit(): + task["taskId"] = int(task_id) + assert sanitized_tasks_actual == sanitized_tasks_expected, ( "Differences found between expected and actual tasks on project in firebase." ) diff --git a/firebase b/firebase index fa606a6c..ffcb9839 160000 --- a/firebase +++ b/firebase @@ -1 +1 @@ -Subproject commit fa606a6c03c2fb3af8eea4e4adc065a53d017a61 +Subproject commit ffcb983933af2ae76027b6323a40613d06a21bc6 From 78d5b8e89d451f9d383befa5c1e2c903a03b4d10 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 28 Apr 2026 16:20:35 +0200 Subject: [PATCH 31/35] feat(street): remove Mapillary from names --- .../graphql/inputs/project_types/street.py | 4 +- .../graphql/types/project_types/street.py | 4 +- project_types/street/project.py | 4 +- schema.graphql | 40 +++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/apps/project/graphql/inputs/project_types/street.py b/apps/project/graphql/inputs/project_types/street.py index 4884d05a..ad535baf 100644 --- a/apps/project/graphql/inputs/project_types/street.py +++ b/apps/project/graphql/inputs/project_types/street.py @@ -4,8 +4,8 @@ from utils.geo.street_image_provider.models import StreetImageProvider -@strawberry.experimental.pydantic.input(model=street_project.StreetMapillaryImageFilters, all_fields=True) -class StreetMapillaryImageFiltersInput: ... +@strawberry.experimental.pydantic.input(model=street_project.StreetImageFilters, all_fields=True) +class StreetImageFiltersInput: ... @strawberry.experimental.pydantic.input(model=StreetImageProvider, all_fields=True) diff --git a/apps/project/graphql/types/project_types/street.py b/apps/project/graphql/types/project_types/street.py index 8ca5384d..67b920c3 100644 --- a/apps/project/graphql/types/project_types/street.py +++ b/apps/project/graphql/types/project_types/street.py @@ -4,8 +4,8 @@ from utils.geo.street_image_provider.models import StreetImageProvider as StreetImageProviderModel -@strawberry.experimental.pydantic.type(model=street_project.StreetMapillaryImageFilters, all_fields=True) -class StreetMapillaryImageFilters: ... +@strawberry.experimental.pydantic.type(model=street_project.StreetImageFilters, all_fields=True) +class StreetImageFilters: ... @strawberry.experimental.pydantic.type(model=StreetImageProviderModel, all_fields=True) diff --git a/project_types/street/project.py b/project_types/street/project.py index 3e639581..774b0569 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -class StreetMapillaryImageFilters(BaseModel): +class StreetImageFilters(BaseModel): pano_only: custom_fields.PydanticBool | None = None creator_id: custom_fields.PydanticLongText | None = None organization_id: custom_fields.PydanticLongText | None = None @@ -44,7 +44,7 @@ class StreetMapillaryImageFilters(BaseModel): class StreetProjectProperty(base_project.BaseProjectProperty): aoi_geometry: custom_fields.PydanticId custom_options: list[CustomOption] | None = None - mapillary_image_filters: StreetMapillaryImageFilters + mapillary_image_filters: StreetImageFilters image_provider: StreetImageProvider | None = None @model_validator(mode="after") diff --git a/schema.graphql b/schema.graphql index dfb6da9a..0c87f856 100644 --- a/schema.graphql +++ b/schema.graphql @@ -2153,23 +2153,7 @@ input StrFilterLookup { startsWith: String } -type StreetImageProvider { - name: StreetImageProviderNameEnum! - url: String -} - -input StreetImageProviderInput { - name: StreetImageProviderNameEnum! = MAPILLARY - url: String = null -} - -enum StreetImageProviderNameEnum { - MAPILLARY - PANORAMAX - PANORAMAX_CUSTOM -} - -type StreetMapillaryImageFilters { +type StreetImageFilters { creatorId: String endTime: String organizationId: String @@ -2179,7 +2163,7 @@ type StreetMapillaryImageFilters { startTime: String } -input StreetMapillaryImageFiltersInput { +input StreetImageFiltersInput { creatorId: String = null endTime: String = null organizationId: String = null @@ -2189,12 +2173,28 @@ input StreetMapillaryImageFiltersInput { startTime: String = null } +type StreetImageProvider { + name: StreetImageProviderNameEnum! + url: String +} + +input StreetImageProviderInput { + name: StreetImageProviderNameEnum! = MAPILLARY + url: String = null +} + +enum StreetImageProviderNameEnum { + MAPILLARY + PANORAMAX + PANORAMAX_CUSTOM +} + input StreetProjectPropertyInput { """Numeric value as string""" aoiGeometry: String! customOptions: [CustomOptionInput!] = null imageProvider: StreetImageProviderInput = null - mapillaryImageFilters: StreetMapillaryImageFiltersInput! + mapillaryImageFilters: StreetImageFiltersInput! } type StreetProjectPropertyType { @@ -2202,7 +2202,7 @@ type StreetProjectPropertyType { aoiGeometry: String! customOptions: [ProjectCustomOption!] imageProvider: StreetImageProvider - mapillaryImageFilters: StreetMapillaryImageFilters! + mapillaryImageFilters: StreetImageFilters! } input StreetTutorialTaskPropertyInput { From 7071a5917b1e3b30f0cf5779703804eb43ba24a2 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 28 Apr 2026 16:53:30 +0200 Subject: [PATCH 32/35] feat(street): mandatory image provider --- ...fill_image_provider_for_street_projects.py | 32 +++++++++++++++++++ project_types/street/project.py | 12 ++----- schema.graphql | 6 ++-- utils/geo/street_image_provider/models.py | 11 ++----- 4 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 apps/project/migrations/0013_backfill_image_provider_for_street_projects.py diff --git a/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py b/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py new file mode 100644 index 00000000..ac3ab43e --- /dev/null +++ b/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py @@ -0,0 +1,32 @@ +# Generated by Django 5.2.5 on 2026-04-28 14:28 + +from django.db import migrations + +def backfill_image_provider(apps, schema_editor): + Project = apps.get_model('project', 'Project') + + for project in Project._default_manager.all(): + if project.project_type != 7: # STREET type + continue + + if project.project_type_specifics is None: + project.project_type_specifics = {} + + if 'imageProvider' not in project.project_type_specifics: + project.project_type_specifics['imageProvider'] = { + 'name': 'MAPILLARY' + } + project.save(update_fields=['project_type_specifics']) + +def reverse_backfill(apps, schema_editor): + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('project', '0012_alter_organization_firebase_id_and_more'), + ] + + operations = [ + migrations.RunPython(backfill_image_provider, reverse_backfill), + ] diff --git a/project_types/street/project.py b/project_types/street/project.py index 774b0569..ac92fb4d 100644 --- a/project_types/street/project.py +++ b/project_types/street/project.py @@ -5,7 +5,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.core.files.base import ContentFile -from pydantic import BaseModel, model_validator +from pydantic import BaseModel from pyfirebase_mapswipe import models as firebase_models from shapely import wkt from ulid import ULID @@ -45,15 +45,7 @@ class StreetProjectProperty(base_project.BaseProjectProperty): aoi_geometry: custom_fields.PydanticId custom_options: list[CustomOption] | None = None mapillary_image_filters: StreetImageFilters - image_provider: StreetImageProvider | None = None - - @model_validator(mode="after") - def default_image_provider(self) -> "StreetProjectProperty": - if self.image_provider is None: - self.image_provider = StreetImageProvider( - name=StreetImageProviderNameEnum.MAPILLARY, - ) - return self + image_provider: StreetImageProvider class StreetTaskGroupProperty(base_project.BaseProjectTaskGroupProperty): ... diff --git a/schema.graphql b/schema.graphql index 0c87f856..cd6b7a71 100644 --- a/schema.graphql +++ b/schema.graphql @@ -2179,7 +2179,7 @@ type StreetImageProvider { } input StreetImageProviderInput { - name: StreetImageProviderNameEnum! = MAPILLARY + name: StreetImageProviderNameEnum! url: String = null } @@ -2193,7 +2193,7 @@ input StreetProjectPropertyInput { """Numeric value as string""" aoiGeometry: String! customOptions: [CustomOptionInput!] = null - imageProvider: StreetImageProviderInput = null + imageProvider: StreetImageProviderInput! mapillaryImageFilters: StreetImageFiltersInput! } @@ -2201,7 +2201,7 @@ type StreetProjectPropertyType { """Numeric value as string""" aoiGeometry: String! customOptions: [ProjectCustomOption!] - imageProvider: StreetImageProvider + imageProvider: StreetImageProvider! mapillaryImageFilters: StreetImageFilters! } diff --git a/utils/geo/street_image_provider/models.py b/utils/geo/street_image_provider/models.py index b138c076..91ca89a7 100644 --- a/utils/geo/street_image_provider/models.py +++ b/utils/geo/street_image_provider/models.py @@ -1,7 +1,7 @@ from typing import assert_never from django.db import models -from pydantic import BaseModel, field_validator, model_validator +from pydantic import BaseModel, field_validator from utils import fields as custom_fields @@ -13,16 +13,9 @@ class StreetImageProviderNameEnum(models.TextChoices): class StreetImageProvider(BaseModel): - name: StreetImageProviderNameEnum = StreetImageProviderNameEnum.MAPILLARY + name: StreetImageProviderNameEnum url: custom_fields.PydanticUrl | None = None - @model_validator(mode="before") - @classmethod - def default_name(cls, data): - if isinstance(data, dict) and data.get("name") is None: - data["name"] = StreetImageProviderNameEnum.MAPILLARY - return data - @field_validator("url", mode="after") @classmethod def url_only_for_custom_panoramax(cls, v, info): From 3bdd7c0490c676463a18d1c7f063cc215faca19d Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 28 Apr 2026 17:12:11 +0200 Subject: [PATCH 33/35] fix(street): format image provider backfill migration, adapt tests to mandatory image provider --- .../0013_backfill_image_provider_for_street_projects.py | 4 ++-- apps/project/tests/mutation_test.py | 3 +++ assets | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py b/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py index ac3ab43e..06dcbc8e 100644 --- a/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py +++ b/apps/project/migrations/0013_backfill_image_provider_for_street_projects.py @@ -8,10 +8,10 @@ def backfill_image_provider(apps, schema_editor): for project in Project._default_manager.all(): if project.project_type != 7: # STREET type continue - + if project.project_type_specifics is None: project.project_type_specifics = {} - + if 'imageProvider' not in project.project_type_specifics: project.project_type_specifics['imageProvider'] = { 'name': 'MAPILLARY' diff --git a/apps/project/tests/mutation_test.py b/apps/project/tests/mutation_test.py index 8a14e937..21a2d214 100644 --- a/apps/project/tests/mutation_test.py +++ b/apps/project/tests/mutation_test.py @@ -1660,6 +1660,9 @@ def test_project_street(self, mock_requests): # type: ignore[reportMissingParam "randomizeOrder": False, "samplingThreshold": None, }, + "imageProvider": { + "name": "MAPILLARY", + }, }, }, } diff --git a/assets b/assets index d8f94e75..f6c4516f 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit d8f94e75101088ad3e944cbfc1f738b1a7da898c +Subproject commit f6c4516f2b2cda6c03e1b0c1db5bf2981d06686e From 2196cc0e566c577f1f7cbcd38b3b4d4cd2a08d96 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 28 Apr 2026 17:13:05 +0200 Subject: [PATCH 34/35] feat(street): migrate isPano to panoOnly --- ...4_migrate_pano_only_for_street_projects.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 apps/project/migrations/0014_migrate_pano_only_for_street_projects.py diff --git a/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py b/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py new file mode 100644 index 00000000..f4b63671 --- /dev/null +++ b/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py @@ -0,0 +1,75 @@ +# Generated by Django 5.2.5 on 2026-04-28 14:59 + +import logging + +from django.db import migrations + +logger = logging.getLogger(__name__) + + +def migrate_is_pano_to_pano_only(apps, schema_editor): + """ + Migrate isPano field to panoOnly in mapillaryImageFilters + for all STREET-type projects to conform with updated schema. + """ + Project = apps.get_model('project', 'Project') + + migrated_count = 0 + + for project in Project._default_manager.all(): + if project.project_type != 7: # STREET type + continue + + if project.project_type_specifics is None: + continue + + if 'mapillaryImageFilters' in project.project_type_specifics: + filters = project.project_type_specifics['mapillaryImageFilters'] + if isinstance(filters, dict) and 'isPano' in filters: + logger.info( + f"Migrating isPano to panoOnly for project {project.id} ({project.name})" + ) + filters['panoOnly'] = filters.pop('isPano') + project.save(update_fields=['project_type_specifics']) + migrated_count += 1 + + logger.info(f"Field migration completed: {migrated_count} projects migrated isPano to panoOnly") + + +def reverse_pano_migration(apps, schema_editor): + """ + Revert panoOnly back to isPano for compatibility. + """ + Project = apps.get_model('project', 'Project') + + reverted_count = 0 + + for project in Project._default_manager.all(): + if project.project_type != 7: # STREET type + continue + + if project.project_type_specifics is None: + continue + + if 'mapillaryImageFilters' in project.project_type_specifics: + filters = project.project_type_specifics['mapillaryImageFilters'] + if isinstance(filters, dict) and 'panoOnly' in filters: + logger.info( + f"Reverting panoOnly to isPano for project {project.id} ({project.name})" + ) + filters['isPano'] = filters.pop('panoOnly') + project.save(update_fields=['project_type_specifics']) + reverted_count += 1 + + logger.info(f"Reverse migration completed: {reverted_count} projects reverted panoOnly to isPano") + + +class Migration(migrations.Migration): + + dependencies = [ + ('project', '0013_backfill_image_provider_for_street_projects'), + ] + + operations = [ + migrations.RunPython(migrate_is_pano_to_pano_only, reverse_pano_migration), + ] From 5a5553cd46e649b5cd0a62ece187c59073ef9547 Mon Sep 17 00:00:00 2001 From: ofritz Date: Tue, 28 Apr 2026 17:30:38 +0200 Subject: [PATCH 35/35] fix(street): always call Mapillary API at MAPILLARY_API_LINK, fix whitespace in migration --- ...14_migrate_pano_only_for_street_projects.py | 18 +++++++++--------- project_types/street/api_calls.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py b/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py index f4b63671..97a70c46 100644 --- a/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py +++ b/apps/project/migrations/0014_migrate_pano_only_for_street_projects.py @@ -13,16 +13,16 @@ def migrate_is_pano_to_pano_only(apps, schema_editor): for all STREET-type projects to conform with updated schema. """ Project = apps.get_model('project', 'Project') - + migrated_count = 0 - + for project in Project._default_manager.all(): if project.project_type != 7: # STREET type continue - + if project.project_type_specifics is None: continue - + if 'mapillaryImageFilters' in project.project_type_specifics: filters = project.project_type_specifics['mapillaryImageFilters'] if isinstance(filters, dict) and 'isPano' in filters: @@ -32,7 +32,7 @@ def migrate_is_pano_to_pano_only(apps, schema_editor): filters['panoOnly'] = filters.pop('isPano') project.save(update_fields=['project_type_specifics']) migrated_count += 1 - + logger.info(f"Field migration completed: {migrated_count} projects migrated isPano to panoOnly") @@ -41,16 +41,16 @@ def reverse_pano_migration(apps, schema_editor): Revert panoOnly back to isPano for compatibility. """ Project = apps.get_model('project', 'Project') - + reverted_count = 0 for project in Project._default_manager.all(): if project.project_type != 7: # STREET type continue - + if project.project_type_specifics is None: continue - + if 'mapillaryImageFilters' in project.project_type_specifics: filters = project.project_type_specifics['mapillaryImageFilters'] if isinstance(filters, dict) and 'panoOnly' in filters: @@ -60,7 +60,7 @@ def reverse_pano_migration(apps, schema_editor): filters['isPano'] = filters.pop('panoOnly') project.save(update_fields=['project_type_specifics']) reverted_count += 1 - + logger.info(f"Reverse migration completed: {reverted_count} projects reverted panoOnly to isPano") diff --git a/project_types/street/api_calls.py b/project_types/street/api_calls.py index 715b2ad6..c6a44983 100644 --- a/project_types/street/api_calls.py +++ b/project_types/street/api_calls.py @@ -188,7 +188,7 @@ def download_and_process_tile( y = row["y"] if provider.name == StreetImageProviderNameEnum.MAPILLARY: - base = (provider.url or Config.MAPILLARY_API_LINK).rstrip("/") + base = Config.MAPILLARY_API_LINK.rstrip("/") url = f"{base}/{z}/{x}/{y}?access_token={Config.MAPILLARY_API_KEY}" elif provider.name in (StreetImageProviderNameEnum.PANORAMAX, StreetImageProviderNameEnum.PANORAMAX_CUSTOM): base = (provider.url or Config.PANORAMAX_API_LINK).rstrip("/")