support zarrv3

.github/workflows/test.yaml

@@ -18,15 +18,17 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                python: ["3.10", "3.12"]
+                python: ["3.11", "3.13"]
                 os: [ubuntu-latest]
                 include:
                     - os: macos-latest
-                      python: "3.10"
+                      python: "3.11"
                     - os: macos-latest
                       python: "3.12"
                       pip-flags: "--pre"
                       name: "Python 3.12 (pre-release)"
+                    - os: windows-latest
+                      python: "3.11"
 
         env:
             OS: ${{ matrix.os }}

.mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.10
+python_version = 3.11
 
 ignore_errors = False
 warn_redundant_casts = True

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
     os: ubuntu-20.04
     tools:
-        python: "3.10"
+        python: "3.11"
 sphinx:
     configuration: docs/conf.py
     fail_on_warning: true

docs/api/data_formats.md

@@ -1,7 +1,7 @@
 # Data formats (advanced)
 
-The SpatialData format is defined as a set of versioned subclasses of `spatialdata._io.format.SpatialDataFormat`, one per type of element.
-These classes are useful to ensure backward compatibility whenever a major version change is introduced. We also provide pointers to the latest format.
+The SpatialData format is defined as a set of versioned subclasses of `ome_zarr.format.Format`, one per type of element. The `spatialdata.SpatialDataFormatType` is a union type encompassing the possible valid formats.
+These format subclasses are useful to ensure backward compatibility whenever a major version change is introduced. We also provide pointers to the latest format.
 
 ```{eval-rst}
 .. currentmodule:: spatialdata._io.format

docs/api/io.md

@@ -7,6 +7,5 @@ use any of the [spatialdata-io readers](https://spatialdata.scverse.org/projects
 .. currentmodule:: spatialdata
 
 .. autofunction:: read_zarr
-.. autofunction:: save_transformations
 .. autofunction:: get_dask_backing_files
 ```

pyproject.toml

@@ -15,7 +15,7 @@ maintainers = [
 urls.Documentation = "https://spatialdata.scverse.org/en/latest"
 urls.Source = "https://github.com/scverse/spatialdata.git"
 urls.Home-page = "https://github.com/scverse/spatialdata.git"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dynamic= [
   "version" # allow version to be set by git tags
 ]
@@ -25,15 +25,15 @@ dependencies = [
     "anndata>=0.9.1",
     "click",
     "dask-image",
-    "dask>=2024.4.1,<=2024.11.2",
+    "dask>=2024.10.0,<=2024.11.2",
     "datashader",
-    "fsspec",
+    "fsspec[s3,http]",
     "geopandas>=0.14",
     "multiscale_spatial_image>=2.0.3",
     "networkx",
     "numba>=0.55.0",
     "numpy",
-    "ome_zarr>=0.8.4",
+    "ome_zarr>=0.12.2",
     "pandas",
     "pooch",
     "pyarrow",
@@ -44,15 +44,17 @@ dependencies = [
     "scikit-image",
     "scipy",
     "typing_extensions>=4.8.0",
+    "universal_pathlib>=0.2.6",
     "xarray>=2024.10.0",
     "xarray-schema",
     "xarray-spatial>=0.3.5",
-    "zarr<3",
+    "zarr>=3.0.0",
 ]
 
 [project.optional-dependencies]
 dev = [
     "bump2version",
+    "sentry-prevent-cli",
 ]
 test = [
     "pytest",
@@ -97,7 +99,13 @@ xfail_strict = true
 addopts = [
 #    "-Werror",  # if 3rd party libs raise DeprecationWarnings, just use filterwarnings below
     "--import-mode=importlib",  # allow using test files with same name
-    "-s"  # print output from tests
+    "-s",  # print output from tests
+]
+# These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "gpu: run test on GPU using CuPY.",
+    "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
 ]
 # info on how to use this https://stackoverflow.com/questions/57925071/how-do-i-avoid-getting-deprecationwarning-from-inside-dependencies-with-pytest
 filterwarnings = [
@@ -131,7 +139,7 @@ exclude = [
 
 ]
 line-length = 120
-target-version = "py310"
+target-version = "py311"
 
 [tool.ruff.lint]
 ignore = [

src/spatialdata/__init__.py

@@ -42,10 +42,10 @@
     "SpatialData",
     "get_extent",
     "get_centroids",
+    "SpatialDataFormatType",
     "read_zarr",
     "unpad_raster",
     "get_pyramid_levels",
-    "save_transformations",
     "get_dask_backing_files",
     "are_extents_equal",
     "relabel_sequential",
@@ -79,7 +79,7 @@
 )
 from spatialdata._core.query.spatial_query import bounding_box_query, polygon_query
 from spatialdata._core.spatialdata import SpatialData
-from spatialdata._io._utils import get_dask_backing_files, save_transformations
-from spatialdata._io.format import SpatialDataFormat
+from spatialdata._io._utils import get_dask_backing_files
+from spatialdata._io.format import SpatialDataFormatType
 from spatialdata._io.io_zarr import read_zarr
 from spatialdata._utils import get_pyramid_levels, unpad_raster

src/spatialdata/_core/_deepcopy.py

@@ -46,7 +46,7 @@ def _(sdata: SpatialData) -> SpatialData:
     for _, element_name, element in sdata.gen_elements():
         elements_dict[element_name] = deepcopy(element)
     deepcopied_attrs = _deepcopy(sdata.attrs)
-    return SpatialData.from_elements_dict(elements_dict, attrs=deepcopied_attrs)
+    return SpatialData.init_from_elements(elements_dict, attrs=deepcopied_attrs)
 
 
 @deepcopy.register(DataArray)

src/spatialdata/_core/_elements.py

@@ -3,7 +3,6 @@
 from collections import UserDict
 from collections.abc import Iterable, KeysView, ValuesView
 from typing import TypeVar
-from warnings import warn
 
 from anndata import AnnData
 from dask.dataframe import DataFrame as DaskDataFrame
@@ -41,9 +40,7 @@ def _remove_shared_key(self, key: str) -> None:
     @staticmethod
     def _check_key(key: str, element_keys: Iterable[str], shared_keys: set[str | None]) -> None:
         check_valid_name(key)
-        if key in element_keys:
-            warn(f"Key `{key}` already exists. Overwriting it in-memory.", UserWarning, stacklevel=2)
-        else:
+        if key not in element_keys:
             try:
                 check_key_is_case_insensitively_unique(key, shared_keys)
             except ValueError as e:

src/spatialdata/_core/concatenate.py

@@ -68,8 +68,7 @@ def _concatenate_tables(
         TableModel.INSTANCE_KEY: instance_key,
     }
     merged_table.uns[TableModel.ATTRS_KEY] = attrs
-
-    return TableModel().validate(merged_table)
+    return TableModel().parse(merged_table)
 
 
 def concatenate(
@@ -252,6 +251,8 @@ def _fix_ensure_unique_element_names(
         tables_by_sdata.append(tables)
     sdatas_fixed = []
     for elements, tables in zip(elements_by_sdata, tables_by_sdata, strict=True):
-        sdata = SpatialData.init_from_elements(elements, tables=tables)
+        if tables is not None:
+            elements.update(tables)
+        sdata = SpatialData.init_from_elements(elements)
         sdatas_fixed.append(sdata)
     return sdatas_fixed

src/spatialdata/_core/operations/_utils.py

@@ -136,7 +136,7 @@ def transform_to_data_extent(
             set_transformation(el, transformation={coordinate_system: Identity()}, set_all=True)
     for k, v in sdata.tables.items():
         sdata_to_return_elements[k] = v.copy()
-    return SpatialData.from_elements_dict(sdata_to_return_elements, attrs=sdata.attrs)
+    return SpatialData.init_from_elements(sdata_to_return_elements, attrs=sdata.attrs)
 
 
 def _parse_element(

src/spatialdata/_core/operations/aggregate.py

@@ -232,7 +232,7 @@ def _create_sdata_from_table_and_shapes(
             f"Instance key column dtype in table resulting from aggregation cannot be cast to the dtype of"
             f"element {shapes_name}.index"
         ) from err
-    table.obs[region_key] = shapes_name
+    table.obs[region_key] = pd.Categorical([shapes_name] * len(table))
     table = TableModel.parse(table, region=shapes_name, region_key=region_key, instance_key=instance_key)
 
     # labels case, needs conversion from str to int
@@ -242,7 +242,7 @@ def _create_sdata_from_table_and_shapes(
     if deepcopy:
         shapes = _deepcopy(shapes)
 
-    return SpatialData.from_elements_dict({shapes_name: shapes, table_name: table})
+    return SpatialData.init_from_elements({shapes_name: shapes, table_name: table})
 
 
 def _aggregate_image_by_labels(
@@ -440,7 +440,7 @@ def _aggregate_shapes(
         vk = value_key[0]
         if fractions_of_values is not None:
             joined[ONES_COLUMN] = fractions_of_values
-        aggregated = joined.groupby([INDEX, vk])[ONES_COLUMN].agg(agg_func).reset_index()
+        aggregated = joined.groupby([INDEX, vk], observed=False)[ONES_COLUMN].agg(agg_func).reset_index()
         aggregated_values = aggregated[ONES_COLUMN].values
     else:
         if fractions_of_values is not None:
@@ -478,7 +478,7 @@ def _aggregate_shapes(
 
     anndata = ad.AnnData(
         X,
-        obs=pd.DataFrame(index=rows_categories),
+        obs=pd.DataFrame(index=list(map(str, rows_categories))),
         var=pd.DataFrame(index=columns_categories),
     )
 

src/spatialdata/_core/operations/rasterize_bins.py

@@ -132,7 +132,8 @@ def rasterize_bins(
 
         random_indices = RNG.choice(table.n_obs, min(20, table.n_obs), replace=True)
         location_ids = table.obs[instance_key].iloc[random_indices].values
-        sub_df, sub_table = element.loc[location_ids], table[random_indices]
+        sub_df = element.loc[location_ids]
+        sub_table = table[random_indices]
 
         src = np.stack([sub_table.obs[col_key] - min_col, sub_table.obs[row_key] - min_row], axis=1)
         if isinstance(sub_df, GeoDataFrame):
@@ -281,7 +282,7 @@ def rasterize_bins_link_table_to_labels(sdata: SpatialData, table_name: str, ras
         The name of the rasterized labels in the spatial data object.
     """
     _, region_key, instance_key = get_table_keys(sdata[table_name])
-    sdata[table_name].obs[region_key] = rasterized_labels_name
+    sdata[table_name].obs[region_key] = pd.Categorical([rasterized_labels_name] * sdata[table_name].n_obs)
     relabled_instance_key = _get_relabeled_column_name(instance_key)
     sdata.set_table_annotates_spatialelement(
         table_name=table_name, region=rasterized_labels_name, region_key=region_key, instance_key=relabled_instance_key

src/spatialdata/_core/operations/vectorize.py

@@ -272,7 +272,7 @@ def _(gdf: GeoDataFrame, buffer_resolution: int = 16) -> GeoDataFrame:
         if isinstance(gdf.geometry.iloc[0], Point):
             buffered_df = gdf.copy()
             buffered_df["geometry"] = buffered_df.apply(
-                lambda row: row.geometry.buffer(row[ShapesModel.RADIUS_KEY], resolution=buffer_resolution), axis=1
+                lambda row: row.geometry.buffer(row[ShapesModel.RADIUS_KEY], quad_segs=buffer_resolution), axis=1
             )
 
             # Ensure the GeoDataFrame recognizes the updated geometry column

src/spatialdata/_core/query/relational_query.py

@@ -679,7 +679,7 @@ def join_spatialelement_table(
     if sdata is not None:
         elements_dict = _create_sdata_elements_dict_for_join(sdata, spatial_element_names)
     else:
-        derived_sdata = SpatialData.from_elements_dict(dict(zip(spatial_element_names, spatial_elements, strict=True)))
+        derived_sdata = SpatialData.init_from_elements(dict(zip(spatial_element_names, spatial_elements, strict=True)))
         element_types = ["labels", "shapes", "points"]
         elements_dict = defaultdict(lambda: defaultdict(dict))
         for element_type in element_types:
@@ -738,14 +738,6 @@ def match_table_to_element(sdata: SpatialData, element_name: str, table_name: st
     match_element_to_table : Function to match a spatial element to a table.
     join_spatialelement_table : General function, to join spatial elements with a table with more control.
     """
-    if table_name is None:
-        warnings.warn(
-            "Assumption of table with name `table` being present is being deprecated in SpatialData v0.1. "
-            "Please provide the name of the table as argument to table_name.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        table_name = "table"
     _, table = join_spatialelement_table(
         sdata=sdata, spatial_element_names=element_name, table_name=table_name, how="left", match_rows="left"
     )

src/spatialdata/_core/query/spatial_query.py

@@ -797,30 +797,13 @@ def _(
     return queried_polygons
 
 
-# TODO: we can replace the manually triggered deprecation warning heres with the decorator from Wouter
-def _check_deprecated_kwargs(kwargs: dict[str, Any]) -> None:
-    deprecated_args = ["shapes", "points", "images", "labels"]
-    for arg in deprecated_args:
-        if arg in kwargs and kwargs[arg] is False:
-            warnings.warn(
-                f"The '{arg}' argument is deprecated and will be removed in one of the next following releases. Please "
-                f"filter the SpatialData object before calling this function.",
-                DeprecationWarning,
-                stacklevel=2,
-            )
-
-
 @singledispatch
 def polygon_query(
     element: SpatialElement | SpatialData,
     polygon: Polygon | MultiPolygon,
     target_coordinate_system: str,
     filter_table: bool = True,
     clip: bool = False,
-    shapes: bool = True,
-    points: bool = True,
-    images: bool = True,
-    labels: bool = True,
 ) -> SpatialElement | SpatialData | None:
     """
     Query a SpatialData object or a SpatialElement by a polygon or multipolygon.
@@ -845,18 +828,6 @@ def polygon_query(
         Importantly, when clipping is enabled, the circles will be converted to polygons before the clipping. This may
         affect downstream operations that rely on the circle radius or on performance, so it is recommended to disable
         clipping when querying circles or when querying a `SpatialData` object that contains circles.
-    shapes [Deprecated]
-        This argument is now ignored and will be removed. Please filter the SpatialData object before calling this
-        function.
-    points [Deprecated]
-        This argument is now ignored and will be removed. Please filter the SpatialData object before calling this
-        function.
-    images [Deprecated]
-        This argument is now ignored and will be removed. Please filter the SpatialData object before calling this
-        function.
-    labels [Deprecated]
-        This argument is now ignored and will be removed. Please filter the SpatialData object before calling this
-        function.
 
     Returns
     -------
@@ -879,12 +850,7 @@ def _(
     target_coordinate_system: str,
     filter_table: bool = True,
     clip: bool = False,
-    shapes: bool = True,
-    points: bool = True,
-    images: bool = True,
-    labels: bool = True,
 ) -> SpatialData:
-    _check_deprecated_kwargs({"shapes": shapes, "points": points, "images": images, "labels": labels})
     new_elements = {}
     for element_type in ["points", "images", "labels", "shapes"]:
         elements = getattr(sdata, element_type)
@@ -911,7 +877,6 @@ def _(
     return_request_only: bool = False,
     **kwargs: Any,
 ) -> DataArray | DataTree | None:
-    _check_deprecated_kwargs(kwargs)
     gdf = GeoDataFrame(geometry=[polygon])
     min_x, min_y, max_x, max_y = gdf.bounds.values.flatten().tolist()
     return bounding_box_query(
@@ -933,7 +898,6 @@ def _(
 ) -> DaskDataFrame | None:
     from spatialdata.transformations import get_transformation, set_transformation
 
-    _check_deprecated_kwargs(kwargs)
     polygon_gdf = _get_polygon_in_intrinsic_coordinates(points, target_coordinate_system, polygon)
 
     points_gdf = points_dask_dataframe_to_geopandas(points, suppress_z_warning=True)
@@ -966,7 +930,6 @@ def _(
 ) -> GeoDataFrame | None:
     from spatialdata.transformations import get_transformation, set_transformation
 
-    _check_deprecated_kwargs(kwargs)
     polygon_gdf = _get_polygon_in_intrinsic_coordinates(element, target_coordinate_system, polygon)
     polygon = polygon_gdf["geometry"].iloc[0]