Improve basic functions and extend readme example

README.md

@@ -5,7 +5,7 @@ It's a Python wrapper around [MetopDatasets.jl](https://github.com/eumetsat/Meto
 [MetopDatasets.jl](https://github.com/eumetsat/MetopDatasets.jl) is a package for reading products from the [METOP satellites](https://www.eumetsat.int/our-satellites/metop-series) using the native binary format specified for each product. The METOP satellites are part of the EUMETSAT-POLAR-SYSTEM (EPS) and have produced near real-time, global weather and climate observation since 2007. Learn more METOP and the data access on [EUMETSATs user-portal](https://user.eumetsat.int/dashboard).
 
 ## Status
-Metopdatasetpy is under development and is not ready for use yet.
+MetopPy is under development and is not ready for use yet.
 
 ## Copyright and License
 This code is licensed under MIT license. See file LICENSE for details on the usage and distribution terms.
@@ -56,7 +56,7 @@ reduced_data_files = [f for f in reduced_data_folder.iterdir() if f.is_file()]
 
 test_file_name = next((s for s in reduced_data_files if s.name.startswith("ASCA_SZO")))
 test_file_path = reduced_data_folder / test_file_name
-ds = metop_reader.load_dataset(file_path=str(test_file_path))
+ds = metop_reader.open_dataset(file_path=str(test_file_path), maskingvalue = float("nan"))
 ```
 
 2. Check keys
@@ -86,7 +86,7 @@ print(ds['latitude'])
 
 ```
 latitude (42 × 10)
-  Datatype:    Union{Missing, Float64} (Int32)
+  Datatype:    Float64 (Int32)
   Dimensions:  xtrack × atrack
   Attributes:
    description          = Latitude (-90 to 90 deg)
@@ -96,18 +96,18 @@ latitude (42 × 10)
 
 </details>
 
-4. Read variable
+4. Read variable as neasted Python list
 
 ```python
-from juliacall import Main
 
-# Convert CFVariable to a full Julia Array
-latitude_julia = Main.Array(ds['latitude'])  # preserves the 2D shape
+# Load CFVariable as a full Julia Array
+latitude_julia = metop_reader.as_array(ds['latitude']) # preserves the 2D shape
+latitude_shape = metop_reader.shape(latitude_julia)
 
 # Convert to nested Python list
 latitude_list = [
-  [latitude_julia[i, j] for j in range(latitude_julia.size[1])]
-  for i in range(latitude_julia.size[0])
+  [latitude_julia[i, j] for j in range(latitude_shape[1])]
+  for i in range(latitude_shape[0])
 ]
 
 # Print first 5x5 elements
@@ -128,7 +128,48 @@ for row in latitude_list[:5]:
 
 </details>
 
+5. Read a slice of the variable (This is a good way to limit memory use for large variables)
 
+```python
+longitude_slice = metop_reader.as_array(ds['longitude'][10:14,0:2])
+print(metop_reader.shape(longitude_slice))
+```
+<details>
+
+<summary>Output of the print </summary>
+
+```
+(4, 2)
+```
+
+</details>
+
+6. Covert julia array to numpy (requires that numpy is also installed)
+
+```python
+import numpy as np
+# Covert the julia array as numpy
+longitude_slice_np = np.array(longitude_slice) # "copy = None" can be used to reduce memory
+print(longitude_slice_np)
+```
+<details>
+
+<summary>Output of the print </summary>
+
+```
+[[233.685948 233.033544]
+ [233.276564 232.620004]
+ [232.858097 232.197423]
+ [232.430274 231.765534]]
+```
+
+</details>
+
+7. Close dataset and free file lock
+
+```python
+metop_reader.close_dataset(ds)
+```
 
 ## Development
 

metoppy/metopreader.py

@@ -25,8 +25,7 @@ def __init__(self):
         # Import Julia package installed via juliapkg.json
         Main.seval("import MetopDatasets")
         # Store module and commonly used functions
-        self._keys = Main.MetopDatasets.keys
-        self._load_dataset = Main.MetopDatasets.MetopDataset
+        self._open_dataset = Main.MetopDatasets.MetopDataset
         self._get_test_data_artifact = Main.MetopDatasets.get_test_data_artifact
 
     def get_keys(self, dataset):
@@ -43,27 +42,79 @@ def get_keys(self, dataset):
         list
             The list of keys available in the dataset.
         """
-        return self._keys(dataset)
+        return Main.keys(dataset)
+
+    def as_array(self, variable):
+        """
+        Load the variable as a Julia array.
+
+        Parameters
+        ----------
+        variable : Julia object
+            A variable from a MetopDatasets.MetopDataset object.
+
+        Returns
+        -------
+        Julia array
+            The data from the variable loaded as an array.
+        """
+        return Main.Array(variable)
+
+    def shape(self, variable_or_j_array):
+        """
+        Get the shape a Julia array or variable.
+
+        Parameters
+        ----------
+        variable_or_j_array : Julia object
+            A variable from a MetopDatasets.MetopDataset object or a Julia Array.
+
+        Returns
+        -------
+        Tuple of ints
+            The shape of the variable
+        """
+        return Main.size(variable_or_j_array)
 
-    def load_dataset(self, file_path: str):
+    def open_dataset(self, file_path: str, maskingvalue = Main.missing):
         """
-        Load a dataset from a record path using MetopDatasets.MetopDataset.
+        Open a dataset from a record path using MetopDatasets.MetopDataset.
 
         Parameters
         ----------
         file_path : str
             Path to the dataset record.
 
+        maskingvalue
+            The masking values are used to replace missing observations. Defaults to Julia Missing type. 
+            A recommended alternative is float("nan") which increasse performance for float data.
+
         Returns
         -------
         Julia object
-            A MetopDataset object loaded from the provided path.
+            A MetopDataset object opened from the provided path.
         """
         try:
-            return self._load_dataset(file_path)
+            return self._open_dataset(file_path, maskingvalue = maskingvalue)
         except Exception as e:
-            raise RuntimeError(f"Failed to load dataset: {file_path}") from e
+            raise RuntimeError(f"Failed to open dataset: {file_path}") from e
+
+    def close_dataset(self, dataset):
+        """
+        Close a dataset and free the file lock created by MetopReader.open_dataset
 
+        Parameters
+        ----------
+        dataset : Julia object
+            A dataset object created by MetopDatasets.MetopDataset.
+
+        Returns
+        -------
+        None
+        """
+        Main.close(dataset)
+        return None
+
     def get_test_data_artifact(self):
         """
         Retrieve the test dataset artifact from MetopDatasets.

metoppy/tests/test_get_test_data_artifact.py

@@ -12,7 +12,6 @@
 
 # TODO: Convert to pytest
 from pathlib import Path
-from juliacall import Main
 from metoppy.metopreader import MetopReader
 
 metop_reader = MetopReader()
@@ -23,20 +22,21 @@
 
 test_file_name = next((s for s in reduced_data_files if s.name.startswith("ASCA_SZO")))
 test_file_path = reduced_data_folder / test_file_name
-ds = metop_reader.load_dataset(file_path=str(test_file_path))
+ds = metop_reader.open_dataset(file_path=str(test_file_path), maskingvalue = float("nan"))
 
 keys = metop_reader.get_keys(ds)
 print(list(keys))
 
 print(ds["latitude"])
 
 # Convert CFVariable to a full Julia Array
-latitude_julia = Main.Array(ds["latitude"])  # preserves the 2D shape
+latitude_julia = metop_reader.as_array(ds['latitude'])  # preserves the 2D shape
+latitude_shape = metop_reader.shape(latitude_julia)
 
 # Convert to nested Python list
 latitude_list = [
-    [latitude_julia[i, j] for j in range(latitude_julia.size[1])]
-    for i in range(latitude_julia.size[0])
+    [latitude_julia[i, j] for j in range(latitude_shape[1])]
+    for i in range(latitude_shape[0])
 ]
 
 # Print first 5x5 elements