From 718818a78d447be7aea05ba324084cdd03b8875f Mon Sep 17 00:00:00 2001 From: sean Date: Thu, 28 Apr 2022 11:01:30 +0200 Subject: [PATCH 1/8] test rioxarray cog --- src/openeo_processes/cubes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index 581c204c..6632f069 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -286,7 +286,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod if len(final_datasets[0].dims) > 3: raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") for idx, dataset in enumerate(final_datasets): - dataset.rio.to_raster(raster_path=dataset_filenames[idx], **options) + dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', **options) # Write and odc product yml file if write_prod: From 9dd572bebaa3457b16339828efaf19f86bf76bb0 Mon Sep 17 00:00:00 2001 From: sean Date: Thu, 28 Apr 2022 11:50:57 +0200 Subject: [PATCH 2/8] cog save result --- src/openeo_processes/cubes.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index 6632f069..f4d8c766 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -226,7 +226,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod format: str, optional data format (default: GTiff) """ - formats = ['gtiff', 'netcdf', 'geotiff'] + formats = ['cog', 'gtiff', 'netcdf', 'geotiff'] if format.lower() in formats: format = format.lower() else: @@ -283,11 +283,19 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod # Iterate datasets and save to tif elif format in ['gtiff','geotiff']: + if len(final_datasets[0].dims) > 3: + raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") + for idx, dataset in enumerate(final_datasets): + dataset.rio.to_raster(raster_path=dataset_filenames[idx], **options) + + elif format in ['cog']: if len(final_datasets[0].dims) > 3: raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") for idx, dataset in enumerate(final_datasets): dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', **options) + + # Write and odc product yml file if write_prod: write_odc_product(datasets[0], output_filepath) From 4b3d688a6d572a7a957722a3f6b8d5162b8eefc6 Mon Sep 17 00:00:00 2001 From: sean Date: Thu, 28 Apr 2022 14:35:52 +0200 Subject: [PATCH 3/8] original solution --- src/openeo_processes/cubes.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index f4d8c766..a044e066 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -226,7 +226,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod format: str, optional data format (default: GTiff) """ - formats = ['cog', 'gtiff', 'netcdf', 'geotiff'] + formats = ['gtiff', 'netcdf', 'geotiff'] if format.lower() in formats: format = format.lower() else: @@ -283,19 +283,11 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod # Iterate datasets and save to tif elif format in ['gtiff','geotiff']: - if len(final_datasets[0].dims) > 3: - raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") - for idx, dataset in enumerate(final_datasets): - dataset.rio.to_raster(raster_path=dataset_filenames[idx], **options) - - elif format in ['cog']: if len(final_datasets[0].dims) > 3: raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") for idx, dataset in enumerate(final_datasets): dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', **options) - - # Write and odc product yml file if write_prod: write_odc_product(datasets[0], output_filepath) From 5c3362bdfe61b1b2eea46d3d8296073a4c8a4af1 Mon Sep 17 00:00:00 2001 From: sean Date: Thu, 28 Apr 2022 16:04:33 +0200 Subject: [PATCH 4/8] add windowed to provide valids cogs --- src/openeo_processes/cubes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index a044e066..caa16b71 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -1,3 +1,4 @@ +from curses import window from datetime import datetime from os.path import splitext from pathlib import Path @@ -286,7 +287,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod if len(final_datasets[0].dims) > 3: raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") for idx, dataset in enumerate(final_datasets): - dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', **options) + dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', windowed=True, **options) # Write and odc product yml file if write_prod: From c50d11dec99a2a4401da50d248c95541d67c0ae9 Mon Sep 17 00:00:00 2001 From: sean Date: Fri, 29 Apr 2022 10:51:44 +0200 Subject: [PATCH 5/8] drop windowed --- src/openeo_processes/cubes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index caa16b71..677fc27b 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -287,7 +287,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod if len(final_datasets[0].dims) > 3: raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") for idx, dataset in enumerate(final_datasets): - dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', windowed=True, **options) + dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG') # Write and odc product yml file if write_prod: From e236623c0eb0dfbc309c7541ebe293d1386c1b04 Mon Sep 17 00:00:00 2001 From: sean Date: Fri, 29 Apr 2022 12:27:13 +0200 Subject: [PATCH 6/8] include options --- src/openeo_processes/cubes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index 677fc27b..82c2cea0 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -287,7 +287,7 @@ def exec_xar(data, output_filepath='out', format='GTiff', options={}, write_prod if len(final_datasets[0].dims) > 3: raise Exception("[!] Not possible to write a 4-dimensional GeoTiff, use NetCDF instead.") for idx, dataset in enumerate(final_datasets): - dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG') + dataset.rio.to_raster(raster_path=dataset_filenames[idx], driver='COG', **options) # Write and odc product yml file if write_prod: From 3fbdafd5e1b2ba82289de5a72cb2b6a1490fd289 Mon Sep 17 00:00:00 2001 From: sean Date: Fri, 29 Apr 2022 12:37:21 +0200 Subject: [PATCH 7/8] prunes added --- src/openeo_processes/cubes.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index 82c2cea0..79c3dff3 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -1,11 +1,7 @@ -from curses import window from datetime import datetime -from os.path import splitext -from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict import numpy as np -import pandas as pd import odc.algo import rioxarray # needed by save_result even if not directly called import xarray as xr @@ -15,9 +11,7 @@ from openeo_processes.errors import DimensionNotAvailable, TooManyDimensions from scipy import optimize from datetime import datetime -import datacube import dask -from datacube.utils.cog import write_cog try: from pyproj import Transformer, CRS except ImportError: From dac046eef861609c16417588f595753991660389 Mon Sep 17 00:00:00 2001 From: sean Date: Fri, 29 Apr 2022 13:26:34 +0200 Subject: [PATCH 8/8] reinstate imports for Lukas's fear --- src/openeo_processes/cubes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/openeo_processes/cubes.py b/src/openeo_processes/cubes.py index 79c3dff3..8b28f25e 100644 --- a/src/openeo_processes/cubes.py +++ b/src/openeo_processes/cubes.py @@ -1,7 +1,10 @@ from datetime import datetime -from typing import Any, Dict +from os.path import splitext +from pathlib import Path +from typing import Any, Dict, List import numpy as np +import pandas as pd import odc.algo import rioxarray # needed by save_result even if not directly called import xarray as xr @@ -10,7 +13,7 @@ from openeo_processes.utils import process, get_time_dimension_from_data, xarray_dataset_from_dask_dataframe, get_equi7_tiles, derive_datasets_and_filenames_from_tiles from openeo_processes.errors import DimensionNotAvailable, TooManyDimensions from scipy import optimize -from datetime import datetime +import datacube import dask try: from pyproj import Transformer, CRS