Skip to content
This repository was archived by the owner on Mar 6, 2023. It is now read-only.

Handle large area processing #49

Merged
sophieherrmann merged 19 commits into from
Oct 6, 2021
Merged

Handle large area processing #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
e3e87f3
Changed odc_load_helper to improve CPU and MEM USAGE
ValentinaHutter Sep 8, 2021
d87ca75
Change in fit_curve output_size
ValentinaHutter Sep 8, 2021
f9ed43e
Updated save_result, updated processes that load values, no values lo…
ValentinaHutter Sep 15, 2021
1f8f9c1
Added attributes
ValentinaHutter Sep 24, 2021
618173b
Update comparison processes to check datatype of input
ValentinaHutter Sep 24, 2021
c84442e
Changed attributes in math processes
ValentinaHutter Sep 24, 2021
f75148f
Added rename_dimension process, needed for save_result process
ValentinaHutter Sep 27, 2021
f182978
Inserted keep_attrs function
ValentinaHutter Sep 28, 2021
9bbeab3
Save netCDF files with save_mfdataset
ValentinaHutter Sep 28, 2021
4c61600
Create netcdf file with multiple timestamps.
sophieherrmann Sep 30, 2021
ccbd2cb
Use xr.curvefit for fit curve function
ValentinaHutter Oct 4, 2021
49de4ab
rechunk data in fit curve
ValentinaHutter Oct 4, 2021
252394d
rechunk time
ValentinaHutter Oct 4, 2021
e661815
No only rename dimension, but whole coordinate.
sophieherrmann Oct 4, 2021
f658a7d
rechunk data time, without rechunking x,y
ValentinaHutter Oct 4, 2021
7c5c459
try splitting in to arrays
ValentinaHutter Oct 4, 2021
c1600c8
old fit_curve version
ValentinaHutter Oct 4, 2021
2113956
Split fit curve into two datacubes
ValentinaHutter Oct 4, 2021
09675ca
old fit_curve version
ValentinaHutter Oct 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions src/openeo_processes/arrays.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -923,8 +923,6 @@ def exec_xar(data, ignore_nodata=True, dimension=None):
while (data_first.fillna(999) != data_first).any() and i < len(data_dim):
data_first = data_first.fillna(data_dim[i])
i += 1
else:
pass
return data_first

@staticmethod
Expand Down Expand Up @@ -1030,8 +1028,6 @@ def exec_xar(data, ignore_nodata=True, dimension=None):
while (data_last.fillna(999) != data_last).any() and i >= 0:
data_last = data_last.fillna(data_dim[i])
i -= 1
else:
pass
return data_last

@staticmethod
Expand Down
61 changes: 31 additions & 30 deletions src/openeo_processes/comparison.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import xarray as xr
from openeo_processes.utils import process
from openeo_processes.utils import str2time
from openeo_processes.utils import keep_attrs


# TODO: test if this works for different data types
Expand Down Expand Up @@ -383,9 +384,9 @@ def exec_xar(x, y, delta=False, case_sensitive=True, reduce=False): # TODO: add

Parameters
----------
x : xr.DataArray
x : xr.DataArray, integer, float
First operand.
y : xr.DataArray
y : xr.DataArray, integer, float
Second operand.
delta : float, optional
Only applicable for comparing two arrays containing numbers. If this optional parameter is set to a
Expand Down Expand Up @@ -430,7 +431,7 @@ def exec_xar(x, y, delta=False, case_sensitive=True, reduce=False): # TODO: add
ar_eq = x_time == y_time # comparison of dates
else:
ar_eq = x == y

ar_eq = keep_attrs(x, y, ar_eq)
if reduce:
return ar_eq.all()
else:
Expand Down Expand Up @@ -546,9 +547,9 @@ def exec_xar(x, y, delta=None, case_sensitive=True, reduce=False): # TODO: add

Parameters
----------
x : xr.DataArray
x : xr.DataArray, integer, float
First operand.
y : xr.DataArray
y : xr.DataArray, integer, float
Second operand.
delta : float, optional
Only applicable for comparing two arrays containing numbers. If this optional parameter is set to a
Expand Down Expand Up @@ -687,7 +688,7 @@ def exec_xar(x, y, reduce=False):

Parameters
----------
x : xr.DataArray
x : xr.DataArray, integer, float
First operand.
y : xr.DataArray, integer, float
Second operand.
Expand All @@ -700,18 +701,17 @@ def exec_xar(x, y, reduce=False):
xr.DataArray: :
Returns True if `x` is strictly greater than `y`, None if any operand is None, otherwise False.
"""

## x has to be a datacube, whereas y can be another datacube, an integer or a float
Comment thread
sophieherrmann marked this conversation as resolved.

if x is None or y is None:
return None
elif isinstance(y, xr.DataArray) or isinstance(y, int) or isinstance(y, float):
elif type(x) in [int, float, xr.DataArray] and type(y) in [int, float, xr.DataArray]:
gt_ar = x > y
gt_ar = keep_attrs(x, y, gt_ar)
if reduce:
return gt_ar.all()
else:
return gt_ar
else:
return False
return False

@staticmethod
def exec_da():
Expand Down Expand Up @@ -827,9 +827,9 @@ def exec_xar(x, y, reduce = False):

Parameters
----------
x : xr.DataArray
x : xr.DataArray, integer, float
First operand.
y : xr.DataArray
y : xr.DataArray, integer, float
Second operand.
reduce : bool, optional
If True, one value will be returned.
Expand All @@ -843,14 +843,14 @@ def exec_xar(x, y, reduce = False):
"""
if x is None or y is None:
return None
elif isinstance(y, xr.DataArray) or isinstance(y, int) or isinstance(y, float):
elif type(x) in [int, float, xr.DataArray] and type(y) in [int, float, xr.DataArray]:
gte_ar = ((x-y) >= 0)
gte_ar = keep_attrs(x, y, gte_ar)
if reduce:
return gte_ar.all()
else:
return gte_ar
else:
return False
return False

@staticmethod
def exec_da():
Expand Down Expand Up @@ -966,9 +966,9 @@ def exec_xar(x, y, reduce = False):

Parameters
----------
x : xr.DataArray
x : xr.DataArray, integer, float
First operand.
y : xr.DataArray
y : xr.DataArray, integer, float
Second operand.
reduce : bool, optional
If True, one value will be returned.
Expand All @@ -980,17 +980,16 @@ def exec_xar(x, y, reduce = False):
Returns True if `x` is strictly lower than `y`, None if any operand is None, otherwise False.

"""
## x has to be a datacube, whereas y can be another datacube, an integer or a float
if x is None or y is None:
return None
elif isinstance(y, xr.DataArray) or isinstance(y, int) or isinstance(y, float):
elif type(x) in [int, float, xr.DataArray] and type(y) in [int, float, xr.DataArray]:
lt_ar = x < y
lt_ar = keep_attrs(x, y, lt_ar)
if reduce:
return lt_ar.all()
else:
return lt_ar
else:
return False
return False

@staticmethod
def exec_da():
Expand Down Expand Up @@ -1106,9 +1105,9 @@ def exec_xar(x, y, reduce = False):

Parameters
----------
x : xr.DataArray
x : xr.DataArray, integer, float
First operand.
y : xr.DataArray
y : xr.DataArray, integer, float
Second operand.
reduce : bool, optional
If True, one value will be returned.
Expand All @@ -1120,17 +1119,16 @@ def exec_xar(x, y, reduce = False):
Returns True if `x` is strictly lower than or equal to `y`, None if any operand is None, otherwise False.

"""
## x has to be a datacube, whereas y can be another datacube, an integer or a float
if x is None or y is None:
return None
elif isinstance(y, xr.DataArray) or isinstance(y, int) or isinstance(y, float):
elif type(x) in [int, float, xr.DataArray] and type(y) in [int, float, xr.DataArray]:
lte_ar = x <= y
lte_ar = keep_attrs(x, y, lte_ar)
if reduce:
return lte_ar.all()
else:
return lte_ar
else:
return False
return False

@staticmethod
def exec_da():
Expand Down Expand Up @@ -1286,9 +1284,12 @@ def exec_xar(x, min, max, exclude_max=False, reduce=False):
return False

if exclude_max:
return xr.ufuncs.logical_and(Gte.exec_xar(x, min, reduce=reduce) , Lt.exec_xar(x, max, reduce=reduce))
bet = xr.ufuncs.logical_and(Gte.exec_xar(x, min, reduce=reduce) , Lt.exec_xar(x, max, reduce=reduce))
else:
return xr.ufuncs.logical_and(Gte.exec_xar(x, min, reduce=reduce) , Lte.exec_xar(x, max, reduce=reduce))
bet = xr.ufuncs.logical_and(Gte.exec_xar(x, min, reduce=reduce) , Lte.exec_xar(x, max, reduce=reduce))
if isinstance(x, xr.DataArray):
bet.attrs = x.attrs
return bet

@staticmethod
def exec_da():
Expand Down
Loading