Skip to content

Improvements #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
meomancer merged 5 commits into from
Jun 3, 2024
Merged

Improvements #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
93143af
Remove procedure name restriction
meomancer Jun 3, 2024
0c35d17
Improve DS Response by calling Materialized View
meomancer Jun 3, 2024
50f8945
Improve DescribeSensor
meomancer Jun 3, 2024
6ee1ce1
Improve GOResponse
meomancer Jun 3, 2024
4c19d92
Improve GetObservation for single measurement
meomancer Jun 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions istsoslib/renderers/DSresponseRender.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,9 @@ def render(DS,sosConfig):

interval = et.SubElement(quantity, "{%s}interval" % ns['swe'])
begin = et.SubElement(interval, "{%s}begin" % ns['swe'])
begin.text = field['begin'].strftime("%Y-%m-%dT%H:%M:%S.%fZ")
begin.text = field['stime_prc'].strftime("%Y-%m-%dT%H:%M:%S.%fZ")
end = et.SubElement(interval, "{%s}end" % ns['swe'])
end.text = field['end'].strftime("%Y-%m-%dT%H:%M:%S.%fZ")
end.text = field['etime_prc'].strftime("%Y-%m-%dT%H:%M:%S.%fZ")
"""
if not (field["desc_opr"]=="" or field["desc_opr"]==None or field["desc_opr"]=="NULL"):
description = et.SubElement(quantity,"{%s}description" % ns["swe"])
Expand Down
10 changes: 1 addition & 9 deletions istsoslib/responders/DSresponse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,6 @@ def __init__(self, filter, pgdb):

# clean up the procedure name to produce a valid file name
filename = filter.procedure
valid_chars = "-_.()'\"<> %s%s" % (string.ascii_letters, string.digits)
for c in filename:
if not c in valid_chars:
raise Exception("procedure name '%s' is not a valid: use only letters or digits!"%(filter.procedure))
filename += '.xml'

self.smlFile = os.path.join(filter.sosConfig.sensorMLpath, 'well.xml')
Expand All @@ -118,11 +114,7 @@ def __init__(self, filter, pgdb):

# TODO:
# IGRAC specified
sqlProc = "SELECT def_opr, name_opr, desc_opr, constr_pro, name_uom, id_pro, po.begin_measurement as begin, po.end_measurement as end"
sqlProc += " FROM %s.observed_properties opr, %s.proc_obs po," % (filter.sosConfig.schema, filter.sosConfig.schema)
sqlProc += " %s.procedures pr, %s.uoms um" % (filter.sosConfig.schema, filter.sosConfig.schema)
sqlProc += " WHERE opr.id_opr=po.id_opr_fk AND pr.id_prc=po.id_prc_fk AND um.id_uom = po.id_uom_fk"
sqlProc += " AND name_prc = %s ORDER BY id_pro"
sqlProc = "SELECT * FROM istsos.observed_properties_sensor WHERE name_prc = %s"
params = (str(filter.procedure),)
try:
self.observedProperties = pgdb.select(sqlProc, params)
Expand Down
206 changes: 84 additions & 122 deletions istsoslib/responders/GOresponse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,13 +703,21 @@ def BuildProcedureList(pgdb,offering,sosConfig):

return list


def BuildProcedureCount(pgdb,offering, procedures, sosConfig):
sql = "SELECT name_prc FROM %s.procedures, %s.off_proc, %s.offerings" %(sosConfig.schema,sosConfig.schema,sosConfig.schema)
sql += f''' WHERE id_prc=id_prc_fk AND id_off=id_off_fk AND name_off='{offering}' AND name_prc IN ({','.join([f"'{procedure}'" for procedure in procedures])})'''
sql += " ORDER BY name_prc"
rows=pgdb.select(sql)

return len(rows)

def BuildOfferingList(pgdb,sosConfig):
list=[]
sql = "SELECT distinct(name_off) FROM %s.offerings" %(sosConfig.schema,)
rows=pgdb.select(sql)
for row in rows:
list.append(row["name_off"])

return list


Expand Down Expand Up @@ -980,161 +988,125 @@ def setData(self, pgdb, row, filter):

valeFieldName = []
qi_field_name = []

multi_obs = len(obspr_res) > 1

for idx, obspr_row in enumerate(obspr_res):
key = f'C{idx}'
if not multi_obs:
key = 'et'

if self.qualityIndex==True:

cols += [
"C%s.val_msr as c%s_v" % (idx, idx),
"COALESCE(C%s.id_qi_fk, %s) as c%s_qi" % (
idx,
"%s.val_msr as %s_v" % (key, key),
"COALESCE(%s.id_qi_fk, %s) as %s_qi" % (
key,
filter.aggregate_nodata_qi,
idx
key
)
]
csv_sql_cols += [
#"C%s.val_msr" % idx,
"COALESCE(C%s.val_msr, %s)" % (idx, filter.aggregate_nodata),
"COALESCE(C%s.id_qi_fk, %s)" % (idx, filter.aggregate_nodata_qi)
"COALESCE(%s.val_msr, %s)" % (key, filter.aggregate_nodata),
"COALESCE(%s.id_qi_fk, %s)" % (key, filter.aggregate_nodata_qi)
]

valeFieldName.append("c%s_v" %(idx))
valeFieldName.append("c%s_qi" %(idx))
qi_field_name.append("C%s.id_qi_fk" %(idx))

else:
cols.append("C%s.val_msr as c%s_v" %(idx,idx))
csv_sql_cols.append("C%s.val_msr" %(idx))
valeFieldName.append("c%s_v" %(idx))
cols.append("%s.val_msr as c%s_v" %(key,idx))
csv_sql_cols.append("%s.val_msr" %(key))
valeFieldName.append("%s_v" %(key))

# If Aggregatation funtion is set
if filter.aggregate_interval != None:
# This accept only numeric results
aggrCols.append(
"COALESCE(%s(nullif(dt.c%s_v, 'NaN')),'%s')" % (
"COALESCE(%s(nullif(dt.%s_v, 'NaN')),'%s')" % (
filter.aggregate_function,
idx,
key,
filter.aggregate_nodata
)
)
csv_aggr_cols.append(
"COALESCE(%s(nullif(dt.c%s_v, 'NaN')),'%s')" % (
"COALESCE(%s(nullif(dt.%s_v, 'NaN')),'%s')" % (
filter.aggregate_function,
idx,
key,
filter.aggregate_nodata
)
)
if self.qualityIndex==True:
aggrCols.append("COALESCE(MIN(dt.c%s_qi),%s) as c%s_qi\n" % (
idx,
aggrCols.append("COALESCE(MIN(dt.%s_qi),%s) as %s_qi\n" % (
key,
filter.aggregate_nodata_qi,
idx
key
))
csv_aggr_cols.append("COALESCE(MIN(dt.c%s_qi),%s)" % (
idx,
csv_aggr_cols.append("COALESCE(MIN(dt.%s_qi),%s)" % (
key,
filter.aggregate_nodata_qi
))

aggrNotNull.append(" c%s_v > -900 " %(idx))

# Set SQL JOINS
join_txt = """
JOIN (
SELECT
A%s.id_msr,
A%s.val_msr,
A%s.id_eti_fk
""" % (idx, idx, idx)

if self.qualityIndex==True:
join_txt += ", A%s.id_qi_fk\n" %(idx)

join_txt += """
FROM
%s.measures A%s
WHERE
A%s.id_pro_fk = '%s'
""" % (
filter.sosConfig.schema, idx,
idx, obspr_row["id_pro"]
)
aggrNotNull.append(" %s_v > -900 " %(key))

# ATTENTION: HERE -999 VALUES ARE EXCLUDED WHEN ASKING AN AGGREAGATE FUNCTION
if filter.aggregate_interval != None: # >> Should be removed because measures data is not inserted if there is a nodata value
join_txt += " AND A%s.val_msr > -900 " % idx
if multi_obs:
# Set SQL JOINS
join_txt = """
JOIN (
SELECT
A%s.id_msr,
A%s.val_msr,
A%s.id_eti_fk
""" % (idx, idx, idx)

# close SQL JOINS
join_txt += " ) as C%s\n" %(idx)
join_txt += " on C%s.id_eti_fk = et.id_eti" %(idx)
joinar.append(join_txt)
if self.qualityIndex==True:
join_txt += ", A%s.id_qi_fk\n" %(idx)

join_txt += """
FROM
%s.measures A%s
WHERE
A%s.id_pro_fk = '%s'
""" % (
filter.sosConfig.schema, idx,
idx, obspr_row["id_pro"]
)

# If MOBILE PROCEDURE
if self.procedureType=="insitu-mobile-point":
join_txt = """
LEFT JOIN (
SELECT
Ax.id_pos,
st_X(ST_Transform(Ax.geom_pos,%s)) as x,
st_Y(ST_Transform(Ax.geom_pos,%s)) as y,
st_Z(ST_Transform(Ax.geom_pos,%s)) as z,
Ax.id_eti_fk
""" %(
filter.srsName,
filter.srsName,
filter.srsName
)
# ATTENTION: HERE -999 VALUES ARE EXCLUDED WHEN ASKING AN AGGREAGATE FUNCTION
if filter.aggregate_interval != None: # >> Should be removed because measures data is not inserted if there is a nodata value
join_txt += " AND A%s.val_msr > -900 " % idx

if self.qualityIndex==True:
join_txt += ", Ax.id_qi_fk as posqi\n"

join_txt += """
FROM
%s.positions Ax,
%s.event_time Bx
WHERE
Ax.id_eti_fk = Bx.id_eti
AND
Bx.id_prc_fk = %s
""" %(
filter.sosConfig.schema,
filter.sosConfig.schema,
row["id_prc"]
)

join_txt += " ) as Cx on Cx.id_eti_fk = et.id_eti\n"
cols.extend([
"Cx.x as x",
"Cx.y as y",
"Cx.z as z"
])
csv_sql_cols.extend([
"Cx.x",
"Cx.y",
"Cx.z"
])
if self.qualityIndex==True:
cols.append("Cx.posqi")
csv_sql_cols.append("Cx.posqi")
# close SQL JOINS
join_txt += " ) as C%s\n" %(idx)
join_txt += " on C%s.id_eti_fk = et.id_eti" %(idx)
joinar.append(join_txt)

joinar.append(join_txt)

# Set FROM CLAUSE
sqlSel += "%s FROM %s.event_time et" % (
", ".join(cols), filter.sosConfig.schema
table = f'{filter.sosConfig.schema}.event_time'
if not multi_obs:
table = 'istsos.measures'

sqlSel += "%s FROM %s et" % (
", ".join(cols), table
)

# Set FROM CLAUSE
csv_sql_sel += "%s FROM %s.event_time et" % (
csv_sql_sel += "%s FROM %s et" % (
(
",".join(cols)
if filter.aggregate_interval != None
else " || ',' || ".join(csv_sql_cols)
),
filter.sosConfig.schema
table
)

# Set WHERE CLAUSES
sqlData = " ".join(joinar)
sqlData += " WHERE et.id_prc_fk=%s\n" %(row["id_prc"])
if not multi_obs:
sqlData += f""" AND et.id_pro_fk = '{obspr_row["id_pro"]}'"""

# Set FILTER ON RESULT (OGC:COMPARISON)
if filter.result:
Expand Down Expand Up @@ -1424,10 +1396,10 @@ def __init__(self, filter, pgdb):
raise sosException.SOSException("InvalidParameterValue","offering","Parameter \"offering\" sent with invalid value: %s - available options for offering are %s" %(filter.offering,off_list))
"""
if filter.procedure:
pl = BuildProcedureList(pgdb, filter.offering, filter.sosConfig)
for p in filter.procedure:
if not p in pl:
raise sosException.SOSException("InvalidParameterValue","procedure","Parameter \"procedure\" sent with invalid value: %s - available options for offering \"%s\": %s"%(p,filter.offering,pl))
# IGRAC SPECIFIED
pl = BuildProcedureCount(pgdb, filter.offering, filter.procedure, filter.sosConfig)
if not pl:
raise sosException.SOSException("InvalidParameterValue","procedure","Parameter \"procedure\" sent with invalid value")

if filter.featureOfInterest:
fl = BuildfeatureOfInterestList(pgdb,filter.offering, filter.sosConfig)
Expand Down Expand Up @@ -1481,35 +1453,25 @@ def __init__(self, filter, pgdb):

# BUILD PROCEDURES LIST
# select part of query
sqlSel = "SELECT DISTINCT"
sqlSel += " id_prc, name_prc, name_oty, stime_prc, etime_prc, time_res_prc"
sqlSel = "SELECT DISTINCT id_prc, name_prc, concat('insitu-fixed-point','') as name_oty, stime_prc, etime_prc, null as time_res_prc"

# from part of query
sqlFrom = "FROM %s.procedures, %s.proc_obs p, %s.observed_properties, %s.uoms," %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
sqlFrom += " %s.off_proc o, %s.offerings, %s.obs_type" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema)
if filter.featureOfInterest or filter.featureOfInterestSpatial:
sqlFrom += " ,%s.foi, %s.feature_type" %(filter.sosConfig.schema,filter.sosConfig.schema)

sqlWhere = "WHERE id_prc=p.id_prc_fk AND id_opr_fk=id_opr AND o.id_prc_fk=id_prc AND id_off_fk=id_off AND id_uom=id_uom_fk AND id_oty=id_oty_fk"
sqlWhere += " AND name_off='%s'" %(filter.offering)

# where condition based on featureOfInterest
if filter.featureOfInterest:
sqlWhere += " AND id_foi=id_foi_fk AND id_fty=id_fty_fk AND (name_foi IN (%s))" %(",".join( [ "'"+f+"'" for f in filter.featureOfInterest.split(",")]))
if filter.featureOfInterestSpatial:
sqlWhere += " AND id_foi_fk=id_foi AND %s" %(filter.featureOfInterestSpatial)
sqlFrom = "FROM istsos.observed_properties_sensor"
sqlWhere = "WHERE "

# where condition based on procedures
if filter.procedure:
sqlWhere += " AND ("
sqlWhere += " ("
procWhere = []
for proc in filter.procedure:
procWhere.append("name_prc='%s'" %(proc))
sqlWhere += " OR ".join(procWhere)
sqlWhere += ")"

# where condition based on observed properties
sqlWhere += " AND ("
if sqlWhere != 'WHERE':
sqlWhere += " AND "
sqlWhere += " ("
obsprWhere = []
for obs in opr_filtered:
obsprWhere.append("def_opr='%s'" %(obs["def_opr"]))
Expand Down