diff --git a/app/blueprints/frontend.py b/app/blueprints/frontend.py
index 8936fc8..0eb3229 100644
--- a/app/blueprints/frontend.py
+++ b/app/blueprints/frontend.py
@@ -4,6 +4,7 @@
from typing import Dict
import pandas as pd
+import pytz
from flask import Blueprint
from flask import current_app
from flask import flash
@@ -67,7 +68,9 @@ def flag_widget_params(force_display: bool = False) -> Dict[str, Any]:
return dict(
boathouses=boathouses,
website_options=website_options,
- model_last_updated_time=get_latest_prediction_time(),
+ model_last_updated_time=get_latest_prediction_time().astimezone(
+ pytz.timezone("US/Eastern")
+ ),
boating_season=force_display or website_options.boating_season,
flagging_message=website_options.rendered_flagging_message,
)
diff --git a/app/data/processing/hobolink.py b/app/data/processing/hobolink.py
index 130f263..468374c 100644
--- a/app/data/processing/hobolink.py
+++ b/app/data/processing/hobolink.py
@@ -48,42 +48,67 @@ def get_live_hobolink_data(
end_date = datetime.now(tz=UTC)
if start_date is None:
start_date = end_date - timedelta(days=days_ago)
- res = request_to_hobolink(start_date=start_date, end_date=end_date, loggers=loggers)
- df = parse_hobolink_data(res.json(), exclude_sensors=exclude_sensors)
+ data = request_to_hobolink(start_date=start_date, end_date=end_date, loggers=loggers)
+ df = parse_hobolink_data(data, exclude_sensors=exclude_sensors)
return df
def request_to_hobolink(
start_date: datetime, end_date: datetime, loggers: str = None, token: str | None = None
-) -> requests.models.Response:
+) -> list[dict[str, Any]]:
""" """
if loggers is None:
loggers = current_app.config["HOBOLINK_LOGGERS"]
if token is None:
token = current_app.config["HOBOLINK_BEARER_TOKEN"]
- res = requests.get(
- urljoin(BASE_URL, "/v1/data"),
- params={
- "start_date_time": start_date.strftime("%Y-%m-%d %H:%M:%S"),
- "end_date_time": end_date.strftime("%Y-%m-%d %H:%M:%S"),
- "loggers": loggers,
- },
- headers={"Authorization": f"Bearer {token}", "accept": "application/json"},
- )
-
- if res.status_code >= 400:
- error_msg = (
- f"API request to the HOBOlink endpoint failed with status code {res.status_code}:"
- + res.text
+ # HOBOLink API returns max of 100,000 results at a time.
+ # Additionally, there is no pagination.
+ # Therefore we must paginate ourselves.
+ #
+ # Note that API timestamps also pull full closed interval of data,
+ # so we need to be careful to not accidentally pull duplicates by timestamp.
+ pagination_delta = timedelta(days=10)
+ epsilon_delta = timedelta(seconds=1)
+ data: list[dict[str, Any]] = []
+
+ start_date_for_req = start_date
+ end_date_for_req = min(start_date + pagination_delta, end_date)
+ half_interval = False
+
+ while True:
+ res = requests.get(
+ urljoin(BASE_URL, "/v1/data"),
+ params={
+ "start_date_time": start_date_for_req.strftime("%Y-%m-%d %H:%M:%S"),
+ "end_date_time": end_date_for_req.strftime("%Y-%m-%d %H:%M:%S"),
+ "loggers": loggers,
+ },
+ headers={"Authorization": f"Bearer {token}", "accept": "application/json"},
)
- abort(500, error_msg)
+ if res.status_code >= 400:
+ error_msg = (
+ f"API request to the HOBOlink endpoint failed with status code {res.status_code}:"
+ + res.text
+ )
+ abort(500, error_msg)
- return res
+ data.extend(res.json()["data"])
+
+ if end_date_for_req == end_date:
+ break
+
+ end_date_for_req += pagination_delta
+ start_date_for_req += pagination_delta
+ if not half_interval:
+ start_date_for_req += epsilon_delta
+ half_interval = True
+
+ return data
def parse_hobolink_data(
- data: dict[str, Any], exclude_sensors: list[str] | None = None
+ data: list[dict[str, Any]], exclude_sensors: list[str] | None = None
) -> pd.DataFrame:
"""
Clean the response from the HOBOlink API.
@@ -98,7 +123,7 @@ def parse_hobolink_data(
# This sensor is for internal temp of device.
exclude_sensors = current_app.config["HOBOLINK_EXCLUDE_SENSORS"]
- df = pd.DataFrame(data["data"])
+ df = pd.DataFrame(data)
df = df.loc[~df["sensor_sn"].isin(exclude_sensors), :]
df["time"] = pd.to_datetime(df["timestamp"])
df["sensor_measurement_type"] = df["sensor_measurement_type"].str.lower().str.replace(" ", "_")
diff --git a/app/templates/boathouses.html b/app/templates/boathouses.html
index eda1db0..78d65f9 100644
--- a/app/templates/boathouses.html
+++ b/app/templates/boathouses.html
@@ -21,6 +21,6 @@ Map
{%with flagging_message='' %}{% include get_widget_filename(version=2) %}{% endwith %}
- ( Last updated: {{ model_last_updated_time | strftime }} )
+ ( Last updated: {{ model_last_updated_time | strftime("%B %-d, %Y at %-I:%M %p") }} )
{% endblock %}
diff --git a/app/templates/flags.html b/app/templates/flags.html
index e9884fc..a98544c 100644
--- a/app/templates/flags.html
+++ b/app/templates/flags.html
@@ -36,7 +36,7 @@
{% include get_widget_filename(version=version) %}
- ( Last updated: {{ model_last_updated_time | strftime }} )
+ ( Last updated: {{ model_last_updated_time | strftime("%B %-d, %Y at %-I:%M %p") }} )
Click here for more information.
diff --git a/app/templates/index.html b/app/templates/index.html
index 5ea2f7c..b39c3ed 100644
--- a/app/templates/index.html
+++ b/app/templates/index.html
@@ -14,7 +14,7 @@
Water Quality Today
{% include get_widget_filename() %}
- ( Last updated: {{ model_last_updated_time | strftime }} )
+ ( Last updated: {{ model_last_updated_time | strftime("%B %-d, %Y at %-I:%M %p") }} )
What Do the Flags Mean?