From 9c8b1db04cb3af14a93c20a4f9cc868d2c7804b4 Mon Sep 17 00:00:00 2001
From: samkoebrich <samkoebrich@gmail.com>
Date: Thu, 13 Feb 2025 12:08:57 -0700
Subject: [PATCH] Speed up historical forecast parsing by avoiding
 pd.json_normalize

---
 watttime/api.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/watttime/api.py b/watttime/api.py
index 4d451dff..eb4e6347 100644
--- a/watttime/api.py
+++ b/watttime/api.py
@@ -395,13 +395,16 @@ def _parse_historical_forecast_json(
         Returns:
             pd.DataFrame: A pandas DataFrame containing the parsed historical forecast data.
         """
-        out = pd.DataFrame()
-        for json in json_list:
-            for entry in json.get("data", []):
-                _df = pd.json_normalize(entry, record_path=["forecast"])
-                _df = _df.assign(generated_at=pd.to_datetime(entry["generated_at"]))
-                out = pd.concat([out, _df], ignore_index=True)
-        return out
+        data = []
+        for j in json_list:
+            for gen_at in j["data"]:
+                for point_time in gen_at["forecast"]:
+                    point_time["generated_at"] = gen_at["generated_at"]
+                    data.append(point_time)
+        df = pd.DataFrame.from_records(data)
+        df["point_time"] = pd.to_datetime(df["point_time"])
+        df["generated_at"] = pd.to_datetime(df["generated_at"])
+        return df
 
     def get_forecast_json(
         self,