From 321d86bbf60038d65cb3cb9466d9f74a639ab30f Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sun, 23 Feb 2020 23:37:03 -0800 Subject: [PATCH 01/13] fix soiling_hsu for tmy not monotonic * if tmy is coerced to year, then the last hour is in the same year, before the other timestamps, so timestamps are no monotonic * closes #889 * add underscore to is_tmy * add is_tmy to soiling_hsu * add private _fix_tmy_monotonicity * also fix reference footnote in first line of docstring, since used in api.rst, causes doc warning * don't use mutable input args * don't create pd.timedelta object every single time soiling_hsu() called * always create soiling pd.Series for consistency and robust to future edits * fix rain_index is not a tz-aware pd.DatetimeIndex series, so rename as rain_index_vals and use rainfall.index in Soiling * add new test for gh889, check soiling ratio for 24 hours, and check timestamp of first and last hours, and last day --- .../plot_greensboro_kimber_soiling.py | 2 +- pvlib/losses.py | 73 ++++++++++++++----- pvlib/tests/test_losses.py | 43 ++++++++++- 3 files changed, 94 insertions(+), 24 deletions(-) diff --git a/docs/examples/plot_greensboro_kimber_soiling.py b/docs/examples/plot_greensboro_kimber_soiling.py index 03b445b986..b0ac10cc87 100644 --- a/docs/examples/plot_greensboro_kimber_soiling.py +++ b/docs/examples/plot_greensboro_kimber_soiling.py @@ -42,7 +42,7 @@ # calculate soiling with no wash dates THRESHOLD = 25.0 soiling_no_wash = soiling_kimber( - greensboro_rain, cleaning_threshold=THRESHOLD, istmy=True) + greensboro_rain, cleaning_threshold=THRESHOLD, is_tmy=True) soiling_no_wash.name = 'soiling' # daily rain totals daily_rain = greensboro_rain.iloc[:-1].resample('D').sum() diff --git a/pvlib/losses.py b/pvlib/losses.py index 6bb6df9df1..990954f866 100644 --- a/pvlib/losses.py +++ b/pvlib/losses.py @@ -9,13 +9,20 @@ import pandas as pd from pvlib.tools import cosd +_RAIN_ACC_PERIOD = pd.Timedelta('1h') +_DEPO_VELOCITY = {'2_5': 0.004, '10': 0.0009} + def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, - depo_veloc={'2_5': 0.004, '10': 0.0009}, - rain_accum_period=pd.Timedelta('1h')): + depo_veloc=None, rain_accum_period=_RAIN_ACC_PERIOD, + is_tmy=False): """ Calculates soiling ratio given particulate and rain data using the model - from Humboldt State University [1]_. + from Humboldt State University (HSU). + + The HSU soiling model [1]_ returns the soiling ratio, a value between zero + and one which is equivalent to (1 - transmission loss). Therefore a soiling + ratio of 1.0 is equivalent to zero transmission loss. Parameters ---------- @@ -46,6 +53,9 @@ def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, It is recommended that `rain_accum_period` be between 1 hour and 24 hours. + is_tmy : bool, default False + Fix last timestep in TMY so that it is monotonically increasing. + Returns ------- soiling_ratio : Series @@ -65,6 +75,22 @@ def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, except ImportError: raise ImportError("The soiling_hsu function requires scipy.") + # never use mutable input arguments + if depo_veloc is None: + depo_veloc = _DEPO_VELOCITY + + # if TMY fix to be monotonically increasing by rolling index by 1 interval + # and then adding 1 interval, while the values stay the same + if is_tmy: + # get rainfall timezone, timestep as timedelta64 + rain_tz = rainfall.index.tz + rain_index = rainfall.index.values + timestep_interval = (rain_index[1] - rain_index[0]) + rain_vals = rainfall.values + rain_name = rainfall.name + rainfall = _fix_tmy_monotonicity( + rain_index, rain_vals, timestep_interval, rain_tz, rain_name) + # accumulate rainfall into periods for comparison with threshold accum_rain = rainfall.rolling(rain_accum_period, closed='right').sum() # cleaning is True for intervals with rainfall greater than threshold @@ -89,14 +115,26 @@ def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, return soiling_ratio +def _fix_tmy_monotonicity(rain_index, rain_vals, timestep_interval, rain_tz, + rain_name): + # fix TMY to be monotonically increasing by rolling index by 1 interval + # and then adding 1 interval, while the values stay the same + rain_index = np.roll(rain_index, 1) + timestep_interval + # NOTE: numpy datetim64[ns] has no timezone + # convert to datetimeindex at UTC and convert to original timezone + rain_index = pd.DatetimeIndex(rain_index, tz='UTC').tz_convert(rain_tz) + # fixed rainfall timeseries with monotonically increasing index + return pd.Series(rain_vals, index=rain_index, name=rain_name) + + def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, grace_period=14, max_soiling=0.3, manual_wash_dates=None, - initial_soiling=0, rain_accum_period=24, istmy=False): + initial_soiling=0, rain_accum_period=24, is_tmy=False): """ Calculate soiling ratio with rainfall data and a daily soiling rate using - the Kimber soiling model [1]_. + the Kimber soiling model. - Kimber soiling model assumes soiling builds up at a daily rate unless + Kimber soiling model [1]_ assumes soiling builds up at a daily rate unless the daily rainfall is greater than a threshold. The model also assumes that if daily rainfall has exceeded the threshold within a grace period, then the ground is too damp to cause soiling build-up. The model also assumes @@ -127,7 +165,7 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, rain_accum_period : int, default 24 Period for accumulating rainfall to check against `cleaning_threshold`. The Kimber model defines this period as one day. [hours] - istmy : bool, default False + is_tmy : bool, default False Fix last timestep in TMY so that it is monotonically increasing. Returns @@ -166,22 +204,19 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, # convert grace_period to timedelta grace_period = datetime.timedelta(days=grace_period) - # get rainfall timezone, timestep as timedelta64, and timestep in int days + # get rainfall timezone, timestep as timedelta64, and timestep as day-frac rain_tz = rainfall.index.tz - rain_index = rainfall.index.values - timestep_interval = (rain_index[1] - rain_index[0]) + rain_index_vals = rainfall.index.values + timestep_interval = (rain_index_vals[1] - rain_index_vals[0]) day_fraction = timestep_interval / np.timedelta64(24, 'h') # if TMY fix to be monotonically increasing by rolling index by 1 interval # and then adding 1 interval, while the values stay the same - if istmy: - rain_index = np.roll(rain_index, 1) + timestep_interval - # NOTE: numpy datetim64[ns] has no timezone - # convert to datetimeindex at UTC and convert to original timezone - rain_index = pd.DatetimeIndex(rain_index, tz='UTC').tz_convert(rain_tz) - # fixed rainfall timeseries with monotonically increasing index - rainfall = pd.Series( - rainfall.values, index=rain_index, name=rainfall.name) + if is_tmy: + rain_vals = rainfall.values + rain_name = rainfall.name + rainfall = _fix_tmy_monotonicity( + rain_index_vals, rain_vals, timestep_interval, rain_tz, rain_name) # accumulate rainfall accumulated_rainfall = rainfall.rolling( @@ -191,6 +226,7 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, soiling = np.ones_like(rainfall.values) * soiling_loss_rate * day_fraction soiling[0] = initial_soiling soiling = np.cumsum(soiling) + soiling = pd.Series(soiling, index=rainfall.index, name='soiling') # rainfall events that clean the panels rain_events = accumulated_rainfall > cleaning_threshold @@ -206,7 +242,6 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, # manual wash dates if manual_wash_dates is not None: manual_wash_dates = pd.DatetimeIndex(manual_wash_dates, tz=rain_tz) - soiling = pd.Series(soiling, index=rain_index, name='soiling') cleaning[manual_wash_dates] = soiling[manual_wash_dates] # remove soiling by foward filling cleaning where NaN diff --git a/pvlib/tests/test_losses.py b/pvlib/tests/test_losses.py index d86babd4ed..75623202b9 100644 --- a/pvlib/tests/test_losses.py +++ b/pvlib/tests/test_losses.py @@ -10,6 +10,7 @@ from conftest import ( requires_scipy, needs_pandas_0_22, DATA_DIR) import pytest +import pytz @pytest.fixture @@ -121,7 +122,7 @@ def test_kimber_soiling_nowash(greensboro_rain, # Greensboro typical expected annual rainfall is 8345mm assert greensboro_rain.sum() == 8345 # calculate soiling with no wash dates - soiling_nowash = soiling_kimber(greensboro_rain, istmy=True) + soiling_nowash = soiling_kimber(greensboro_rain, is_tmy=True) # test no washes assert np.allclose( soiling_nowash.values, @@ -143,7 +144,7 @@ def test_kimber_soiling_manwash(greensboro_rain, manwash = [datetime.date(1990, 2, 15), ] # calculate soiling with manual wash soiling_manwash = soiling_kimber( - greensboro_rain, manual_wash_dates=manwash, istmy=True) + greensboro_rain, manual_wash_dates=manwash, is_tmy=True) # test manual wash assert np.allclose( soiling_manwash.values, @@ -168,7 +169,7 @@ def test_kimber_soiling_norain(greensboro_rain, # a year with no rain norain = pd.Series(0, index=greensboro_rain.index) # calculate soiling with no rain - soiling_norain = soiling_kimber(norain, istmy=True) + soiling_norain = soiling_kimber(norain, is_tmy=True) # test no rain, soiling reaches maximum assert np.allclose(soiling_norain.values, expected_kimber_soiling_norain) @@ -191,7 +192,41 @@ def test_kimber_soiling_initial_soil(greensboro_rain, # a year with no rain norain = pd.Series(0, index=greensboro_rain.index) # calculate soiling with no rain - soiling_norain = soiling_kimber(norain, initial_soiling=0.1, istmy=True) + soiling_norain = soiling_kimber(norain, initial_soiling=0.1, is_tmy=True) # test no rain, soiling reaches maximum assert np.allclose( soiling_norain.values, expected_kimber_soiling_initial_soil) + + +@pytest.fixture +def expected_greensboro_hsu_soil(): + return np.array([ + 0.99927224, 0.99869067, 0.99815393, 0.99764437, 0.99715412, + 0.99667873, 0.99621536, 0.99576203, 0.99531731, 0.99488010, + 0.99444954, 0.99402494, 0.99360572, 0.99319142, 1.00000000, + 1.00000000, 0.99927224, 0.99869067, 0.99815393, 0.99764437, + 0.99715412, 1.00000000, 0.99927224, 0.99869067]) + + +@requires_scipy +def test_gh889_soiing_hsu_tmy_not_monotonic(expected_greensboro_hsu_soil): + """doesn't raise value error""" + greensboro = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) + greensboro_rain = greensboro[0].Lprecipdepth + soiling_ratio = soiling_hsu( + greensboro_rain, cleaning_threshold=10.0, tilt=0.0, pm2_5=1.0, + pm10=2.0, is_tmy=True) + # check first day of soiling ratio, actually (1 - transmission loss) + # greensboro rains hours 3pm, 4pm, and 10pm, so expect soiling ratio of one + assert np.allclose(expected_greensboro_hsu_soil, soiling_ratio.values[:24]) + # greensboro timezone is UTC-5 or Eastern time + gmt_5 = pytz.timezone('Etc/GMT+5') + # check last day, should be 1991 now + lastday = datetime.datetime(1991, 1, 1, 0, 0, 0) + assert gmt_5.localize(lastday) == soiling_ratio.index[-1] + # check last hour is still 1990 + lasthour = datetime.datetime(1990, 12, 31, 23, 0, 0) + assert gmt_5.localize(lasthour) == soiling_ratio.index[-2] + # check first hour is still 1990 + firsthour = datetime.datetime(1990, 1, 1, 1, 0, 0) + assert gmt_5.localize(firsthour) == soiling_ratio.index[0] From 2b1438eec664dd11825233adebb22c5d6ffb32ab Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sun, 23 Feb 2020 23:57:05 -0800 Subject: [PATCH 02/13] update what's new with fix for TMY3 in soiling_hsu * also change 1st line of kimber soiling docstring, to say calculates energy loss, not soiling ratio, since kimber is 1-hsu, they're opposite! --- docs/sphinx/source/whatsnew/v0.7.2.rst | 2 ++ pvlib/losses.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index a4a93ddaa7..6b78a05873 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -39,6 +39,8 @@ Bug fixes :py:func:`~pvlib.iam.martin_ruiz_diffuse`, :py:func:`~pvlib.losses.soiling_hsu`, and various test functions. +* Fix :py:func:`~pvlib.losses.soiling_hsu` TMY3 data not monotonic exception + (:pull:`910`) Documentation ~~~~~~~~~~~~~ diff --git a/pvlib/losses.py b/pvlib/losses.py index 990954f866..43f36bebc9 100644 --- a/pvlib/losses.py +++ b/pvlib/losses.py @@ -131,8 +131,8 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, grace_period=14, max_soiling=0.3, manual_wash_dates=None, initial_soiling=0, rain_accum_period=24, is_tmy=False): """ - Calculate soiling ratio with rainfall data and a daily soiling rate using - the Kimber soiling model. + Calculates fraction of energy lossed due to soiling given rainfall data and + daily loss rate using the Kimber model. Kimber soiling model [1]_ assumes soiling builds up at a daily rate unless the daily rainfall is greater than a threshold. The model also assumes that From 13e75d56a3562b42a4d774c89612dee1e75e051b Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 24 Feb 2020 15:41:58 -0800 Subject: [PATCH 03/13] move fix_tmy_monotonicity to tmy in iotools * and remove it from losses, remove is_tmy args, and remove calls to fix tmy monotonicity * test to make sure if coerced year, last record is Jan 1 of next year * add fix_tmy3_coerce_year_monotonicity to iotools api * revert changes to soiling_hsu, don't use private constants for rain accumulation period or deposition velocity * update kimber soiling example in gallery - to use fix_tmy3_coerce_year_monotonicity - and not use conftest, instead build path - also better comments * remove test for gh889 in test_losses * apply fix to greensboro rain data fixture and remove is_tmy args from kimber tests * add needs_pandas_0_22 to soiling_hsu tests to show that it is required, since requies_scipy masks the requirement --- .../plot_greensboro_kimber_soiling.py | 21 ++++--- pvlib/iotools/__init__.py | 4 +- pvlib/iotools/tmy.py | 42 ++++++++++++++ pvlib/losses.py | 55 +++---------------- pvlib/tests/iotools/test_tmy.py | 34 ++++++++++++ pvlib/tests/test_losses.py | 52 ++++-------------- 6 files changed, 109 insertions(+), 99 deletions(-) diff --git a/docs/examples/plot_greensboro_kimber_soiling.py b/docs/examples/plot_greensboro_kimber_soiling.py index b0ac10cc87..fc450e58d7 100644 --- a/docs/examples/plot_greensboro_kimber_soiling.py +++ b/docs/examples/plot_greensboro_kimber_soiling.py @@ -30,19 +30,24 @@ # step. from datetime import datetime +import pathlib from matplotlib import pyplot as plt -from pvlib.iotools import read_tmy3 +from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity from pvlib.losses import soiling_kimber -from pvlib.tests.conftest import DATA_DIR + +# get full path to the data directory +EXAMPLES_DIR = pathlib.Path(__file__).parent +DATA_DIR = EXAMPLES_DIR.parent.parent / 'pvlib' / 'data' # get TMY3 data with rain -greensboro = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) -# NOTE: can't use Sand Point, AK b/c Lprecipdepth is -9900, ie: missing -greensboro_rain = greensboro[0].Lprecipdepth -# calculate soiling with no wash dates +greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) +# fix TMY3 index to be monotonically increasing +greensboro = fix_tmy3_coerce_year_monotonicity(greensboro) +# get the rain data +greensboro_rain = greensboro.Lprecipdepth +# calculate soiling with no wash dates and cleaning threshold of 25-mm of rain THRESHOLD = 25.0 -soiling_no_wash = soiling_kimber( - greensboro_rain, cleaning_threshold=THRESHOLD, is_tmy=True) +soiling_no_wash = soiling_kimber(greensboro_rain, cleaning_threshold=THRESHOLD) soiling_no_wash.name = 'soiling' # daily rain totals daily_rain = greensboro_rain.iloc[:-1].resample('D').sum() diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index cb1cff48c5..bfa7b427a0 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -1,5 +1,5 @@ -from pvlib.iotools.tmy import read_tmy2 # noqa: F401 -from pvlib.iotools.tmy import read_tmy3 # noqa: F401 +from pvlib.iotools.tmy import ( + read_tmy2, read_tmy3, fix_tmy3_coerce_year_monotonicity) # noqa: F401 from pvlib.iotools.epw import read_epw, parse_epw # noqa: F401 from pvlib.iotools.srml import read_srml # noqa: F401 from pvlib.iotools.srml import read_srml_month_from_solardat # noqa: F401 diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 20792b6ded..f5cb3d58d7 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -7,6 +7,7 @@ import re from urllib.request import urlopen, Request import pandas as pd +import numpy as np def read_tmy3(filename=None, coerce_year=None, recolumn=True): @@ -526,3 +527,44 @@ def _read_tmy2(string, columns, hdr_columns, fname): columns=columns.split(',')).tz_localize(int(meta['TZ'] * 3600)) return data, meta + + +def fix_tmy3_coerce_year_monotonicity(tmy3): + """ + Fix TMY3 coerced to a single year to be monotonically increasing by + changing the last record to be in January 1st of the next year. + + Paramaters + ---------- + tmy3 : pandas.DataFrame + TMY3 data frame from :func:`pvlib.iotools.read_tmy3` with year coearced + + Returns + ------- + pandas.DataFrame + Copy of the TMY3 data frame with monotonically increasing index + """ + # NOTE: pandas index is immutable, therefore it's not possible to change a + # single item in the index, so the entire index must be replaced + + # get tmy3 timezone, index values as np.datetime64[ns], and index frequency + # as np.timedelta64[ns] + # NOTE: numpy converts index values to UTC + index_tz = tmy3.index.tz + index_values = tmy3.index.values + timestep_interval = index_values[1] - index_values[0] + + # fix index to be monotonically increasing by rolling indices 1 interval, + # then adding 1 interval to all indices + index_values = np.roll(index_values, 1) + timestep_interval + + # create new datetime index and convert it to the original timezone + new_index = pd.DatetimeIndex(index_values, tz='UTC').tz_convert(index_tz) + + # copy the original TMY3 so it doesn't change, then replace the index of + # the copy with the new fixed monotonically increasing index + new_tmy3 = tmy3.copy() + new_tmy3.index = new_index + + # fixed TMY3 with monotonically increasing index + return new_tmy3 diff --git a/pvlib/losses.py b/pvlib/losses.py index 43f36bebc9..7093afc938 100644 --- a/pvlib/losses.py +++ b/pvlib/losses.py @@ -9,13 +9,9 @@ import pandas as pd from pvlib.tools import cosd -_RAIN_ACC_PERIOD = pd.Timedelta('1h') -_DEPO_VELOCITY = {'2_5': 0.004, '10': 0.0009} - def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, - depo_veloc=None, rain_accum_period=_RAIN_ACC_PERIOD, - is_tmy=False): + depo_veloc=None, rain_accum_period=pd.Timedelta('1h')): """ Calculates soiling ratio given particulate and rain data using the model from Humboldt State University (HSU). @@ -53,9 +49,6 @@ def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, It is recommended that `rain_accum_period` be between 1 hour and 24 hours. - is_tmy : bool, default False - Fix last timestep in TMY so that it is monotonically increasing. - Returns ------- soiling_ratio : Series @@ -77,19 +70,7 @@ def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, # never use mutable input arguments if depo_veloc is None: - depo_veloc = _DEPO_VELOCITY - - # if TMY fix to be monotonically increasing by rolling index by 1 interval - # and then adding 1 interval, while the values stay the same - if is_tmy: - # get rainfall timezone, timestep as timedelta64 - rain_tz = rainfall.index.tz - rain_index = rainfall.index.values - timestep_interval = (rain_index[1] - rain_index[0]) - rain_vals = rainfall.values - rain_name = rainfall.name - rainfall = _fix_tmy_monotonicity( - rain_index, rain_vals, timestep_interval, rain_tz, rain_name) + depo_veloc = {'2_5': 0.004, '10': 0.0009} # accumulate rainfall into periods for comparison with threshold accum_rain = rainfall.rolling(rain_accum_period, closed='right').sum() @@ -115,23 +96,11 @@ def soiling_hsu(rainfall, cleaning_threshold, tilt, pm2_5, pm10, return soiling_ratio -def _fix_tmy_monotonicity(rain_index, rain_vals, timestep_interval, rain_tz, - rain_name): - # fix TMY to be monotonically increasing by rolling index by 1 interval - # and then adding 1 interval, while the values stay the same - rain_index = np.roll(rain_index, 1) + timestep_interval - # NOTE: numpy datetim64[ns] has no timezone - # convert to datetimeindex at UTC and convert to original timezone - rain_index = pd.DatetimeIndex(rain_index, tz='UTC').tz_convert(rain_tz) - # fixed rainfall timeseries with monotonically increasing index - return pd.Series(rain_vals, index=rain_index, name=rain_name) - - def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, grace_period=14, max_soiling=0.3, manual_wash_dates=None, - initial_soiling=0, rain_accum_period=24, is_tmy=False): + initial_soiling=0, rain_accum_period=24): """ - Calculates fraction of energy lossed due to soiling given rainfall data and + Calculates fraction of energy lost due to soiling given rainfall data and daily loss rate using the Kimber model. Kimber soiling model [1]_ assumes soiling builds up at a daily rate unless @@ -165,8 +134,6 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, rain_accum_period : int, default 24 Period for accumulating rainfall to check against `cleaning_threshold`. The Kimber model defines this period as one day. [hours] - is_tmy : bool, default False - Fix last timestep in TMY so that it is monotonically increasing. Returns ------- @@ -204,20 +171,12 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, # convert grace_period to timedelta grace_period = datetime.timedelta(days=grace_period) - # get rainfall timezone, timestep as timedelta64, and timestep as day-frac - rain_tz = rainfall.index.tz + # get indices as numpy datetime64, calculate timestep as numpy timedelta64, + # and convert timestep to fraction of days rain_index_vals = rainfall.index.values timestep_interval = (rain_index_vals[1] - rain_index_vals[0]) day_fraction = timestep_interval / np.timedelta64(24, 'h') - # if TMY fix to be monotonically increasing by rolling index by 1 interval - # and then adding 1 interval, while the values stay the same - if is_tmy: - rain_vals = rainfall.values - rain_name = rainfall.name - rainfall = _fix_tmy_monotonicity( - rain_index_vals, rain_vals, timestep_interval, rain_tz, rain_name) - # accumulate rainfall accumulated_rainfall = rainfall.rolling( rain_accum_period, closed='right').sum() @@ -241,6 +200,8 @@ def soiling_kimber(rainfall, cleaning_threshold=6, soiling_loss_rate=0.0015, # manual wash dates if manual_wash_dates is not None: + rain_tz = rainfall.index.tz + # convert manual wash dates to datetime index in the timezone of rain manual_wash_dates = pd.DatetimeIndex(manual_wash_dates, tz=rain_tz) cleaning[manual_wash_dates] = soiling[manual_wash_dates] diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index 80deb469eb..b9b59445e8 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -1,8 +1,11 @@ +import datetime from pandas.util.testing import network import numpy as np import pandas as pd import pytest +import pytz from pvlib.iotools import tmy +from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity from conftest import DATA_DIR # test the API works @@ -77,3 +80,34 @@ def test_gh865_read_tmy3_feb_leapyear_hr24(): # hour so check that the 1st hour is 1AM and the last hour is midnite assert data.index[0].hour == 1 assert data.index[-1].hour == 0 + + +def test_fix_tmy_coerce_year_monotonicity(): + # greensboro timezone is UTC-5 or Eastern time + gmt_5 = pytz.timezone('Etc/GMT+5') + + # tmy3 coerced to year is not monotonically increasing + greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) + + # check first hour was coerced to 1990 + firsthour = gmt_5.localize(datetime.datetime(1990, 1, 1, 1, 0, 0)) + assert firsthour == greensboro.index[0] + + # check last hour was coerced to 1990 + lasthour = gmt_5.localize(datetime.datetime(1990, 12, 31, 23, 0, 0)) + assert lasthour == greensboro.index[-2] + + # check last day, was coerced to 1990 + lastday1990 = gmt_5.localize(datetime.datetime(1990, 1, 1, 0, 0, 0)) + assert lastday1990 == greensboro.index[-1] + + # fix the index to be monotonically increasing + greensboro = fix_tmy3_coerce_year_monotonicity(greensboro) + + # check first and last hours are still 1990 + assert firsthour == greensboro.index[0] + assert lasthour == greensboro.index[-2] + + # check last day, should be 1991 now + lastday1991 = lastday1990.replace(year=1991) + assert lastday1991 == greensboro.index[-1] diff --git a/pvlib/tests/test_losses.py b/pvlib/tests/test_losses.py index 75623202b9..89290be9fe 100644 --- a/pvlib/tests/test_losses.py +++ b/pvlib/tests/test_losses.py @@ -6,7 +6,7 @@ import pandas as pd from pandas.util.testing import assert_series_equal from pvlib.losses import soiling_hsu, soiling_kimber -from pvlib.iotools import read_tmy3 +from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity from conftest import ( requires_scipy, needs_pandas_0_22, DATA_DIR) import pytest @@ -65,6 +65,7 @@ def rainfall_input(): @requires_scipy +@needs_pandas_0_22 def test_soiling_hsu_no_cleaning(rainfall_input, expected_output): """Test Soiling HSU function""" @@ -82,6 +83,7 @@ def test_soiling_hsu_no_cleaning(rainfall_input, expected_output): @requires_scipy +@needs_pandas_0_22 def test_soiling_hsu(rainfall_input, expected_output_2): """Test Soiling HSU function""" @@ -103,9 +105,9 @@ def test_soiling_hsu(rainfall_input, expected_output_2): @pytest.fixture def greensboro_rain(): # get TMY3 data with rain - greensboro = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) - # NOTE: can't use Sand Point, AK b/c Lprecipdepth is -9900, ie: missing - return greensboro[0].Lprecipdepth + greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) + greensboro = fix_tmy3_coerce_year_monotonicity(greensboro) + return greensboro.Lprecipdepth @pytest.fixture @@ -122,7 +124,7 @@ def test_kimber_soiling_nowash(greensboro_rain, # Greensboro typical expected annual rainfall is 8345mm assert greensboro_rain.sum() == 8345 # calculate soiling with no wash dates - soiling_nowash = soiling_kimber(greensboro_rain, is_tmy=True) + soiling_nowash = soiling_kimber(greensboro_rain) # test no washes assert np.allclose( soiling_nowash.values, @@ -144,7 +146,7 @@ def test_kimber_soiling_manwash(greensboro_rain, manwash = [datetime.date(1990, 2, 15), ] # calculate soiling with manual wash soiling_manwash = soiling_kimber( - greensboro_rain, manual_wash_dates=manwash, is_tmy=True) + greensboro_rain, manual_wash_dates=manwash) # test manual wash assert np.allclose( soiling_manwash.values, @@ -169,7 +171,7 @@ def test_kimber_soiling_norain(greensboro_rain, # a year with no rain norain = pd.Series(0, index=greensboro_rain.index) # calculate soiling with no rain - soiling_norain = soiling_kimber(norain, is_tmy=True) + soiling_norain = soiling_kimber(norain) # test no rain, soiling reaches maximum assert np.allclose(soiling_norain.values, expected_kimber_soiling_norain) @@ -192,41 +194,7 @@ def test_kimber_soiling_initial_soil(greensboro_rain, # a year with no rain norain = pd.Series(0, index=greensboro_rain.index) # calculate soiling with no rain - soiling_norain = soiling_kimber(norain, initial_soiling=0.1, is_tmy=True) + soiling_norain = soiling_kimber(norain, initial_soiling=0.1) # test no rain, soiling reaches maximum assert np.allclose( soiling_norain.values, expected_kimber_soiling_initial_soil) - - -@pytest.fixture -def expected_greensboro_hsu_soil(): - return np.array([ - 0.99927224, 0.99869067, 0.99815393, 0.99764437, 0.99715412, - 0.99667873, 0.99621536, 0.99576203, 0.99531731, 0.99488010, - 0.99444954, 0.99402494, 0.99360572, 0.99319142, 1.00000000, - 1.00000000, 0.99927224, 0.99869067, 0.99815393, 0.99764437, - 0.99715412, 1.00000000, 0.99927224, 0.99869067]) - - -@requires_scipy -def test_gh889_soiing_hsu_tmy_not_monotonic(expected_greensboro_hsu_soil): - """doesn't raise value error""" - greensboro = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) - greensboro_rain = greensboro[0].Lprecipdepth - soiling_ratio = soiling_hsu( - greensboro_rain, cleaning_threshold=10.0, tilt=0.0, pm2_5=1.0, - pm10=2.0, is_tmy=True) - # check first day of soiling ratio, actually (1 - transmission loss) - # greensboro rains hours 3pm, 4pm, and 10pm, so expect soiling ratio of one - assert np.allclose(expected_greensboro_hsu_soil, soiling_ratio.values[:24]) - # greensboro timezone is UTC-5 or Eastern time - gmt_5 = pytz.timezone('Etc/GMT+5') - # check last day, should be 1991 now - lastday = datetime.datetime(1991, 1, 1, 0, 0, 0) - assert gmt_5.localize(lastday) == soiling_ratio.index[-1] - # check last hour is still 1990 - lasthour = datetime.datetime(1990, 12, 31, 23, 0, 0) - assert gmt_5.localize(lasthour) == soiling_ratio.index[-2] - # check first hour is still 1990 - firsthour = datetime.datetime(1990, 1, 1, 1, 0, 0) - assert gmt_5.localize(firsthour) == soiling_ratio.index[0] From ecbaf1f99208832dffc4aad31edc42a7f9b88de4 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 24 Feb 2020 16:19:04 -0800 Subject: [PATCH 04/13] fix path to data in kimber example * add test for HSU soiling defaults for coverage * fix iotools api stickler warning * remove unused pytz import in test_losses --- .../plot_greensboro_kimber_soiling.py | 4 +-- pvlib/iotools/__init__.py | 4 +-- pvlib/tests/test_losses.py | 26 +++++++++++++++++-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/docs/examples/plot_greensboro_kimber_soiling.py b/docs/examples/plot_greensboro_kimber_soiling.py index fc450e58d7..84a528db28 100644 --- a/docs/examples/plot_greensboro_kimber_soiling.py +++ b/docs/examples/plot_greensboro_kimber_soiling.py @@ -34,10 +34,10 @@ from matplotlib import pyplot as plt from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity from pvlib.losses import soiling_kimber +import pvlib # get full path to the data directory -EXAMPLES_DIR = pathlib.Path(__file__).parent -DATA_DIR = EXAMPLES_DIR.parent.parent / 'pvlib' / 'data' +DATA_DIR = pathlib.Path(pvlib.__file__).parent / 'data' # get TMY3 data with rain greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index bfa7b427a0..01b2d6b60c 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -1,5 +1,5 @@ -from pvlib.iotools.tmy import ( - read_tmy2, read_tmy3, fix_tmy3_coerce_year_monotonicity) # noqa: F401 +from pvlib.iotools.tmy import read_tmy2, read_tmy3 # noqa: F401 +from pvlib.iotools.tmy import fix_tmy3_coerce_year_monotonicity # noqa: F401 from pvlib.iotools.epw import read_epw, parse_epw # noqa: F401 from pvlib.iotools.srml import read_srml # noqa: F401 from pvlib.iotools.srml import read_srml_month_from_solardat # noqa: F401 diff --git a/pvlib/tests/test_losses.py b/pvlib/tests/test_losses.py index 89290be9fe..3937342371 100644 --- a/pvlib/tests/test_losses.py +++ b/pvlib/tests/test_losses.py @@ -10,7 +10,6 @@ from conftest import ( requires_scipy, needs_pandas_0_22, DATA_DIR) import pytest -import pytz @pytest.fixture @@ -33,6 +32,16 @@ def expected_output(): return expected_no_cleaning +@pytest.fixture +def expected_output_1(): + return np.array([ + 0.99927224, 0.99869067, 0.99815393, 0.99764437, 1.0, + 0.99927224, 0.99869067, 0.99815393, 1.0, 1.0, + 0.99927224, 0.99869067, 0.99815393, 0.99764437, 0.99715412, + 0.99667873, 0.99621536, 0.99576203, 0.99531731, 0.9948801, + 0.99444954, 0.99402494, 0.99360572, 0.99319142]) + + @pytest.fixture def expected_output_2(expected_output): # Sample output (calculated manually) @@ -85,7 +94,7 @@ def test_soiling_hsu_no_cleaning(rainfall_input, expected_output): @requires_scipy @needs_pandas_0_22 def test_soiling_hsu(rainfall_input, expected_output_2): - """Test Soiling HSU function""" + """Test Soiling HSU function with cleanings""" rainfall = rainfall_input pm2_5 = 1.0 @@ -102,6 +111,19 @@ def test_soiling_hsu(rainfall_input, expected_output_2): assert_series_equal(result, expected) +@requires_scipy +@needs_pandas_0_22 +def test_soiling_hsu_defaults(rainfall_input, expected_output_1): + """ + Test Soiling HSU function with default deposition velocity and default rain + accumulation period. + """ + result = soiling_hsu( + rainfall=rainfall_input, cleaning_threshold=0.5, tilt=0.0, pm2_5=1.0, + pm10=2.0) + assert np.allclose(result.values, expected_output_1) + + @pytest.fixture def greensboro_rain(): # get TMY3 data with rain From cec88e8776e8a071c5b97bf24b404d5bb1cbf006 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 24 Feb 2020 16:45:12 -0800 Subject: [PATCH 05/13] update what's new and api.rst with fix_tmy3_coerce_year_monotonicity --- docs/sphinx/source/api.rst | 1 + docs/sphinx/source/whatsnew/v0.7.2.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 8237e3908a..06df67823b 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -369,6 +369,7 @@ relevant to solar energy modeling. iotools.read_tmy2 iotools.read_tmy3 + iotools.fix_tmy3_coerce_year_monotonicity iotools.read_epw iotools.parse_epw iotools.read_srml diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index 6b78a05873..011c9e77d5 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -14,6 +14,8 @@ Enhancements the original ``Date (MM/DD/YYYY)`` and ``Time (HH:MM)`` columns that the indices were parsed from (:pull:`866`) * Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber` (:pull:`860`) +* Add :py:func:`~pvlib.iotools.fix_tmy3_coerce_year_monotonicity` to fix TMY3 + data coerced to a single year to be monotonically increasing (:pull:`910`) Bug fixes ~~~~~~~~~ From 64ec94d811a6608842151b6b519f845f7ea9613e Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 24 Feb 2020 16:58:42 -0800 Subject: [PATCH 06/13] parameters spelling typo in fix_tmy3_monotonicity docstring * also move reference in kimber soiling example to comment in first code cell --- .../examples/plot_greensboro_kimber_soiling.py | 18 +++++++++--------- pvlib/iotools/tmy.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/examples/plot_greensboro_kimber_soiling.py b/docs/examples/plot_greensboro_kimber_soiling.py index 84a528db28..9eec8f3d9c 100644 --- a/docs/examples/plot_greensboro_kimber_soiling.py +++ b/docs/examples/plot_greensboro_kimber_soiling.py @@ -2,20 +2,20 @@ Kimber Soiling Model ==================== -Examples of soiling using the Kimber model [1]_. - -References ----------- -.. [1] "The Effect of Soiling on Large Grid-Connected Photovoltaic Systems - in California and the Southwest Region of the United States," Adrianne - Kimber, et al., IEEE 4th World Conference on Photovoltaic Energy - Conference, 2006, :doi:`10.1109/WCPEC.2006.279690` +Examples of soiling using the Kimber model. """ # %% -# This example shows basic usage of pvlib's Kimber Soiling model with +# This example shows basic usage of pvlib's Kimber Soiling model [1]_ with # :py:meth:`pvlib.losses.soiling_kimber`. # +# References +# ---------- +# .. [1] "The Effect of Soiling on Large Grid-Connected Photovoltaic Systems +# in California and the Southwest Region of the United States," Adrianne +# Kimber, et al., IEEE 4th World Conference on Photovoltaic Energy +# Conference, 2006, :doi:`10.1109/WCPEC.2006.279690` +# # The Kimber Soiling model assumes that soiling builds up at a constant rate # until cleaned either manually or by rain. The rain must reach a threshold to # clean the panels. When rains exceeds the threshold, it's assumed the earth is diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index f5cb3d58d7..4b7ea8f103 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -534,7 +534,7 @@ def fix_tmy3_coerce_year_monotonicity(tmy3): Fix TMY3 coerced to a single year to be monotonically increasing by changing the last record to be in January 1st of the next year. - Paramaters + Parameters ---------- tmy3 : pandas.DataFrame TMY3 data frame from :func:`pvlib.iotools.read_tmy3` with year coearced From f025854a93791dd79fa6e8c71e114e9160354c2e Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Tue, 25 Feb 2020 15:57:15 -0800 Subject: [PATCH 07/13] add monotonic_index kwarg to read_tmy3 * change name to tmy3_monotonic_index * set monotonic_index default to false but warn that this will change to True in v0.8 * improve tmy3_monotonic_index docstring with reviewer comments, and add note to call only after read_tmy3 with coerce year, to use monotonic_index=True to combine calls, and that there's no validation that input TMY3 has been coerced or if it has already been fixed * also if only used for TMY3 no need to calculate timestep interval, it's always 1-hour * that's all folks --- .../plot_greensboro_kimber_soiling.py | 4 +- docs/sphinx/source/api.rst | 2 +- docs/sphinx/source/whatsnew/v0.7.2.rst | 2 +- pvlib/iotools/__init__.py | 2 +- pvlib/iotools/tmy.py | 51 ++++++++++++++++--- pvlib/tests/iotools/test_tmy.py | 4 +- pvlib/tests/test_losses.py | 4 +- 7 files changed, 52 insertions(+), 17 deletions(-) diff --git a/docs/examples/plot_greensboro_kimber_soiling.py b/docs/examples/plot_greensboro_kimber_soiling.py index 9eec8f3d9c..e7a3a1feb0 100644 --- a/docs/examples/plot_greensboro_kimber_soiling.py +++ b/docs/examples/plot_greensboro_kimber_soiling.py @@ -32,7 +32,7 @@ from datetime import datetime import pathlib from matplotlib import pyplot as plt -from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity +from pvlib.iotools import read_tmy3, tmy3_monotonic_index from pvlib.losses import soiling_kimber import pvlib @@ -42,7 +42,7 @@ # get TMY3 data with rain greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) # fix TMY3 index to be monotonically increasing -greensboro = fix_tmy3_coerce_year_monotonicity(greensboro) +greensboro = tmy3_monotonic_index(greensboro) # get the rain data greensboro_rain = greensboro.Lprecipdepth # calculate soiling with no wash dates and cleaning threshold of 25-mm of rain diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 06df67823b..6340e2431b 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -369,7 +369,7 @@ relevant to solar energy modeling. iotools.read_tmy2 iotools.read_tmy3 - iotools.fix_tmy3_coerce_year_monotonicity + iotools.tmy3_monotonic_index iotools.read_epw iotools.parse_epw iotools.read_srml diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index 011c9e77d5..07d0950030 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -14,7 +14,7 @@ Enhancements the original ``Date (MM/DD/YYYY)`` and ``Time (HH:MM)`` columns that the indices were parsed from (:pull:`866`) * Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber` (:pull:`860`) -* Add :py:func:`~pvlib.iotools.fix_tmy3_coerce_year_monotonicity` to fix TMY3 +* Add :py:func:`~pvlib.iotools.tmy3_monotonic_index` to fix TMY3 data coerced to a single year to be monotonically increasing (:pull:`910`) Bug fixes diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 01b2d6b60c..6bd7ab8481 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -1,5 +1,5 @@ from pvlib.iotools.tmy import read_tmy2, read_tmy3 # noqa: F401 -from pvlib.iotools.tmy import fix_tmy3_coerce_year_monotonicity # noqa: F401 +from pvlib.iotools.tmy import tmy3_monotonic_index # noqa: F401 from pvlib.iotools.epw import read_epw, parse_epw # noqa: F401 from pvlib.iotools.srml import read_srml # noqa: F401 from pvlib.iotools.srml import read_srml_month_from_solardat # noqa: F401 diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 4b7ea8f103..ecdeb5e8d1 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -10,7 +10,8 @@ import numpy as np -def read_tmy3(filename=None, coerce_year=None, recolumn=True): +def read_tmy3(filename=None, coerce_year=None, recolumn=True, + monotonic_index=False): ''' Read a TMY3 file in to a pandas dataframe. @@ -32,9 +33,15 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): If supplied, the year of the data will be set to this value. recolumn : bool, default True - If True, apply standard names to TMY3 columns. Typically this + If ``True``, apply standard names to TMY3 columns. Typically this results in stripping the units from the column name. + monotonic_index : bool, default False + If ``True`` and `coerce_year` is not ``None`` then the date-time index + will be adjusted to be monotonically increasing by changing the last + date-time index to be midnight on January 1st of the next year. If + `coerce_year` is ``None``, does nothing. + Returns ------- Tuple of the form (data, metadata). @@ -220,6 +227,15 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): data.index = data_ymd + pd.to_timedelta(shifted_hour, unit='h') if coerce_year is not None: data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) + if monotonic_index: + data = tmy3_monotonic_index(data) + else: + import warnings + msg = ( + 'date-time index will be changed in v0.8 from to be' + ' monotonically increasing, set monotonic_index=True to' + ' silence this warning') + warnings.warn(msg, RuntimeWarning) if recolumn: data = _recolumn(data) # rename to standard column names @@ -529,20 +545,40 @@ def _read_tmy2(string, columns, hdr_columns, fname): return data, meta -def fix_tmy3_coerce_year_monotonicity(tmy3): +def tmy3_monotonic_index(tmy3): """ - Fix TMY3 coerced to a single year to be monotonically increasing by - changing the last record to be in January 1st of the next year. + Fix the date-time index of TMY3 coerced to a single year to be + monotonically increasing by changing the last date-time index to be + midnight on January 1st of the next year. Parameters ---------- tmy3 : pandas.DataFrame - TMY3 data frame from :func:`pvlib.iotools.read_tmy3` with year coearced + TMY3 data frame from :func:`pvlib.iotools.read_tmy3` with coerced year Returns ------- pandas.DataFrame Copy of the TMY3 data frame with monotonically increasing index + + Notes + ----- + Use this function after calling :func:`~pvlib.iotools.read_tmy3` with + ``coerce_year`` set as desired. + + .. warning:: This function will only work on TMY3 with date-time index + coerced to a single year. There is no validation to check if input TMY3 + data frame has been coerced to a single year or if the date-time index + has already been fixed to be monotonically increasing. + + In :func:`~pvlib.iotools.read_tmy3`, set ``monotonic_index=True`` to read + TMY3, coerce the year, and fix the date-time index to be monotonically + increasing without calling this function afterward. + + See Also + -------- + pvlib.iotools.read_tmy3 + """ # NOTE: pandas index is immutable, therefore it's not possible to change a # single item in the index, so the entire index must be replaced @@ -552,11 +588,10 @@ def fix_tmy3_coerce_year_monotonicity(tmy3): # NOTE: numpy converts index values to UTC index_tz = tmy3.index.tz index_values = tmy3.index.values - timestep_interval = index_values[1] - index_values[0] # fix index to be monotonically increasing by rolling indices 1 interval, # then adding 1 interval to all indices - index_values = np.roll(index_values, 1) + timestep_interval + index_values = np.roll(index_values, 1) + np.timedelta64(1, 'h') # create new datetime index and convert it to the original timezone new_index = pd.DatetimeIndex(index_values, tz='UTC').tz_convert(index_tz) diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index b9b59445e8..82ade9139e 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -5,7 +5,7 @@ import pytest import pytz from pvlib.iotools import tmy -from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity +from pvlib.iotools import read_tmy3, tmy3_monotonic_index from conftest import DATA_DIR # test the API works @@ -102,7 +102,7 @@ def test_fix_tmy_coerce_year_monotonicity(): assert lastday1990 == greensboro.index[-1] # fix the index to be monotonically increasing - greensboro = fix_tmy3_coerce_year_monotonicity(greensboro) + greensboro = tmy3_monotonic_index(greensboro) # check first and last hours are still 1990 assert firsthour == greensboro.index[0] diff --git a/pvlib/tests/test_losses.py b/pvlib/tests/test_losses.py index 3937342371..deb26af0a5 100644 --- a/pvlib/tests/test_losses.py +++ b/pvlib/tests/test_losses.py @@ -6,7 +6,7 @@ import pandas as pd from pandas.util.testing import assert_series_equal from pvlib.losses import soiling_hsu, soiling_kimber -from pvlib.iotools import read_tmy3, fix_tmy3_coerce_year_monotonicity +from pvlib.iotools import read_tmy3, tmy3_monotonic_index from conftest import ( requires_scipy, needs_pandas_0_22, DATA_DIR) import pytest @@ -128,7 +128,7 @@ def test_soiling_hsu_defaults(rainfall_input, expected_output_1): def greensboro_rain(): # get TMY3 data with rain greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) - greensboro = fix_tmy3_coerce_year_monotonicity(greensboro) + greensboro = tmy3_monotonic_index(greensboro) return greensboro.Lprecipdepth From dc7f21768e4719137da2d02395fa8e29e4aa2b8a Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Tue, 25 Feb 2020 16:29:51 -0800 Subject: [PATCH 08/13] update what's new with monotonic_index kwarg --- docs/sphinx/source/whatsnew/v0.7.2.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index 07d0950030..ee0fbaa51b 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -16,6 +16,9 @@ Enhancements * Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber` (:pull:`860`) * Add :py:func:`~pvlib.iotools.tmy3_monotonic_index` to fix TMY3 data coerced to a single year to be monotonically increasing (:pull:`910`) +* Add ``monotonic_index`` kwarg to :py:func:`~pvlib.iotools.read_tmy3` to fix + TMY3 data coerced to a single year to be monotonically increasing, default is + ``False`` and warns that it will be changed to ``True`` in v0.8 (:pull:`910`) Bug fixes ~~~~~~~~~ From 28fbfdc24134bfaa92ee18290e6268c0e1c322da Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Wed, 26 Feb 2020 09:35:15 -0800 Subject: [PATCH 09/13] Update pvlib/tests/iotools/test_tmy.py * change test name to `test_tmy3_monotonic_index` Co-Authored-By: Cliff Hansen --- pvlib/tests/iotools/test_tmy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index 82ade9139e..84ea6f49e6 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -82,7 +82,7 @@ def test_gh865_read_tmy3_feb_leapyear_hr24(): assert data.index[-1].hour == 0 -def test_fix_tmy_coerce_year_monotonicity(): +def test_tmy3_monotonic_index(): # greensboro timezone is UTC-5 or Eastern time gmt_5 = pytz.timezone('Etc/GMT+5') From e089815ea0ccf06af4c130a873218d6abecbcd9f Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 28 Feb 2020 21:03:11 -0800 Subject: [PATCH 10/13] fix bug in read_tmy3 when year coerced - so the indices are monotonically increasing - add note to API changes and update bug fix in what's new - update docstring in read_tmy3 * explain in coerce_year parmaeter that all of the indices except the last one will have this, value, but the last index will be the value+1 - update the warnings to explain that if the year is coerced that the last index will be the next year - do coerce year on date_ymd which is a Series of Timestamps, not an index so mutable (can assign by integer index) which makes it trivial to change the last index to be the next year - revert most of the changes from the previous commits in this PR * remove `tmy3_monotonic_index` from `api.rst`, `iotools/__init__.py`, and from `tmy.py` * remove `monotonic_index` kwarg from read_tmy3 * remove use of `tmy3_monotonic_index` from sphinx gallery example, test_losses, and test_tmy * remove test for tmy3_monotonic_index and the monotonic_index kwarg * fix test_tmy test to treat last index differently, except for the leap year test which now has a constant diff, yay! --- .../plot_greensboro_kimber_soiling.py | 4 +- docs/sphinx/source/api.rst | 1 - docs/sphinx/source/whatsnew/v0.7.2.rst | 12 +- pvlib/iotools/__init__.py | 1 - pvlib/iotools/tmy.py | 103 +++--------------- pvlib/tests/iotools/test_tmy.py | 47 ++------ pvlib/tests/test_losses.py | 3 +- 7 files changed, 31 insertions(+), 140 deletions(-) diff --git a/docs/examples/plot_greensboro_kimber_soiling.py b/docs/examples/plot_greensboro_kimber_soiling.py index e7a3a1feb0..aa000b6c22 100644 --- a/docs/examples/plot_greensboro_kimber_soiling.py +++ b/docs/examples/plot_greensboro_kimber_soiling.py @@ -32,7 +32,7 @@ from datetime import datetime import pathlib from matplotlib import pyplot as plt -from pvlib.iotools import read_tmy3, tmy3_monotonic_index +from pvlib.iotools import read_tmy3 from pvlib.losses import soiling_kimber import pvlib @@ -41,8 +41,6 @@ # get TMY3 data with rain greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) -# fix TMY3 index to be monotonically increasing -greensboro = tmy3_monotonic_index(greensboro) # get the rain data greensboro_rain = greensboro.Lprecipdepth # calculate soiling with no wash dates and cleaning threshold of 25-mm of rain diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 6340e2431b..8237e3908a 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -369,7 +369,6 @@ relevant to solar energy modeling. iotools.read_tmy2 iotools.read_tmy3 - iotools.tmy3_monotonic_index iotools.read_epw iotools.parse_epw iotools.read_srml diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index ee0fbaa51b..b3e6dddf55 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -7,6 +7,9 @@ API Changes ~~~~~~~~~~~ * :py:class:`pvlib.forecast.ForecastModel` now requires ``start`` and ``end`` arguments to be tz-localized. (:issue:`877`, :pull:`879`) +* :py:func:`pvlib.iotools.read_tmy3` when coerced to a single year now returns + indices that are monotonically increasing, for example the last index will be + midnight of the *next* year. (:pull:`910`) Enhancements ~~~~~~~~~~~~ @@ -14,11 +17,6 @@ Enhancements the original ``Date (MM/DD/YYYY)`` and ``Time (HH:MM)`` columns that the indices were parsed from (:pull:`866`) * Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber` (:pull:`860`) -* Add :py:func:`~pvlib.iotools.tmy3_monotonic_index` to fix TMY3 - data coerced to a single year to be monotonically increasing (:pull:`910`) -* Add ``monotonic_index`` kwarg to :py:func:`~pvlib.iotools.read_tmy3` to fix - TMY3 data coerced to a single year to be monotonically increasing, default is - ``False`` and warns that it will be changed to ``True`` in v0.8 (:pull:`910`) Bug fixes ~~~~~~~~~ @@ -44,8 +42,8 @@ Bug fixes :py:func:`~pvlib.iam.martin_ruiz_diffuse`, :py:func:`~pvlib.losses.soiling_hsu`, and various test functions. -* Fix :py:func:`~pvlib.losses.soiling_hsu` TMY3 data not monotonic exception - (:pull:`910`) +* Fix :py:func:`~pvlib.iotools.read_tmy3` so that when coerced to a single year + the TMY3 data will be monotonically increasing (:pull:`910`) Documentation ~~~~~~~~~~~~~ diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 6bd7ab8481..d25616bfd3 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -1,5 +1,4 @@ from pvlib.iotools.tmy import read_tmy2, read_tmy3 # noqa: F401 -from pvlib.iotools.tmy import tmy3_monotonic_index # noqa: F401 from pvlib.iotools.epw import read_epw, parse_epw # noqa: F401 from pvlib.iotools.srml import read_srml # noqa: F401 from pvlib.iotools.srml import read_srml_month_from_solardat # noqa: F401 diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index ecdeb5e8d1..f76940a2e6 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -10,8 +10,7 @@ import numpy as np -def read_tmy3(filename=None, coerce_year=None, recolumn=True, - monotonic_index=False): +def read_tmy3(filename=None, coerce_year=None, recolumn=True): ''' Read a TMY3 file in to a pandas dataframe. @@ -30,18 +29,14 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True, a relative file path, absolute file path, or url. coerce_year : None or int, default None - If supplied, the year of the data will be set to this value. + If supplied, the year of the data will be set to this value, except for + the last index which will be set to the *next* year so that the indices + increase monotonically. recolumn : bool, default True If ``True``, apply standard names to TMY3 columns. Typically this results in stripping the units from the column name. - monotonic_index : bool, default False - If ``True`` and `coerce_year` is not ``None`` then the date-time index - will be adjusted to be monotonically increasing by changing the last - date-time index to be midnight on January 1st of the next year. If - `coerce_year` is ``None``, does nothing. - Returns ------- Tuple of the form (data, metadata). @@ -57,7 +52,6 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True, Notes ----- - The returned structures have the following fields. =============== ====== =================== @@ -147,15 +141,16 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True, TMYData.PresWthUncertainty Present weather code uncertainty, see [2]_. ============================= ====================================================================================================================================================== - .. warning:: TMY3 irradiance data corresponds to the previous hour, so the - first hour is 1AM, corresponding to the net irradiance from midnite to - 1AM, and the last hour is midnite of the *next* year, unless the year - has been coerced. EG: if TMY3 was 1988-12-31 24:00:00 this becomes - 1989-01-01 00:00:00 + .. warning:: TMY3 irradiance data corresponds to the *previous* hour, so + the first index is 1AM, corresponding to the irradiance from midnight + to 1AM, and the last index is midnight of the *next* year. For example, + if the last index in the TMY3 file was 1988-12-31 24:00:00 this becomes + 1989-01-01 00:00:00 after calling :func:`~pvlib.iotools.read_tmy3`. .. warning:: When coercing the year, the last index in the dataframe will - be the first hour of the same year, EG: if TMY3 was 1988-12-31 24:00:00 - and year is coerced to 1990 this becomes 1990-01-01 + become midnight of the *next* year. For example, if the last index in + the TMY3 was 1988-12-31 24:00:00, and year is coerced to 1990 then this + becomes 1991-01-01 00:00:00. References ---------- @@ -222,20 +217,12 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True, data_ymd[leapday] += datetime.timedelta(days=1) # shifted_hour is a pd.Series, so use pd.to_timedelta to get a pd.Series of # timedeltas + if coerce_year is not None: + data_ymd = data_ymd.map(lambda dt: dt.replace(year=coerce_year)) + data_ymd.iloc[-1] = data_ymd.iloc[-1].replace(year=coerce_year+1) # NOTE: as of pvlib-0.6.3, min req is pandas-0.18.1, so pd.to_timedelta # unit must be in (D,h,m,s,ms,us,ns), but pandas>=0.24 allows unit='hour' data.index = data_ymd + pd.to_timedelta(shifted_hour, unit='h') - if coerce_year is not None: - data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) - if monotonic_index: - data = tmy3_monotonic_index(data) - else: - import warnings - msg = ( - 'date-time index will be changed in v0.8 from to be' - ' monotonically increasing, set monotonic_index=True to' - ' silence this warning') - warnings.warn(msg, RuntimeWarning) if recolumn: data = _recolumn(data) # rename to standard column names @@ -543,63 +530,3 @@ def _read_tmy2(string, columns, hdr_columns, fname): columns=columns.split(',')).tz_localize(int(meta['TZ'] * 3600)) return data, meta - - -def tmy3_monotonic_index(tmy3): - """ - Fix the date-time index of TMY3 coerced to a single year to be - monotonically increasing by changing the last date-time index to be - midnight on January 1st of the next year. - - Parameters - ---------- - tmy3 : pandas.DataFrame - TMY3 data frame from :func:`pvlib.iotools.read_tmy3` with coerced year - - Returns - ------- - pandas.DataFrame - Copy of the TMY3 data frame with monotonically increasing index - - Notes - ----- - Use this function after calling :func:`~pvlib.iotools.read_tmy3` with - ``coerce_year`` set as desired. - - .. warning:: This function will only work on TMY3 with date-time index - coerced to a single year. There is no validation to check if input TMY3 - data frame has been coerced to a single year or if the date-time index - has already been fixed to be monotonically increasing. - - In :func:`~pvlib.iotools.read_tmy3`, set ``monotonic_index=True`` to read - TMY3, coerce the year, and fix the date-time index to be monotonically - increasing without calling this function afterward. - - See Also - -------- - pvlib.iotools.read_tmy3 - - """ - # NOTE: pandas index is immutable, therefore it's not possible to change a - # single item in the index, so the entire index must be replaced - - # get tmy3 timezone, index values as np.datetime64[ns], and index frequency - # as np.timedelta64[ns] - # NOTE: numpy converts index values to UTC - index_tz = tmy3.index.tz - index_values = tmy3.index.values - - # fix index to be monotonically increasing by rolling indices 1 interval, - # then adding 1 interval to all indices - index_values = np.roll(index_values, 1) + np.timedelta64(1, 'h') - - # create new datetime index and convert it to the original timezone - new_index = pd.DatetimeIndex(index_values, tz='UTC').tz_convert(index_tz) - - # copy the original TMY3 so it doesn't change, then replace the index of - # the copy with the new fixed monotonically increasing index - new_tmy3 = tmy3.copy() - new_tmy3.index = new_index - - # fixed TMY3 with monotonically increasing index - return new_tmy3 diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index 82ade9139e..9204922b8a 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -5,7 +5,7 @@ import pytest import pytz from pvlib.iotools import tmy -from pvlib.iotools import read_tmy3, tmy3_monotonic_index +from pvlib.iotools import read_tmy3 from conftest import DATA_DIR # test the API works @@ -33,19 +33,20 @@ def test_read_tmy3_recolumn(): def test_read_tmy3_norecolumn(): - data, meta = tmy.read_tmy3(TMY3_TESTFILE, recolumn=False) + data, _ = tmy.read_tmy3(TMY3_TESTFILE, recolumn=False) assert 'GHI source' in data.columns def test_read_tmy3_coerce_year(): coerce_year = 1987 - data, meta = tmy.read_tmy3(TMY3_TESTFILE, coerce_year=coerce_year) - assert (data.index.year == 1987).all() + data, _ = tmy.read_tmy3(TMY3_TESTFILE, coerce_year=coerce_year) + assert (data.index[:-1].year == 1987).all() + assert data.index[-1].year == 1988 def test_read_tmy3_no_coerce_year(): coerce_year = None - data, meta = tmy.read_tmy3(TMY3_TESTFILE, coerce_year=coerce_year) + data, _ = tmy.read_tmy3(TMY3_TESTFILE, coerce_year=coerce_year) assert 1997 and 1999 in data.index.year @@ -73,41 +74,11 @@ def test_gh865_read_tmy3_feb_leapyear_hr24(): # now check if it parses correctly when we try to coerce the year data, _ = read_tmy3(TMY3_FEB_LEAPYEAR, coerce_year=1990) # if get's here w/o an error, then gh865 is fixed, but let's check anyway - assert all(data.index.year == 1990) + assert all(data.index[:-1].year == 1990) + assert data.index[-1].year == 1991 # let's do a quick sanity check, are the indices monotonically increasing? - assert all(np.diff(data.index[:-1].astype(int)) == 3600000000000) + assert all(np.diff(data.index.astype(int)) == 3600000000000) # according to the TMY3 manual, each record corresponds to the previous # hour so check that the 1st hour is 1AM and the last hour is midnite assert data.index[0].hour == 1 assert data.index[-1].hour == 0 - - -def test_fix_tmy_coerce_year_monotonicity(): - # greensboro timezone is UTC-5 or Eastern time - gmt_5 = pytz.timezone('Etc/GMT+5') - - # tmy3 coerced to year is not monotonically increasing - greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) - - # check first hour was coerced to 1990 - firsthour = gmt_5.localize(datetime.datetime(1990, 1, 1, 1, 0, 0)) - assert firsthour == greensboro.index[0] - - # check last hour was coerced to 1990 - lasthour = gmt_5.localize(datetime.datetime(1990, 12, 31, 23, 0, 0)) - assert lasthour == greensboro.index[-2] - - # check last day, was coerced to 1990 - lastday1990 = gmt_5.localize(datetime.datetime(1990, 1, 1, 0, 0, 0)) - assert lastday1990 == greensboro.index[-1] - - # fix the index to be monotonically increasing - greensboro = tmy3_monotonic_index(greensboro) - - # check first and last hours are still 1990 - assert firsthour == greensboro.index[0] - assert lasthour == greensboro.index[-2] - - # check last day, should be 1991 now - lastday1991 = lastday1990.replace(year=1991) - assert lastday1991 == greensboro.index[-1] diff --git a/pvlib/tests/test_losses.py b/pvlib/tests/test_losses.py index deb26af0a5..170efa88c3 100644 --- a/pvlib/tests/test_losses.py +++ b/pvlib/tests/test_losses.py @@ -6,7 +6,7 @@ import pandas as pd from pandas.util.testing import assert_series_equal from pvlib.losses import soiling_hsu, soiling_kimber -from pvlib.iotools import read_tmy3, tmy3_monotonic_index +from pvlib.iotools import read_tmy3 from conftest import ( requires_scipy, needs_pandas_0_22, DATA_DIR) import pytest @@ -128,7 +128,6 @@ def test_soiling_hsu_defaults(rainfall_input, expected_output_1): def greensboro_rain(): # get TMY3 data with rain greensboro, _ = read_tmy3(DATA_DIR / '723170TYA.CSV', coerce_year=1990) - greensboro = tmy3_monotonic_index(greensboro) return greensboro.Lprecipdepth From 55669b31de3df35849715036a0e652fa6f352ade Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 28 Feb 2020 21:14:36 -0800 Subject: [PATCH 11/13] stickler: rm unused imports from tmy, test --- pvlib/iotools/tmy.py | 1 - pvlib/tests/iotools/test_tmy.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index f76940a2e6..57bf1175e1 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -7,7 +7,6 @@ import re from urllib.request import urlopen, Request import pandas as pd -import numpy as np def read_tmy3(filename=None, coerce_year=None, recolumn=True): diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index 9204922b8a..4432b8e60f 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -1,9 +1,7 @@ -import datetime from pandas.util.testing import network import numpy as np import pandas as pd import pytest -import pytz from pvlib.iotools import tmy from pvlib.iotools import read_tmy3 from conftest import DATA_DIR From ce2abbdf060d0f92f5ddeeaeaff1413a9669faea Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sat, 29 Feb 2020 11:12:29 -0800 Subject: [PATCH 12/13] suggested changes in read_tmy3 for monotonic index * sed s/data/index/g * wordsmithing docstring: coerce_year kwarg * wordsmithing what's new bug fixes * also consistently use periods for bullets in what's new --- docs/sphinx/source/whatsnew/v0.7.2.rst | 16 ++++++++-------- pvlib/iotools/tmy.py | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index b3e6dddf55..78c24b150d 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -15,35 +15,35 @@ Enhancements ~~~~~~~~~~~~ * TMY3 dataframe returned by :py:func:`~pvlib.iotools.read_tmy3` now contains the original ``Date (MM/DD/YYYY)`` and ``Time (HH:MM)`` columns that the - indices were parsed from (:pull:`866`) -* Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber` (:pull:`860`) + indices were parsed from. (:pull:`866`) +* Add Kimber soiling model :py:func:`pvlib.losses.soiling_kimber`. (:pull:`860`) Bug fixes ~~~~~~~~~ * Fix :py:func:`~pvlib.iotools.read_tmy3` parsing when February contains - a leap year (:pull:`866`) + a leap year. (:pull:`866`) * Implement NREL Developer Network API key for consistent success with API - calls in :py:mod:`pvlib.tests.iotools.test_psm3` (:pull:`873`) + calls in :py:mod:`pvlib.tests.iotools.test_psm3`. (:pull:`873`) * Fix issue with :py:class:`pvlib.location.Location` creation when - passing ``tz=datetime.timezone.utc`` (:pull:`879`) + passing ``tz=datetime.timezone.utc``. (:pull:`879`) * Fix documentation homepage title to "pvlib python" based on first heading on the page. (:pull:`890`) (:issue:`888`) * Implement `pytest-remotedata `_ to increase test suite speed. Requires ``--remote-data`` pytest flag to - execute data retrieval tests over a network.(:issue:`882`)(:pull:`896`) + execute data retrieval tests over a network. (:issue:`882`)(:pull:`896`) * Fix missing `0.7.0 what's new `_ entries about changes to ``PVSystem.pvwatts_ac``. Delete unreleased 0.6.4 what's new file. (:issue:`898`) * Compatibility with cftime 1.1. (:issue:`895`) -* Add Python3.8 to Azure Pipelines CI (:issue:`903`)(:pull:`904`) +* Add Python3.8 to Azure Pipelines CI. (:issue:`903`)(:pull:`904`) * Minor implemention changes to avoid runtime and deprecation warnings in :py:func:`~pvlib.clearsky.detect_clearsky`, :py:func:`~pvlib.iam.martin_ruiz_diffuse`, :py:func:`~pvlib.losses.soiling_hsu`, and various test functions. * Fix :py:func:`~pvlib.iotools.read_tmy3` so that when coerced to a single year - the TMY3 data will be monotonically increasing (:pull:`910`) + the TMY3 index will be monotonically increasing. (:pull:`910`) Documentation ~~~~~~~~~~~~~ diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 57bf1175e1..3b9a501fa8 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -28,9 +28,9 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): a relative file path, absolute file path, or url. coerce_year : None or int, default None - If supplied, the year of the data will be set to this value, except for - the last index which will be set to the *next* year so that the indices - increase monotonically. + If supplied, the year of the index will be set to `coerce_year`, except + for the last index value which will be set to the *next* year so that + the index increases monotonically. recolumn : bool, default True If ``True``, apply standard names to TMY3 columns. Typically this From 86a8f3ecd0182d277011155696a9ca31b57043d7 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sat, 29 Feb 2020 14:19:37 -0800 Subject: [PATCH 13/13] Apply suggestions from code review to test last hour in read_tmy3 * and minor wordsmithing in what's new Co-Authored-By: Will Holmgren --- docs/sphinx/source/whatsnew/v0.7.2.rst | 4 ++-- pvlib/tests/iotools/test_tmy.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst index 8a53d305a0..f90b9b670c 100644 --- a/docs/sphinx/source/whatsnew/v0.7.2.rst +++ b/docs/sphinx/source/whatsnew/v0.7.2.rst @@ -8,8 +8,8 @@ API Changes * :py:class:`pvlib.forecast.ForecastModel` now requires ``start`` and ``end`` arguments to be tz-localized. (:issue:`877`, :pull:`879`) * :py:func:`pvlib.iotools.read_tmy3` when coerced to a single year now returns - indices that are monotonically increasing, for example the last index will be - midnight of the *next* year. (:pull:`910`) + indices that are monotonically increasing. Therefore, the last index will be + January 1, 00:00 of the *next* year. (:pull:`910`) Enhancements ~~~~~~~~~~~~ diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index 4432b8e60f..b35f09ac53 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -46,7 +46,8 @@ def test_read_tmy3_no_coerce_year(): coerce_year = None data, _ = tmy.read_tmy3(TMY3_TESTFILE, coerce_year=coerce_year) assert 1997 and 1999 in data.index.year - + assert data.index[-2] == pd.Timestamp('1998-12-31 23:00:00-09:00') + assert data.index[-1] == pd.Timestamp('1999-01-01 00:00:00-09:00') def test_read_tmy2(): tmy.read_tmy2(TMY2_TESTFILE)