From c7bc02e389245fb5c544e4705f366a4c3ef5e8c5 Mon Sep 17 00:00:00 2001 From: Hameedullah Farooki <6323125+hafarooki@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:50:42 -0500 Subject: [PATCH 01/10] updates to swapi ialirt fitting algorithm 1. adds a temporary correction factor based on WIND measurements to be used until the SWAPI L3 science data pipeline is finalized 2. report speed only when the fits fails 3. use `raw count * 16 + 8` instead of `raw count * 16` to avoid truncating to zero counts, which can adversely affect the fits --- .../tests/ialirt/unit/test_process_swapi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index 5125ecbcb..d8a07b681 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -184,8 +184,8 @@ def test_count_rate(): """Use random realistic values to test for expected output of count_rate().""" actual_result = count_rate(1370, *[550, 5.27, 1e5]) - expected_result = 3073.023325893161 - assert actual_result == expected_result, ( + expected_result = 3073.023325893161 * np.exp(1) # factor of exp(1): temporary correction factor + assert np.isclose(actual_result, expected_result), ( f"The actual result of count_rate()" f" {actual_result} does not " f"match the expected result " @@ -202,7 +202,7 @@ def test_optimize_parameters(): "file_name": "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv", "expected_values": { # expected output and acceptable tolerance "pseudo_speed": (550, 0.01), - "pseudo_density": (5, 0.14), + "pseudo_density": (5 / np.exp(1), 0.14), # factor of exp(1): temporary correction factor "pseudo_temperature": (1e5, 0.2), }, }, @@ -210,7 +210,7 @@ def test_optimize_parameters(): "file_name": "ialirt_test_data_u_sw_650_n_sw_3.0_T_sw_120000_v2.csv", "expected_values": { # expected output and acceptable tolerance "pseudo_speed": (650, 0.01), - "pseudo_density": (3, 0.3), + "pseudo_density": (3 / np.exp(1), 0.3), # factor of exp(1): temporary correction factor "pseudo_temperature": (1.2e5, 0.28), }, }, @@ -218,7 +218,7 @@ def test_optimize_parameters(): "file_name": "ialirt_test_data_u_sw_400_n_sw_6.0_T_sw_80000_v2.csv", "expected_values": { # expected output and acceptable tolerance "pseudo_speed": (400, 0.01), - "pseudo_density": (6, 0.39), + "pseudo_density": (6 / np.exp(1), 0.39), # factor of exp(1): temporary correction factor "pseudo_temperature": (8e4, 0.15), }, }, From 2cfe8619f90524618c567cfecdb47797ad94af35 Mon Sep 17 00:00:00 2001 From: Hameedullah Farooki <6323125+hafarooki@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:37:04 -0500 Subject: [PATCH 02/10] define a constant for the temporary density factor --- imap_processing/ialirt/constants.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/imap_processing/ialirt/constants.py b/imap_processing/ialirt/constants.py index 9ea031db4..9e5712085 100644 --- a/imap_processing/ialirt/constants.py +++ b/imap_processing/ialirt/constants.py @@ -39,6 +39,13 @@ class IalirtSwapiConstants: speed_ew = 0.5 * fwhm_width # speed width of energy passband e_charge = 1.602176634e-19 # electronic charge, [C] speed_coeff = np.sqrt(2 * e_charge / prot_mass) / 1e3 + + # temporary correction factor based on WIND data available + # overlapping with the first ~month of SWAPI data. + # to be replaced once SWAPI's L3 processing pipeline is finalized + # this will increase the model count by a factor of e^1, + # changing the output density by a factor of e^-1. + temporary_density_factor = np.exp(1) class StationProperties(NamedTuple): From 16f3d52caede1e4344e84419345e5072fc505d3f Mon Sep 17 00:00:00 2001 From: Hameedullah Farooki <6323125+hafarooki@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:55:42 -0500 Subject: [PATCH 03/10] add log error message --- imap_processing/ialirt/l0/process_swapi.py | 55 +++++++++++++++++----- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 492891ed1..eb08daf03 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -2,6 +2,7 @@ import logging from decimal import Decimal +from math import isfinite import numpy as np import pandas as pd @@ -22,6 +23,7 @@ logger = logging.getLogger(__name__) NUM_IALIRT_ENERGY_STEPS = 63 +FILLVAL_FLOAT32 = -1.0e31 def count_rate( @@ -57,6 +59,9 @@ def count_rate( speed = speed * 1000 # convert km/s to m/s density = density * 1e6 # convert 1/cm**3 to 1/m**3 + # see comment on Consts.temporary_density_factor + density = density * Consts.temporary_density_factor + return ( (density * Consts.eff_area * (beta / np.pi) ** (3 / 2)) * (np.exp(-beta * (center_speed**2 + speed**2 - 2 * center_speed * speed))) @@ -104,15 +109,42 @@ def optimize_pseudo_parameters( 60000 * (initial_speed_guess / 400) ** 2, ] ) - sol = curve_fit( - f=count_rate, - xdata=energy_passbands.take(range(max_index - 3, max_index + 3), mode="clip"), - ydata=count_rates.take(range(max_index - 3, max_index + 3), mode="clip"), - sigma=count_rate_error.take(range(max_index - 3, max_index + 3), mode="clip"), - p0=initial_param_guess, - ) - - return sol[0] + + sol = None + + try: + five_point_range = range(max_index - 2, max_index + 2 + 1) + xdata = energy_passbands.take(five_point_range, mode="clip") + ydata = count_rates.take(five_point_range, mode="clip") + sigma = count_rate_error.take(five_point_range, mode="clip") + curve_fit_output = curve_fit( + f=count_rate, + xdata=xdata, + ydata=ydata, + sigma=sigma, + p0=initial_param_guess, + ) + + # if covariance matrix is not finite, scipy failed to converge to a solution and could just be reporting the initial guess + covariance_matrix_is_finite = np.all(np.isfinite(curve_fit_output[1])) + + # fit has failed if R^2 < 0.7 + yfit = count_rate(xdata, *curve_fit_output[0]) + R2 = 1 - np.sum((ydata - yfit) ** 2) / np.sum((ydata - ydata.mean()) ** 2) + R2_is_acceptable = R2 >= 0.7 + + if covariance_matrix_is_finite and R2_is_acceptable: + sol = curve_fit_output[0] + except RuntimeError as runtime_error: + logger.error(f"curve_fit failed", runtime_error) + sol = None + + # report speed only if fit fails + if sol is None: + sol = initial_param_guess.copy() + sol[1:] = FILLVAL_FLOAT32 + + return sol def geometric_mean( @@ -237,8 +269,9 @@ def process_swapi_ialirt( grouped_subset = grouped_dataset.sel(epoch=grouped_dataset.group == group) raw_coin_count = process_sweep_data(grouped_subset, "swapi_coin_cnt") - # I-ALiRT packets are 16 times less than the regular science packets. - raw_coin_count = raw_coin_count * 16 + # I-ALiRT packets have counts compressed by a factor of 16. + # Add 8 to avoid having counts truncated to 0 and to avoid counts being systematically too low + raw_coin_count = raw_coin_count * 16 + 8 # Subset to only the relevant I-ALiRT energy steps raw_coin_count = raw_coin_count[:, :NUM_IALIRT_ENERGY_STEPS] raw_coin_rate = raw_coin_count / SWAPI_LIVETIME From 454865fbf85540537ad813855acbfc0e1ab071f5 Mon Sep 17 00:00:00 2001 From: Hameedullah Farooki <6323125+hafarooki@users.noreply.github.com> Date: Thu, 29 Jan 2026 16:21:53 -0500 Subject: [PATCH 04/10] fix failing tests --- .../tests/ialirt/unit/test_process_swapi.py | 101 ++++++++++++++++-- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index d8a07b681..150f766a4 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -7,6 +7,8 @@ from imap_processing import imap_module_directory from imap_processing.ialirt.l0.process_swapi import ( + FILLVAL_FLOAT32, + Consts, count_rate, geometric_mean, optimize_pseudo_parameters, @@ -184,7 +186,7 @@ def test_count_rate(): """Use random realistic values to test for expected output of count_rate().""" actual_result = count_rate(1370, *[550, 5.27, 1e5]) - expected_result = 3073.023325893161 * np.exp(1) # factor of exp(1): temporary correction factor + expected_result = 3073.023325893161 * Consts.temporary_density_factor assert np.isclose(actual_result, expected_result), ( f"The actual result of count_rate()" f" {actual_result} does not " @@ -202,15 +204,15 @@ def test_optimize_parameters(): "file_name": "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv", "expected_values": { # expected output and acceptable tolerance "pseudo_speed": (550, 0.01), - "pseudo_density": (5 / np.exp(1), 0.14), # factor of exp(1): temporary correction factor - "pseudo_temperature": (1e5, 0.2), + "pseudo_density": (5 / Consts.temporary_density_factor, 0.14), + "pseudo_temperature": (1e5, 0.25), }, }, "test_set_2": { "file_name": "ialirt_test_data_u_sw_650_n_sw_3.0_T_sw_120000_v2.csv", "expected_values": { # expected output and acceptable tolerance "pseudo_speed": (650, 0.01), - "pseudo_density": (3 / np.exp(1), 0.3), # factor of exp(1): temporary correction factor + "pseudo_density": (3 / Consts.temporary_density_factor, 0.3), "pseudo_temperature": (1.2e5, 0.28), }, }, @@ -218,8 +220,8 @@ def test_optimize_parameters(): "file_name": "ialirt_test_data_u_sw_400_n_sw_6.0_T_sw_80000_v2.csv", "expected_values": { # expected output and acceptable tolerance "pseudo_speed": (400, 0.01), - "pseudo_density": (6 / np.exp(1), 0.39), # factor of exp(1): temporary correction factor - "pseudo_temperature": (8e4, 0.15), + "pseudo_density": (6 / Consts.temporary_density_factor, 0.39), + "pseudo_temperature": (8e4, 0.2), }, }, } @@ -259,6 +261,93 @@ def test_optimize_parameters(): ) +def test_optimize_parameters_exception_handling(): + """Test that the optimize_pseudo_parameters() function reports speed only when given data that causes curve_fit to fail.""" + + expected_speed = 557.279273 # peak passband speed + file_name = "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv" + + calibration_test_file = pd.read_csv( + f"{imap_module_directory}/tests/ialirt/data/l0/swapi_ialirt_energy_steps.csv" + ) + energy_passbands = calibration_test_file["Energy"][0:63].to_numpy().astype(float) + + energy_data = pd.read_csv(f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}") + count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates[0] = 0.0 + count_rates = np.tile(count_rates, (2, 1)) + count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() + + """ + code to select the random seed: + for i in range(100): + np.random.seed(i) + result = optimize_pseudo_parameters(count_rates * np.abs(np.random.standard_normal(size=count_rates.shape)), count_rates_errors, energy_passbands) + if np.isclose(result['pseudo_speed'][0], expected_speed, rtol=1e-6) and np.isnan(result['pseudo_density'][0]): + print(i) + """ + np.random.seed(14) + speed, density, temperature = optimize_pseudo_parameters(count_rates * np.abs(np.random.standard_normal(size=count_rates.shape)), count_rates_errors, energy_passbands) + + np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) + np.testing.assert_allclose(density, FILLVAL_FLOAT32) + np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + + +def test_optimize_parameters_bad_fit_handling(): + """Test that the optimize_pseudo_parameters() function reports speed only when the fit is too poor.""" + + file_name = "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv" + + calibration_test_file = pd.read_csv( + f"{imap_module_directory}/tests/ialirt/data/l0/swapi_ialirt_energy_steps.csv" + ) + energy_passbands = calibration_test_file["Energy"][0:63].to_numpy().astype(float) + + energy_data = pd.read_csv(f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}") + count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates[0] = 0.0 + count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() + + # add high-amplitude randomness to the count rates to make the fit poor + np.random.seed(0) + count_rates = count_rates + np.abs(np.random.standard_normal(size=count_rates.shape) * count_rates.max()) + + speed, density, temperature = optimize_pseudo_parameters(count_rates, count_rates_errors, energy_passbands) + + expected_speed = np.sqrt(energy_passbands[count_rates.argmax(axis=-1)]) * Consts.speed_coeff + + np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) + np.testing.assert_allclose(density, FILLVAL_FLOAT32) + np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + + +def test_optimize_parameters_bad_covariance_handling(): + """Test that the optimize_pseudo_parameters() function reports speed only when output covariance is nonsensical.""" + + file_name = "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv" + + calibration_test_file = pd.read_csv( + f"{imap_module_directory}/tests/ialirt/data/l0/swapi_ialirt_energy_steps.csv" + ) + energy_passbands = calibration_test_file["Energy"][0:63].to_numpy().astype(float) + + energy_data = pd.read_csv(f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}") + count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates[0] = 0.0 + count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() + + # setting errors to 0 results in infinite covariance + count_rates_errors *= 0 + + speed, density, temperature = optimize_pseudo_parameters(count_rates, count_rates_errors, energy_passbands) + + expected_speed = np.sqrt(energy_passbands[count_rates.argmax(axis=-1)]) * Consts.speed_coeff + + np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) + np.testing.assert_allclose(density, FILLVAL_FLOAT32) + np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + def test_geometric_mean(): """Test geometric_mean function.""" From 67b8846451d98394b18b708627b9d55b0659a8da Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 29 Jan 2026 17:07:05 -0700 Subject: [PATCH 05/10] docstring change --- imap_processing/ialirt/l0/process_swapi.py | 33 +++++++++++----------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index eb08daf03..1001859a2 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -2,7 +2,6 @@ import logging from decimal import Decimal -from math import isfinite import numpy as np import pandas as pd @@ -61,7 +60,7 @@ def count_rate( # see comment on Consts.temporary_density_factor density = density * Consts.temporary_density_factor - + return ( (density * Consts.eff_area * (beta / np.pi) ** (3 / 2)) * (np.exp(-beta * (center_speed**2 + speed**2 - 2 * center_speed * speed))) @@ -109,9 +108,9 @@ def optimize_pseudo_parameters( 60000 * (initial_speed_guess / 400) ** 2, ] ) - + sol = None - + try: five_point_range = range(max_index - 2, max_index + 2 + 1) xdata = energy_passbands.take(five_point_range, mode="clip") @@ -124,26 +123,27 @@ def optimize_pseudo_parameters( sigma=sigma, p0=initial_param_guess, ) - - # if covariance matrix is not finite, scipy failed to converge to a solution and could just be reporting the initial guess + + # If covariance matrix is not finite, scipy failed to converge to a + # solution and could just be reporting the initial guess covariance_matrix_is_finite = np.all(np.isfinite(curve_fit_output[1])) # fit has failed if R^2 < 0.7 yfit = count_rate(xdata, *curve_fit_output[0]) - R2 = 1 - np.sum((ydata - yfit) ** 2) / np.sum((ydata - ydata.mean()) ** 2) - R2_is_acceptable = R2 >= 0.7 - - if covariance_matrix_is_finite and R2_is_acceptable: + r2 = 1 - np.sum((ydata - yfit) ** 2) / np.sum((ydata - ydata.mean()) ** 2) + r2_is_acceptable = r2 >= 0.7 + + if covariance_matrix_is_finite and r2_is_acceptable: sol = curve_fit_output[0] - except RuntimeError as runtime_error: - logger.error(f"curve_fit failed", runtime_error) + except RuntimeError: + logger.error("curve_fit failed") sol = None - # report speed only if fit fails + # report speed only if fit fails if sol is None: sol = initial_param_guess.copy() - sol[1:] = FILLVAL_FLOAT32 - + sol[1:] = FILLVAL_FLOAT32 + return sol @@ -270,7 +270,8 @@ def process_swapi_ialirt( raw_coin_count = process_sweep_data(grouped_subset, "swapi_coin_cnt") # I-ALiRT packets have counts compressed by a factor of 16. - # Add 8 to avoid having counts truncated to 0 and to avoid counts being systematically too low + # Add 8 to avoid having counts truncated to 0 and to avoid + # counts being systematically too low raw_coin_count = raw_coin_count * 16 + 8 # Subset to only the relevant I-ALiRT energy steps raw_coin_count = raw_coin_count[:, :NUM_IALIRT_ENERGY_STEPS] From 108abe0acb8a0cea62357e3dd2067e90e9b5e7e0 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 29 Jan 2026 17:48:58 -0700 Subject: [PATCH 06/10] add zero count check --- imap_processing/ialirt/l0/process_swapi.py | 37 ++++--- .../tests/ialirt/unit/test_process_swapi.py | 97 ++++++++++++++++--- 2 files changed, 103 insertions(+), 31 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 1001859a2..b9c42a621 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -324,22 +324,27 @@ def process_swapi_ialirt( pseudo_proton_temperature_list[-5:], ) - swapi_data.append( - _populate_instrument_header_items(met) - | { - "instrument": "swapi", - "swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)), - "swapi_pseudo_proton_speed": Decimal( - f"{avg_pseudo_proton_speed:.3f}" - ), - "swapi_pseudo_proton_density": Decimal( - f"{avg_pseudo_proton_density:.3f}" - ), - "swapi_pseudo_proton_temperature": Decimal( - f"{avg_pseudo_proton_temperature:.3f}" - ), - } - ) + if ( + np.isfinite(avg_pseudo_proton_density) + and np.isfinite(avg_pseudo_proton_temperature) + and np.isfinite(avg_pseudo_proton_speed) + ): + swapi_data.append( + _populate_instrument_header_items(met) + | { + "instrument": "swapi", + "swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)), + "swapi_pseudo_proton_speed": Decimal( + f"{avg_pseudo_proton_speed:.3f}" + ), + "swapi_pseudo_proton_density": Decimal( + f"{avg_pseudo_proton_density:.3f}" + ), + "swapi_pseudo_proton_temperature": Decimal( + f"{avg_pseudo_proton_temperature:.3f}" + ), + } + ) if incomplete_groups: logger.info( f"The following swapi groups were skipped due to " diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index 150f766a4..c43bde30a 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -182,6 +182,46 @@ def test_process_swapi_ialirt( ) +@pytest.mark.external_test_data +@mock.patch("imap_processing.ialirt.l0.process_swapi.process_sweep_data") +def test_process_swapi_ialirt_zero_counts( + mock_process_sweep_data, + xarray_data, + ialirt_test_data, + sc_xarray_data, + esa_unit_conversion_table, +): + """Test that the process_swapi_ialirt() function returns expected keys.""" + + mock_process_sweep_data.return_value = ialirt_test_data[0] + + # Adding necessary time variables from spacecraft packet + xarray_data = xarray_data.assign(sc_sclk_sec=sc_xarray_data["sc_sclk_sec"]) + xarray_data["sc_sclk_sec"].data = sc_xarray_data["sc_sclk_sec"][ + 0 : xarray_data["swapi_flag"].shape[0] + ].data + xarray_data = xarray_data.assign(sc_sclk_sub_sec=sc_xarray_data["sc_sclk_sub_sec"]) + xarray_data["sc_sclk_sub_sec"].data = sc_xarray_data["sc_sclk_sub_sec"][ + 0 : xarray_data["swapi_flag"].shape[0] + ].data + + vars_to_zero = [ + "swapi_coin_cnt0", + "swapi_coin_cnt1", + "swapi_coin_cnt2", + "swapi_coin_cnt3", + "swapi_coin_cnt4", + "swapi_coin_cnt5", + ] + + for v in vars_to_zero: + xarray_data[v] = xr.zeros_like(xarray_data[v]) + + swapi_result = process_swapi_ialirt(xarray_data, esa_unit_conversion_table) + + assert swapi_result == [] + + def test_count_rate(): """Use random realistic values to test for expected output of count_rate().""" @@ -262,9 +302,10 @@ def test_optimize_parameters(): def test_optimize_parameters_exception_handling(): - """Test that the optimize_pseudo_parameters() function reports speed only when given data that causes curve_fit to fail.""" + """Test that the optimize_pseudo_parameters() function reports + speed only when given data that causes curve_fit to fail.""" - expected_speed = 557.279273 # peak passband speed + expected_speed = 557.279273 # peak passband speed file_name = "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv" calibration_test_file = pd.read_csv( @@ -272,7 +313,9 @@ def test_optimize_parameters_exception_handling(): ) energy_passbands = calibration_test_file["Energy"][0:63].to_numpy().astype(float) - energy_data = pd.read_csv(f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}") + energy_data = pd.read_csv( + f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}" + ) count_rates = energy_data["Count Rates [Hz]"].to_numpy() count_rates[0] = 0.0 count_rates = np.tile(count_rates, (2, 1)) @@ -282,12 +325,19 @@ def test_optimize_parameters_exception_handling(): code to select the random seed: for i in range(100): np.random.seed(i) - result = optimize_pseudo_parameters(count_rates * np.abs(np.random.standard_normal(size=count_rates.shape)), count_rates_errors, energy_passbands) - if np.isclose(result['pseudo_speed'][0], expected_speed, rtol=1e-6) and np.isnan(result['pseudo_density'][0]): + result = optimize_pseudo_parameters(count_rates * + np.abs(np.random.standard_normal(size=count_rates.shape)), + count_rates_errors, energy_passbands) + if np.isclose(result['pseudo_speed'][0], expected_speed, + rtol=1e-6) and np.isnan(result['pseudo_density'][0]): print(i) """ np.random.seed(14) - speed, density, temperature = optimize_pseudo_parameters(count_rates * np.abs(np.random.standard_normal(size=count_rates.shape)), count_rates_errors, energy_passbands) + speed, density, temperature = optimize_pseudo_parameters( + count_rates * np.abs(np.random.standard_normal(size=count_rates.shape)), + count_rates_errors, + energy_passbands, + ) np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) np.testing.assert_allclose(density, FILLVAL_FLOAT32) @@ -295,7 +345,8 @@ def test_optimize_parameters_exception_handling(): def test_optimize_parameters_bad_fit_handling(): - """Test that the optimize_pseudo_parameters() function reports speed only when the fit is too poor.""" + """Test that the optimize_pseudo_parameters() function + reports speed only when the fit is too poor.""" file_name = "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv" @@ -304,18 +355,26 @@ def test_optimize_parameters_bad_fit_handling(): ) energy_passbands = calibration_test_file["Energy"][0:63].to_numpy().astype(float) - energy_data = pd.read_csv(f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}") + energy_data = pd.read_csv( + f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}" + ) count_rates = energy_data["Count Rates [Hz]"].to_numpy() count_rates[0] = 0.0 count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() # add high-amplitude randomness to the count rates to make the fit poor np.random.seed(0) - count_rates = count_rates + np.abs(np.random.standard_normal(size=count_rates.shape) * count_rates.max()) + count_rates = count_rates + np.abs( + np.random.standard_normal(size=count_rates.shape) * count_rates.max() + ) - speed, density, temperature = optimize_pseudo_parameters(count_rates, count_rates_errors, energy_passbands) + speed, density, temperature = optimize_pseudo_parameters( + count_rates, count_rates_errors, energy_passbands + ) - expected_speed = np.sqrt(energy_passbands[count_rates.argmax(axis=-1)]) * Consts.speed_coeff + expected_speed = ( + np.sqrt(energy_passbands[count_rates.argmax(axis=-1)]) * Consts.speed_coeff + ) np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) np.testing.assert_allclose(density, FILLVAL_FLOAT32) @@ -323,7 +382,8 @@ def test_optimize_parameters_bad_fit_handling(): def test_optimize_parameters_bad_covariance_handling(): - """Test that the optimize_pseudo_parameters() function reports speed only when output covariance is nonsensical.""" + """Test that the optimize_pseudo_parameters() function + reports speed only when output covariance is nonsensical.""" file_name = "ialirt_test_data_u_sw_550_n_sw_5_T_sw_100000_v2.csv" @@ -332,7 +392,9 @@ def test_optimize_parameters_bad_covariance_handling(): ) energy_passbands = calibration_test_file["Energy"][0:63].to_numpy().astype(float) - energy_data = pd.read_csv(f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}") + energy_data = pd.read_csv( + f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}" + ) count_rates = energy_data["Count Rates [Hz]"].to_numpy() count_rates[0] = 0.0 count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() @@ -340,14 +402,19 @@ def test_optimize_parameters_bad_covariance_handling(): # setting errors to 0 results in infinite covariance count_rates_errors *= 0 - speed, density, temperature = optimize_pseudo_parameters(count_rates, count_rates_errors, energy_passbands) + speed, density, temperature = optimize_pseudo_parameters( + count_rates, count_rates_errors, energy_passbands + ) - expected_speed = np.sqrt(energy_passbands[count_rates.argmax(axis=-1)]) * Consts.speed_coeff + expected_speed = ( + np.sqrt(energy_passbands[count_rates.argmax(axis=-1)]) * Consts.speed_coeff + ) np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) np.testing.assert_allclose(density, FILLVAL_FLOAT32) np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + def test_geometric_mean(): """Test geometric_mean function.""" From ce5c2e2f8ed5d257a0a58cb58ab6c97eb05265dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jan 2026 00:50:18 +0000 Subject: [PATCH 07/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_processing/ialirt/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/ialirt/constants.py b/imap_processing/ialirt/constants.py index 9e5712085..b588ef5ff 100644 --- a/imap_processing/ialirt/constants.py +++ b/imap_processing/ialirt/constants.py @@ -39,7 +39,7 @@ class IalirtSwapiConstants: speed_ew = 0.5 * fwhm_width # speed width of energy passband e_charge = 1.602176634e-19 # electronic charge, [C] speed_coeff = np.sqrt(2 * e_charge / prot_mass) / 1e3 - + # temporary correction factor based on WIND data available # overlapping with the first ~month of SWAPI data. # to be replaced once SWAPI's L3 processing pipeline is finalized From bb1202765a834472dc041a836770a107bbe974d1 Mon Sep 17 00:00:00 2001 From: Hameedullah Farooki <6323125+hafarooki@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:43:36 -0500 Subject: [PATCH 08/10] fix handling of fill values; remove incorrect mock spacecraft packet test --- imap_processing/ialirt/l0/process_swapi.py | 44 ++++++++++--------- .../tests/ialirt/unit/test_process_swapi.py | 39 ---------------- 2 files changed, 23 insertions(+), 60 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index b9c42a621..544cbeb8c 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -324,27 +324,29 @@ def process_swapi_ialirt( pseudo_proton_temperature_list[-5:], ) - if ( - np.isfinite(avg_pseudo_proton_density) - and np.isfinite(avg_pseudo_proton_temperature) - and np.isfinite(avg_pseudo_proton_speed) - ): - swapi_data.append( - _populate_instrument_header_items(met) - | { - "instrument": "swapi", - "swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)), - "swapi_pseudo_proton_speed": Decimal( - f"{avg_pseudo_proton_speed:.3f}" - ), - "swapi_pseudo_proton_density": Decimal( - f"{avg_pseudo_proton_density:.3f}" - ), - "swapi_pseudo_proton_temperature": Decimal( - f"{avg_pseudo_proton_temperature:.3f}" - ), - } - ) + # replace nans (resulting from geometric means that include fill values) with fill values + avg_pseudo_proton_speed, avg_pseudo_proton_density, avg_pseudo_proton_temperature = np.nan_to_num(( + avg_pseudo_proton_speed, + avg_pseudo_proton_density, + avg_pseudo_proton_temperature + ), nan=FILLVAL_FLOAT32) + + swapi_data.append( + _populate_instrument_header_items(met) + | { + "instrument": "swapi", + "swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)), + "swapi_pseudo_proton_speed": Decimal( + f"{avg_pseudo_proton_speed:.3f}" + ), + "swapi_pseudo_proton_density": Decimal( + f"{avg_pseudo_proton_density:.3f}" + ), + "swapi_pseudo_proton_temperature": Decimal( + f"{avg_pseudo_proton_temperature:.3f}" + ), + } + ) if incomplete_groups: logger.info( f"The following swapi groups were skipped due to " diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index c43bde30a..fb3e56604 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -182,45 +182,6 @@ def test_process_swapi_ialirt( ) -@pytest.mark.external_test_data -@mock.patch("imap_processing.ialirt.l0.process_swapi.process_sweep_data") -def test_process_swapi_ialirt_zero_counts( - mock_process_sweep_data, - xarray_data, - ialirt_test_data, - sc_xarray_data, - esa_unit_conversion_table, -): - """Test that the process_swapi_ialirt() function returns expected keys.""" - - mock_process_sweep_data.return_value = ialirt_test_data[0] - - # Adding necessary time variables from spacecraft packet - xarray_data = xarray_data.assign(sc_sclk_sec=sc_xarray_data["sc_sclk_sec"]) - xarray_data["sc_sclk_sec"].data = sc_xarray_data["sc_sclk_sec"][ - 0 : xarray_data["swapi_flag"].shape[0] - ].data - xarray_data = xarray_data.assign(sc_sclk_sub_sec=sc_xarray_data["sc_sclk_sub_sec"]) - xarray_data["sc_sclk_sub_sec"].data = sc_xarray_data["sc_sclk_sub_sec"][ - 0 : xarray_data["swapi_flag"].shape[0] - ].data - - vars_to_zero = [ - "swapi_coin_cnt0", - "swapi_coin_cnt1", - "swapi_coin_cnt2", - "swapi_coin_cnt3", - "swapi_coin_cnt4", - "swapi_coin_cnt5", - ] - - for v in vars_to_zero: - xarray_data[v] = xr.zeros_like(xarray_data[v]) - - swapi_result = process_swapi_ialirt(xarray_data, esa_unit_conversion_table) - - assert swapi_result == [] - def test_count_rate(): """Use random realistic values to test for expected output of count_rate().""" From c6e508cad4666c1edf0b87b1fbd61c07fe72abfe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:43:57 +0000 Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_processing/ialirt/l0/process_swapi.py | 13 ++++++++++--- .../tests/ialirt/unit/test_process_swapi.py | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 544cbeb8c..4be3353d5 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -325,11 +325,18 @@ def process_swapi_ialirt( ) # replace nans (resulting from geometric means that include fill values) with fill values - avg_pseudo_proton_speed, avg_pseudo_proton_density, avg_pseudo_proton_temperature = np.nan_to_num(( + ( avg_pseudo_proton_speed, avg_pseudo_proton_density, - avg_pseudo_proton_temperature - ), nan=FILLVAL_FLOAT32) + avg_pseudo_proton_temperature, + ) = np.nan_to_num( + ( + avg_pseudo_proton_speed, + avg_pseudo_proton_density, + avg_pseudo_proton_temperature, + ), + nan=FILLVAL_FLOAT32, + ) swapi_data.append( _populate_instrument_header_items(met) diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index fb3e56604..3d7b6b9eb 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -182,7 +182,6 @@ def test_process_swapi_ialirt( ) - def test_count_rate(): """Use random realistic values to test for expected output of count_rate().""" From ea286047cbed58e9979a6f62ef9d8b27292672b5 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 30 Jan 2026 11:57:33 -0700 Subject: [PATCH 10/10] docstring update --- imap_processing/ialirt/l0/process_swapi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 4be3353d5..410cc6361 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -324,7 +324,8 @@ def process_swapi_ialirt( pseudo_proton_temperature_list[-5:], ) - # replace nans (resulting from geometric means that include fill values) with fill values + # replace nans (resulting from geometric means that + # include fill values) with fill values ( avg_pseudo_proton_speed, avg_pseudo_proton_density,