From dd70fa96fa9815255962fa9f0826fbe06c25f576 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Fri, 6 Feb 2026 14:38:36 -0700 Subject: [PATCH] Fix Hi L2 naming of bg variables --- imap_processing/ena_maps/ena_maps.py | 4 +- imap_processing/hi/hi_l2.py | 18 ++++----- .../tests/ena_maps/test_ena_maps.py | 2 +- imap_processing/tests/hi/test_hi_l2.py | 38 +++++++++---------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/imap_processing/ena_maps/ena_maps.py b/imap_processing/ena_maps/ena_maps.py index 9c2440ef53..a8c1657431 100644 --- a/imap_processing/ena_maps/ena_maps.py +++ b/imap_processing/ena_maps/ena_maps.py @@ -657,8 +657,8 @@ class HiPointingSet(LoHiBasePointingSet): # renamed to match L2 variables l1c_to_l2_var_mapping: ClassVar[dict[str, str]] = { "exposure_times": "exposure_factor", - "background_rates": "bg_rates", - "background_rates_uncertainty": "bg_rates_unc", + "background_rates": "bg_rate", + "background_rates_uncertainty": "bg_rate_sys_err", } def __init__(self, dataset: xr.Dataset | str | Path): diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index fb599a2a42..d69d21acff 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -28,13 +28,13 @@ SC_FRAME_VARS_TO_PROJECT = { "counts", "exposure_factor", - "bg_rates", - "bg_rates_unc", + "bg_rate", + "bg_rate_sys_err", "obs_date", } HELIO_FRAME_VARS_TO_PROJECT = SC_FRAME_VARS_TO_PROJECT | {"energy_sc"} # TODO: is an exposure time weighted average for obs_date appropriate? -FULL_EXPOSURE_TIME_AVERAGE_SET = {"bg_rates", "bg_rates_unc", "obs_date", "energy_sc"} +FULL_EXPOSURE_TIME_AVERAGE_SET = {"bg_rate", "bg_rate_sys_err", "obs_date", "energy_sc"} # ============================================================================= @@ -297,7 +297,7 @@ def calculate_all_rates_and_intensities( Parameters ---------- map_ds : xarray.Dataset - Map dataset with projected PSET data (counts, exposure_factor, bg_rates, + Map dataset with projected PSET data (counts, exposure_factor, bg_rate, energy_delta_minus, energy_delta_plus, etc.) and an `energy` coordinate containing the ESA nominal central energies in keV. l2_ancillary_path_dict : dict[str, pathlib.Path] @@ -362,7 +362,7 @@ def calculate_ena_signal_rates(map_ds: xr.Dataset) -> xr.Dataset: Parameters ---------- map_ds : xarray.Dataset - Map dataset that has counts, exposure_factor, and bg_rates calculated. + Map dataset that has counts, exposure_factor, and bg_rate calculated. Returns ------- @@ -373,7 +373,7 @@ def calculate_ena_signal_rates(map_ds: xr.Dataset) -> xr.Dataset: with np.errstate(divide="ignore"): # Calculate the ENA Signal Rate map_ds["ena_signal_rates"] = ( - map_ds["counts"] / map_ds["exposure_factor"] - map_ds["bg_rates"] + map_ds["counts"] / map_ds["exposure_factor"] - map_ds["bg_rate"] ) # Calculate the ENA Signal Rate Uncertainties # The minimum count uncertainty is 1 for any pixel that has non-zero @@ -440,7 +440,7 @@ def calculate_ena_intensity( map_ds["ena_signal_rate_stat_unc"] / flux_conversion_divisor ) map_ds["ena_intensity_sys_err"] = ( - np.sqrt(map_ds["bg_rates"] * map_ds["exposure_factor"]) + np.sqrt(map_ds["bg_rate"] * map_ds["exposure_factor"]) / map_ds["exposure_factor"] / flux_conversion_divisor ) @@ -562,7 +562,7 @@ def _calculate_improved_stat_variance( logger.debug("Computing geometric factor normalized signal rates") - # signal_rates = counts / exposure_factor - bg_rates + # signal_rates = counts / exposure_factor - bg_rate # signal_rates shape is: (n_epoch, n_energy, n_cal_prod, n_spatial_pixels) signal_rates = map_ds["ena_signal_rates"] @@ -581,7 +581,7 @@ def _calculate_improved_stat_variance( logger.debug("Including background rates in uncertainty calculation") # Convert averaged signal rates back to flux uncertainties # Total count rates for Poisson uncertainty calculation - total_count_rates_for_uncertainty = map_ds["bg_rates"] + averaged_signal_rates + total_count_rates_for_uncertainty = map_ds["bg_rate"] + averaged_signal_rates logger.debug("Computing improved flux uncertainties") # Statistical variance: diff --git a/imap_processing/tests/ena_maps/test_ena_maps.py b/imap_processing/tests/ena_maps/test_ena_maps.py index 25ecffd6ae..71acac4fd5 100644 --- a/imap_processing/tests/ena_maps/test_ena_maps.py +++ b/imap_processing/tests/ena_maps/test_ena_maps.py @@ -154,7 +154,7 @@ def test_init(self, hi_pset_cdf_path): assert hi_pset.num_points == 3600 np.testing.assert_array_equal(hi_pset.az_el_points.shape, (3600, 2)) - for var_name in ["exposure_factor", "bg_rates", "bg_rates_unc"]: + for var_name in ["exposure_factor", "bg_rate", "bg_rate_sys_err"]: assert var_name in hi_pset.data def test_from_cdf(self, hi_pset_cdf_path): diff --git a/imap_processing/tests/hi/test_hi_l2.py b/imap_processing/tests/hi/test_hi_l2.py index 9f9f983821..e28e54aca9 100644 --- a/imap_processing/tests/hi/test_hi_l2.py +++ b/imap_processing/tests/hi/test_hi_l2.py @@ -128,10 +128,10 @@ def sample_map_dataset(): "ena_intensity_sys_err": xr.DataArray( np.random.rand(*shape) * 5 + 1, dims=list(coords.keys()) ), - "bg_rates": xr.DataArray( + "bg_rate": xr.DataArray( np.random.rand(*shape) * 20 + 5, dims=list(coords.keys()) ), - "bg_rates_unc": xr.DataArray( + "bg_rate_sys_err": xr.DataArray( np.random.rand(*shape) * 2 + 1, dims=list(coords.keys()) ), "exposure_factor": xr.DataArray( @@ -301,10 +301,10 @@ def test_calculate_ena_signal_rates(empty_rectangular_map_dataset): name="exposure_factor", dims=list(exposure_sizes.keys()), ), - "bg_rates": xr.DataArray( + "bg_rate": xr.DataArray( np.arange(np.prod(tuple(map_ds.sizes.values()))).reshape(counts_shape) % 2, - name="bg_rates", + name="bg_rate", dims=list(map_ds.sizes.keys()), ), } @@ -316,7 +316,7 @@ def test_calculate_ena_signal_rates(empty_rectangular_map_dataset): assert var_name in result_ds assert result_ds[var_name].shape == counts_shape # Verify that there are no negative signal rates. The synthetic data combination - # where counts = 0, exposure_factor = 1, and bg_rates = 1 would result in + # where counts = 0, exposure_factor = 1, and bg_rate = 1 would result in # an ena_signal_rate of (0 / 1) - 1 = -1 assert np.nanmin(result_ds["ena_signal_rates"].values) >= 0 # Verify that the minimum finite uncertainty is sqrt(1) / exposure_factor. @@ -345,9 +345,9 @@ def ena_intensity_map_ds(empty_rectangular_map_dataset): name="ena_signal_rate_stat_unc", dims=list(map_ds.sizes.keys()), ), - "bg_rates_unc": xr.DataArray( + "bg_rate_sys_err": xr.DataArray( np.arange(np.prod(tuple(map_ds.sizes.values()))).reshape(var_shape) % 3, - name="bg_rates_unc", + name="bg_rate_sys_err", dims=list(map_ds.sizes.keys()), ), } @@ -359,7 +359,7 @@ def ena_intensity_map_ds(empty_rectangular_map_dataset): ) map_ds.update( { - "bg_rates": xr.DataArray( + "bg_rate": xr.DataArray( np.ones(bg_shape) * 5.0, dims=[d for d in map_ds.sizes.keys() if d != "calibration_prod"], ), @@ -566,7 +566,7 @@ def test_weighted_average_mathematical_correctness(): np.array([100.0, 400.0]).reshape(1, 1, 2, 1, 1), dims=list(coords.keys()), ), - "bg_rates": xr.DataArray( + "bg_rate": xr.DataArray( np.array([5.0]).reshape(1, 1, 1, 1), dims=[d for d in coords.keys() if d != "calibration_prod"], ), @@ -624,7 +624,7 @@ def test_statistical_uncertainty_combination_correctness(): sys_err_values, dims=list(coords.keys()) ), "ena_signal_rates": xr.DataArray(flux_values, dims=list(coords.keys())), - "bg_rates": xr.DataArray( + "bg_rate": xr.DataArray( np.array([1.0, 2.0]).reshape(1, 1, 2, 1, 1), dims=list(coords.keys()) ), "exposure_factor": xr.DataArray( @@ -783,8 +783,8 @@ def test_process_single_pset_renames_variables( # Check that variables were renamed assert "exposure_factor" in result - assert "bg_rates" in result - assert "bg_rates_unc" in result + assert "bg_rate" in result + assert "bg_rate_sys_err" in result # Original names should not exist assert "exposure_times" not in result assert "background_rates" not in result @@ -834,16 +834,16 @@ def test_process_single_pset_exposure_time_weighting( descriptor = MapDescriptor.from_string("h90-ena-h-sf-nsp-full-gcs-6deg-3mo") energy_kev = xr.DataArray([0.5, 0.75, 1.1], dims=["esa_energy_step"]) - # bg_rates should be multiplied by exposure_factor + # bg_rate should be multiplied by exposure_factor result = process_single_pset( mock_pset_dataset, energy_kev, descriptor, - vars_to_exposure_time_average={"bg_rates"}, + vars_to_exposure_time_average={"bg_rate"}, ) - # bg_rates was 5.0, exposure_factor is 100.0, so result should be 500.0 - assert np.allclose(result["bg_rates"].values, 500.0) + # bg_rate was 5.0, exposure_factor is 100.0, so result should be 500.0 + assert np.allclose(result["bg_rate"].values, 500.0) @mock.patch("imap_processing.hi.hi_l2.calculate_ram_mask") @@ -973,10 +973,8 @@ def mock_map_dataset_for_rates(): np.ones(exposure_shape) * 10.0, dims=["epoch", "esa_energy_step", "longitude", "latitude"], ), - "bg_rates": xr.DataArray( - np.ones(shape) * 2.0, dims=list(coords.keys())[:5] - ), - "bg_rates_unc": xr.DataArray( + "bg_rate": xr.DataArray(np.ones(shape) * 2.0, dims=list(coords.keys())[:5]), + "bg_rate_sys_err": xr.DataArray( np.ones(shape) * 0.5, dims=list(coords.keys())[:5] ), "obs_date": xr.DataArray(