From 0aa64df149748cefdfe248b176563cadd8162ac8 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 25 Sep 2026 11:55:27 -0600 Subject: [PATCH 01/13] init commit for updated culling --- ...sensor-de-product-lookup_20251001_v001.csv | 2 - .../tests/ultra/unit/test_l1c_lookup_utils.py | 84 +++++++++++++ .../tests/ultra/unit/test_lookup_utils.py | 89 +++----------- .../tests/ultra/unit/test_ultra_l1b.py | 8 +- imap_processing/ultra/l1b/extendedspin.py | 1 + imap_processing/ultra/l1b/lookup_utils.py | 110 +++++++----------- imap_processing/ultra/l1c/l1c_lookup_utils.py | 62 ++++++++++ 7 files changed, 209 insertions(+), 147 deletions(-) delete mode 100644 imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv deleted file mode 100644 index f3c31692cf..0000000000 --- a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv +++ /dev/null @@ -1,2 +0,0 @@ -repointing_id_start,repointing_id_end,de_product -0,,imap_ultra_l1b_45sensor-de diff --git a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py index d2149d4ea2..0ee81f2caf 100644 --- a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py @@ -1,10 +1,15 @@ +from unittest import mock + import astropy_healpix.healpy as hp import numpy as np +import pandas as pd import pytest import xarray as xr +from imap_processing import imap_module_directory from imap_processing.ultra.l1c.l1c_lookup_utils import ( calculate_accepted_pixels, + get_de_product_name, get_scattering_thresholds_for_energy, get_spacecraft_pointing_lookup_tables, get_static_deadtime_ratios, @@ -12,6 +17,8 @@ mask_below_fwhm_scattering_threshold, ) +TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1" + @pytest.mark.external_test_data def test_get_spacecraft_pointing_lookup_tables(ancillary_files): @@ -202,3 +209,80 @@ def test_calculate_accepted_pixels_restrict_fov(ancillary_files): expected_accepted_px = in_restricted_fov(mock_theta, mock_phi, 45) expected_accepted_px[:, :, :outside_inds] = False assert np.array_equal(expected_accepted_px, valid_spun_pixels) + + +def test_get_de_product_name_no_repoint(): + """Tests function get_de_product_name when the lookup is missing the repoint.""" + ancillary_files = { + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [1, 2], + "repointing_id_end": [3, 4], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + ], + } + ) + with pytest.raises(ValueError, match="No DE product found for repoint ID 0"): + get_de_product_name("repoint00000", 45, ancillary_files) + + +def test_get_de_product_name_multiple_products(): + """Tests function get_de_product_name when the lookup is ambiguous.""" + ancillary_files = { + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [2, 2], + "repointing_id_end": [3, 4], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + ], + } + ) + with pytest.raises(ValueError, match="Multiple DE products found"): + get_de_product_name("repoint00002", 45, ancillary_files) + + +def test_get_de_product_name(): + """Tests function get_de_product_name.""" + ancillary_files = { + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [0, 2, 4], + "repointing_id_end": [1, 4, np.nan], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + "imap_ultra_l1b_45sensor-priority-2-de", + ], + } + ) + # Test with a repoint in the future. Should return the priority 2 de product + # since the last repoint range does not have an end and should be assumed to + # cover all future repoints. + de_product = get_de_product_name("repoint00100", 45, ancillary_files) + assert de_product == "imap_ultra_l1b_45sensor-priority-2-de" + + # Test with valid repoint that falls in the second range. + de_product = get_de_product_name("repoint00003", 45, ancillary_files) + assert de_product == "imap_ultra_l1b_45sensor-priority-1-de" diff --git a/imap_processing/tests/ultra/unit/test_lookup_utils.py b/imap_processing/tests/ultra/unit/test_lookup_utils.py index 051edb56b6..b87b2ff1d9 100644 --- a/imap_processing/tests/ultra/unit/test_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_lookup_utils.py @@ -1,3 +1,4 @@ +import datetime from unittest import mock import numpy as np @@ -7,9 +8,9 @@ from imap_processing import imap_module_directory from imap_processing.quality_flags import ImapDEOutliersUltraFlags from imap_processing.ultra.l1b.lookup_utils import ( + ExtendedSpinConfig, get_angular_profiles, get_back_position, - get_de_product_name, get_ebins, get_energy_efficiencies, get_energy_norm, @@ -219,78 +220,14 @@ def test_get_scattering_thresholds(ancillary_files): assert thresholds[(20.0, np.inf)] == 4.0 -def test_get_de_product_name_no_repoint(): - """Tests function get_de_product_name when the lookup is missing the repoint.""" - ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" - } - with mock.patch( - "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" - ) as mock_read_csv: - mock_read_csv.return_value = pd.DataFrame( - { - "repointing_id_start": [1, 2], - "repointing_id_end": [3, 4], - "de_product": [ - "imap_ultra_l1b_45sensor-de", - "imap_ultra_l1b_45sensor-priority-1-de", - ], - } - ) - with pytest.raises(ValueError, match="No DE product found for repoint ID 0"): - get_de_product_name("repoint00000", 45, "l1b", ancillary_files) - - -def test_get_de_product_name_multiple_products(): - """Tests function get_de_product_name when the lookup is ambiguous.""" - ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" - } - with mock.patch( - "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" - ) as mock_read_csv: - mock_read_csv.return_value = pd.DataFrame( - { - "repointing_id_start": [2, 2], - "repointing_id_end": [3, 4], - "de_product": [ - "imap_ultra_l1b_45sensor-de", - "imap_ultra_l1b_45sensor-priority-1-de", - ], - } - ) - with pytest.raises(ValueError, match="Multiple DE products found"): - get_de_product_name("repoint00002", 45, "l1b", ancillary_files) - - -def test_get_de_product_name(): - """Tests function get_de_product_name.""" - ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" - } - with mock.patch( - "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" - ) as mock_read_csv: - mock_read_csv.return_value = pd.DataFrame( - { - "repointing_id_start": [0, 2, 4], - "repointing_id_end": [1, 4, np.nan], - "de_product": [ - "imap_ultra_l1b_45sensor-de", - "imap_ultra_l1b_45sensor-priority-1-de", - "imap_ultra_l1b_45sensor-priority-2-de", - ], - } - ) - # Test with a repoint in the future. Should return the priority 2 de product - # since the last repoint range does not have an end and should be assumed to - # cover all future repoints. - de_product = get_de_product_name("repoint00100", 45, "l1b", ancillary_files) - assert de_product == "imap_ultra_l1b_45sensor-priority-2-de" - - # Test with valid repoint that falls in the second range. - de_product = get_de_product_name("repoint00003", 45, "l1b", ancillary_files) - assert de_product == "imap_ultra_l1b_45sensor-priority-1-de" +def test_extended_config_class(): + """Tests the ExtendedSpinConfig class.""" + config_path = ( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" + ) + config = ExtendedSpinConfig.from_csv(config_path, 128) + + assert config.priority == "p0" + assert config.calibration == "c1" + assert config.thresholds == {0: 96.5, 1: 80.5, 2: 8.5, 3: 10.5, 4: 3.5} + assert config.date == datetime.datetime(2026, 1, 22) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index 5b59e17e16..a6e6794b95 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -229,7 +229,7 @@ def test_ultra_l1b_extendedspin( ancillary_files = { "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) @@ -264,7 +264,7 @@ def test_cdf_extendedspin( ancillary_files = { "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) """Tests that CDF file is created and contains same attributes as xarray.""" @@ -304,7 +304,7 @@ def test_cdf_goodtimes( ancillary_files = { "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) @@ -348,7 +348,7 @@ def test_cdf_badtimes( ancillary_files = { "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index db8cfe2117..44dd017448 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -79,6 +79,7 @@ def calculate_extendedspin( spin_tbin_edges = get_binned_spins_edges( spin, spin_period, spin_starttime, spin_bin_size ) + # Calculate goodtime quality flags. # The culling algorithms should be called in the following order # 1. Low voltage diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 2181905d79..752f8d01f1 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -1,6 +1,8 @@ """Contains tools for lookup tables for l1b.""" +import datetime import logging +from dataclasses import dataclass import numpy as np import numpy.typing as npt @@ -622,69 +624,47 @@ def get_scattering_thresholds(ancillary_files: dict) -> dict: return threshold_dict -def get_de_product_name( - repoint: str, sensor: int, data_level: str, ancillary_files: dict -) -> str: - """ - Get the name of the de product to use for processing. - - This will be either the raw de product or a priority 1-4 de product, depending on - the pointing and data level. - - Note: Currently the lookup tables are identical between ultra45 and ultra90, - but this function accounts for the possibility of them being different in the - future. - - Parameters - ---------- - repoint : str - The repointing ID in the format "repointXXXXX" where XXXXX is the repointing - number. - sensor : int - Sensor number, either 45 or 90. - data_level : str - Data level, either "l1b" or "l1c". - ancillary_files : dict - Ancillary files containing the lookup tables to determine which DE product - to use based on the repointing ID. - - Returns - ------- - de_product_name : str - Name of the de product to use for processing. - """ - if data_level not in ["l1b", "l1c"]: - raise ValueError(f"Invalid data level: {data_level}. Must be 'l1b' or 'l1c'.") - # load the lookup table. - # The lookup table will have columns for repointing_id_start, repointing_id_end, - # and de_product. If repointing_id_end is NaN that indicates that the de_product - # should be used for all repoint IDs greater than or equal to repointing_id_start. - file_name = f"{data_level}-{sensor}sensor-de-product-lookup" - de_lookup = pd.read_csv(ancillary_files[file_name]) - repoint_id = int(repoint.replace("repoint", "")) - # Filter the dataset to find where the current repoint ID falls within the - # repointing_id_start and repointing_id_end range. OR if repointing_id_end is NaN, - # then just check if repoint_id is greater than or equal to repointing_id_start - repoint_row = de_lookup[ - (de_lookup["repointing_id_start"] <= repoint_id) - & ( - (de_lookup["repointing_id_end"] > repoint_id) - | (pd.isna(de_lookup["repointing_id_end"])) +@dataclass +class ExtendedSpinConfig: + """Pointing dependent l1b culling configurations.""" + + thresholds: dict[int, float] + date: datetime.datetime + priority: str + calibration: str + + @classmethod + def from_csv( + cls, config_file_path: str, pointing_number: int + ) -> "ExtendedSpinConfig": + """ + Construct an ExtendedSpinConfig object from inputs. + + Parameters + ---------- + config_file_path : str + Path to the CSV file containing the configuration data. + pointing_number : int + The current pointing number for which to retrieve the + configuration. + + Returns + ------- + ExtendedSpinConfig + An instance of ExtendedSpinConfig with the thresholds, date, + priority, and calibration values for the specified pointing number. + """ + df = pd.read_csv(config_file_path) + # Get the first row where the pointing number is greater or equal + # to the pointing number in the config file. + config = df[(df["pointing"] >= pointing_number)].iloc[0] + thresh_colnames = [col for col in df.columns if "cullThresh" in col] + thresholds = {int(col.split("_")[-1]): config[col] for col in thresh_colnames} + date = datetime.datetime.strptime(config["date"], "%m/%d/%y") + + return cls( + thresholds=thresholds, + date=date, + priority=config["pri_config"], + calibration=config["cal_config"], ) - ] - if repoint_row.empty: - raise ValueError( - f"No DE product found for repoint ID {repoint_id} in {file_name}" - ) - if len(repoint_row) > 1: - raise ValueError( - f"Multiple DE products found for repoint ID {repoint_id} using " - f"ancillary file {file_name}. Check that the " - f"repointing_id_start and repointing_id_end values are correct" - f" and not overlapping." - ) - product = repoint_row["de_product"].values[0] - logger.info( - f"Using DE product {product} for repoint ID {repoint_id} based on lookup table" - ) - return product diff --git a/imap_processing/ultra/l1c/l1c_lookup_utils.py b/imap_processing/ultra/l1c/l1c_lookup_utils.py index 341c5de8e9..e91dc629ab 100644 --- a/imap_processing/ultra/l1c/l1c_lookup_utils.py +++ b/imap_processing/ultra/l1c/l1c_lookup_utils.py @@ -513,3 +513,65 @@ def build_energy_bins( energy_bin_geometric_means = np.sqrt(energy_bin_edges[:-1] * energy_bin_edges[1:]) return intervals, energy_midpoints, energy_bin_geometric_means + + +def get_de_product_name(repoint: str, sensor: int, ancillary_files: dict) -> str: + """ + Get the name of the de product to use for processing. + + This will be either the raw de product or a priority 1-4 de product, depending on + the pointing and data level. + + Note: Currently the lookup tables are identical between ultra45 and ultra90, + but this function accounts for the possibility of them being different in the + future. + + Parameters + ---------- + repoint : str + The repointing ID in the format "repointXXXXX" where XXXXX is the repointing + number. + sensor : int + Sensor number, either 45 or 90. + ancillary_files : dict + Ancillary files containing the lookup tables to determine which DE product + to use based on the repointing ID. + + Returns + ------- + de_product_name : str + Name of the de product to use for processing. + """ + # load the lookup table. + # The lookup table will have columns for repointing_id_start, repointing_id_end, + # and de_product. If repointing_id_end is NaN that indicates that the de_product + # should be used for all repoint IDs greater than or equal to repointing_id_start. + file_name = f"l1c-{sensor}sensor-de-product-lookup" + de_lookup = pd.read_csv(ancillary_files[file_name]) + repoint_id = int(repoint.replace("repoint", "")) + # Filter the dataset to find where the current repoint ID falls within the + # repointing_id_start and repointing_id_end range. OR if repointing_id_end is NaN, + # then just check if repoint_id is greater than or equal to repointing_id_start + repoint_row = de_lookup[ + (de_lookup["repointing_id_start"] <= repoint_id) + & ( + (de_lookup["repointing_id_end"] > repoint_id) + | (pd.isna(de_lookup["repointing_id_end"])) + ) + ] + if repoint_row.empty: + raise ValueError( + f"No DE product found for repoint ID {repoint_id} in {file_name}" + ) + if len(repoint_row) > 1: + raise ValueError( + f"Multiple DE products found for repoint ID {repoint_id} using " + f"ancillary file {file_name}. Check that the " + f"repointing_id_start and repointing_id_end values are correct" + f" and not overlapping." + ) + product = repoint_row["de_product"].values[0] + logger.info( + f"Using DE product {product} for repoint ID {repoint_id} based on lookup table" + ) + return product From 6b33eb8b164a3e8e5f322067bcfa3b1b28c2b3ed Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 25 Sep 2026 12:54:12 -0600 Subject: [PATCH 02/13] refactor # Conflicts: # imap_processing/ultra/l1b/extendedspin.py --- imap_processing/ultra/l1b/extendedspin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index 44dd017448..c28f10a910 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -79,7 +79,6 @@ def calculate_extendedspin( spin_tbin_edges = get_binned_spins_edges( spin, spin_period, spin_starttime, spin_bin_size ) - # Calculate goodtime quality flags. # The culling algorithms should be called in the following order # 1. Low voltage @@ -96,8 +95,8 @@ def calculate_extendedspin( energy_ranges = get_binned_energy_ranges(intervals) energy_bin_flags = get_energy_range_flags(energy_ranges) - # Get valid event counts per energy range and spin bin, shared across all of - # the culling steps below. + # Get valid events and counts at each spin bin for the + # designated culling channel. de_counts_summary = get_valid_de_count_summary( de_dataset, energy_ranges, From 854d254ec2a77f231edf403b320044c273ef90c5 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 25 Sep 2026 13:14:08 -0600 Subject: [PATCH 03/13] fix comment --- imap_processing/ultra/l1b/extendedspin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index c28f10a910..db8cfe2117 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -95,8 +95,8 @@ def calculate_extendedspin( energy_ranges = get_binned_energy_ranges(intervals) energy_bin_flags = get_energy_range_flags(energy_ranges) - # Get valid events and counts at each spin bin for the - # designated culling channel. + # Get valid event counts per energy range and spin bin, shared across all of + # the culling steps below. de_counts_summary = get_valid_de_count_summary( de_dataset, energy_ranges, From a0a88e6bc45970256cee87c0f2cc7fa4eae067e0 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Tue, 29 Sep 2026 11:27:11 -0600 Subject: [PATCH 04/13] fix tests --- imap_processing/cli.py | 6 +- .../ultra/unit/test_ultra_l1b_culling.py | 230 ++++++++++-------- imap_processing/ultra/constants.py | 4 + imap_processing/ultra/l1b/extendedspin.py | 8 +- imap_processing/ultra/l1b/lookup_utils.py | 16 +- imap_processing/ultra/l1b/ultra_l1b.py | 55 ++--- .../ultra/l1b/ultra_l1b_culling.py | 134 +++++----- 7 files changed, 234 insertions(+), 219 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index bfa8aff830..c366386d83 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -2069,6 +2069,10 @@ def do_processing( ) datasets = ultra_l1a.ultra_l1a(science_files[0], create_derived_l1b=True) elif self.data_level == "l1b": + if self.repointing is None: + raise ValueError( + "Repointing must be provided for ULTRA L1b processing." + ) science_files = dependencies.get_file_paths(source="ultra", data_type="l1a") l1a_dict = { dataset.attrs["Logical_source"]: dataset @@ -2084,7 +2088,7 @@ def do_processing( ancillary_files = {} for path in anc_paths: ancillary_files[path.stem.split("_")[2]] = path - datasets = ultra_l1b.ultra_l1b(combined, ancillary_files) + datasets = ultra_l1b.ultra_l1b(combined, ancillary_files, self.repointing) elif self.data_level == "l1c": science_files = dependencies.get_file_paths(source="ultra", data_type="l1a") l1a_dict = { diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index a9e2702544..7c02cb1fff 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -1,5 +1,6 @@ """Tests Culling for ULTRA L1b.""" +import datetime from unittest import mock import numpy as np @@ -16,6 +17,7 @@ ImapRatesUltraFlags, ) from imap_processing.ultra.constants import UltraConstants +from imap_processing.ultra.l1b.lookup_utils import ExtendedSpinConfig from imap_processing.ultra.l1b.ultra_l1b_culling import ( compare_aux_univ_spin_table, count_rejected_events_per_spin, @@ -42,7 +44,6 @@ get_spin_data, get_valid_de_count_summary, get_valid_earth_angle_events, - get_valid_events_per_energy_range, ) from imap_processing.ultra.l1b.ultra_l1b_extended import get_spin_info from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins @@ -57,6 +58,7 @@ def setup_repoint_47_data(): de_ds = xr.Dataset( { "de_event_met": ("epoch", de_df.event_times.values), + "event_times": ("epoch", de_df.event_times.values), "energy_spacecraft": ("epoch", de_df.energy_spacecraft.values), "quality_outliers": ("epoch", de_df.quality_outliers.values), "quality_scattering": ("epoch", de_df.quality_scattering.values), @@ -71,7 +73,25 @@ def setup_repoint_47_data(): xspin.spin_start_time.values, spin_bin_size, ) - return de_ds, xspin, spin_tbin_edges + + de_datasets = {"p0": de_ds, "p1": de_ds} + spin_config = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", + ) + + # Get the energy ranges + energy_ranges = get_binned_energy_ranges(build_energy_bins()[0]) + + de_counts_summary = get_valid_de_count_summary( + de_datasets, + energy_ranges, + spin_tbin_edges, + spin_config, + 90, + ) + + return de_datasets, xspin, spin_tbin_edges, energy_ranges, de_counts_summary @pytest.fixture @@ -555,8 +575,8 @@ def test_get_valid_earth_angle_events(mock_spkezr): np.testing.assert_array_equal(actual_flags, expected_flags) -def test_get_valid_events_per_energy_range(): - """Tests get_valid_events_per_energy_range function.""" +def test_get_valid_de_count_summary_valid_events(): + """Tests that get_valid_de_count_summary only counts valid events.""" np.random.seed(0) energy_range_edges = np.array([3, 5, 7, 18]) # 3 example energy bins # example energy values that fall into different bins @@ -573,72 +593,59 @@ def test_get_valid_events_per_energy_range(): # mark event 6 as having an invalid ebin ebin[5] = -1 de_dps_velocity = np.random.random((len(energy), 3)) + # Give each event its own time and spin bin so the counts per spin bin show + # exactly which events were counted as valid. + event_times = np.arange(1, len(energy) + 1) + spin_tbin_edges = np.arange(0.5, len(energy) + 1) de_dataset = xr.Dataset( { "de_dps_velocity": (("epoch", "component"), de_dps_velocity), - "event_times": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", event_times), + "de_event_met": ("epoch", event_times), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", quality_outliers), "quality_scattering": ("epoch", quality_scattering), "ebin": ("epoch", ebin), } ) - keepout_angle = np.radians(180) - valid_events = get_valid_events_per_energy_range( - de_dataset, energy_range_edges, keepout_angle, 90 - ) - - # Assert that for the first energy bin (3-5), all are false - assert np.array_equal(valid_events[0], np.full(len(valid_events[0]), False)) - # Assert that for the second energy bin (5-7), all are false except - # events 3 and 8 (event 1 had an outlier flag, event 2 had a scattering flag) - expected_flags_ebin2 = np.array( - [ - False, - False, - True, - False, - False, - False, - False, - True, - False, - False, - False, - False, - ] - ) - assert np.array_equal(valid_events[1], expected_flags_ebin2) - # Assert that for the third energy bin (7-18), all are false except events 4 and 7 - # (event 5 was marked as an outlier and event 6 has an invalid ebin) - expected_flags_ebin3 = np.array( - [ - False, - False, - False, - True, - False, - False, - True, - False, - False, - False, - False, - False, - ] - ) - assert np.array_equal(valid_events[2], expected_flags_ebin3) + spin_config = ExtendedSpinConfig( + thresholds={}, + date=datetime.datetime(2025, 10, 1), + priority="p0", + calibration="", + ) + counts = get_valid_de_count_summary( + {"p0": de_dataset, "p1": de_dataset}, + energy_range_edges, + spin_tbin_edges, + spin_config, + 90, + ) + + # Assert that for the first energy bin (3-5), nothing is counted + assert np.array_equal(counts[0], np.zeros(len(energy))) + # Assert that for the second energy bin (5-7), only events 3 and 8 are counted + # (event 1 had a scattering flag, event 2 had an outlier flag) + expected_counts_ebin2 = np.array([0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]) + assert np.array_equal(counts[1], expected_counts_ebin2) + # Assert that for the third energy bin (7-18), only events 4 and 7 are counted + # (event 5 was marked as an outlier, event 6 has an invalid ebin, and event 9 + # had a scattering flag) + expected_counts_ebin3 = np.array([0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0]) + assert np.array_equal(counts[2], expected_counts_ebin3) @mock.patch("imap_processing.ultra.l1b.ultra_l1b_culling.sp.spkezr") -def test_get_valid_events_per_energy_range_ultra45(mock_spkezr): - """Tests get_valid_events_per_energy_range function.""" +def test_get_valid_de_count_summary_ultra45(mock_spkezr): + """Tests the Earth angle cut in get_valid_de_count_summary for ULTRA 45.""" np.random.seed(0) mock_imap_state = np.random.random(6) # Mock IMAP state for testing mock_spkezr.return_value = (mock_imap_state, None) energy_range_edges = np.array([3, 5, 7, 18]) # 3 example energy bins energy = np.arange(18) + event_times = np.full(len(energy), 798033671) + spin_tbin_edges = np.array([798033670, 798033672]) # mark all events with valid outlier and scattering flags and valid ebins. de_dps_velocity = np.random.random((len(energy), 3)) @@ -646,23 +653,35 @@ def test_get_valid_events_per_energy_range_ultra45(mock_spkezr): de_dataset = xr.Dataset( { "velocity_dps_sc": (("epoch", "component"), de_dps_velocity), - "event_times": ("epoch", np.full(len(energy), 798033671)), + "event_times": ("epoch", event_times), + "de_event_met": ("epoch", event_times), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), "ebin": ("epoch", np.full(len(energy), 10)), } ) + spin_config = ExtendedSpinConfig( + thresholds={}, + date=datetime.datetime(2025, 10, 1), + priority="p0", + calibration="", + ) # ensure that all events fail the earth angle check by setting a very large # keepout angle keepout_angle = np.radians(360) - valid_events = get_valid_events_per_energy_range( - de_dataset, energy_range_edges, keepout_angle, 45 + counts = get_valid_de_count_summary( + {"p0": de_dataset, "p1": de_dataset}, + energy_range_edges, + spin_tbin_edges, + spin_config, + 45, + keepout_angle, ) # although all events were valid for outliers, scattering, and ebin, all events # failed the earth angle check for ultra45 - assert not np.any(valid_events) + assert not np.any(counts) @mock.patch( @@ -689,6 +708,7 @@ def test_flag_high_energy(): de_dataset = xr.Dataset( { "de_event_met": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", np.arange(len(energy))), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), @@ -700,8 +720,18 @@ def test_flag_high_energy(): spin_tbin_edges = np.arange( start=0, stop=len(energy) + 1, step=4 ) # create spin bins of 4 seconds + de_datasets = {"p0": de_dataset, "p1": de_dataset} + spin_config = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", + ) + de_counts_summary = get_valid_de_count_summary( - de_dataset, energy_range_edges, spin_tbin_edges + de_datasets, + energy_range_edges, + spin_tbin_edges, + spin_config, + 90, ) quality_flags = flag_high_energy( de_counts_summary, @@ -730,10 +760,6 @@ def test_flag_high_energy(): assert not np.any(quality_flags[:, 2]) -@mock.patch( - "imap_processing.ultra.l1b.ultra_l1b_culling.UltraConstants.HIGH_ENERGY_CULL_CHANNEL", - 4, -) @pytest.mark.external_test_data def test_validate_high_energy_cull(setup_repoint_47_data): """Validate that high energy spins are correctly flagged""" @@ -743,12 +769,14 @@ def test_validate_high_energy_cull(setup_repoint_47_data): expected_qf = pd.read_csv( TEST_PATH / "validate_high_energy_culling_results_repoint00047_v2.csv" ).to_numpy() - de_ds, _, spin_tbin_edges = setup_repoint_47_data - # Get the energy ranges - energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) - de_counts_summary = get_valid_de_count_summary( - de_ds, energy_ranges, spin_tbin_edges + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data ) + # # Get the energy ranges + # energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) + # de_counts_summary = get_valid_de_count_summary( + # de_ds, energy_ranges, spin_tbin_edges + # ) high_energy_combined_spin_bin_radius = 3 e_flags = flag_high_energy( de_counts_summary, @@ -779,6 +807,7 @@ def test_flag_statistical_outliers(): de_dataset = xr.Dataset( { "de_event_met": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", np.arange(len(energy))), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), @@ -788,10 +817,17 @@ def test_flag_statistical_outliers(): spin_tbin_edges = np.arange( start=0, stop=len(energy) + 1, step=spin_step ) # create spin bins of 7 seconds + de_datasets = {"p0": de_dataset, "p1": de_dataset} + spin_config = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", + ) + de_counts_summary = get_valid_de_count_summary( - de_dataset, + de_datasets, energy_range_edges, spin_tbin_edges, + spin_config, 90, ) quality_flags, convergence, iterations, std_diff = flag_statistical_outliers( @@ -834,6 +870,7 @@ def test_flag_statistical_outliers_invalid_events(): de_dataset = xr.Dataset( { "de_event_met": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", np.arange(len(energy))), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), @@ -844,10 +881,17 @@ def test_flag_statistical_outliers_invalid_events(): start=0, stop=len(energy) + 1, step=5 ) # create spin bins of 5 seconds mask = np.ones((len(energy_range_edges) - 1, len(spin_tbin_edges) - 1), dtype=bool) + de_datasets = {"p0": de_dataset, "p1": de_dataset} + spin_config = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", + ) + de_counts_summary = get_valid_de_count_summary( - de_dataset, + de_datasets, energy_range_edges, spin_tbin_edges, + spin_config, 90, ) quality_flags, convergence, iterations, std_diff = flag_statistical_outliers( @@ -891,9 +935,9 @@ def test_validate_stat_cull(setup_repoint_47_data): results_df = pd.read_csv( TEST_PATH / "validate_stat_culling_results_repoint00047_v3.csv" ) - de_ds, _, spin_tbin_edges = setup_repoint_47_data - # Get the energy ranges - energy_ranges = get_binned_energy_ranges(build_energy_bins()[0]) + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data + ) # Create a mask of flagged events to test that the stat cull algorithm # properly ignores these. The test data was created using this exact mask as well. @@ -901,12 +945,6 @@ def test_validate_stat_cull(setup_repoint_47_data): mask[0:2, 0:2] = ( True # This will mark the first 2 energy bins and first 2 spin bins as flagged ) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, - ) # ignored in the statistics calculation and flagging. flags, con, it, std = flag_statistical_outliers( de_counts_summary, spin_tbin_edges, energy_ranges, mask @@ -950,19 +988,14 @@ def test_validate_upstream_ion_cull(setup_repoint_47_data): expected_results = pd.read_csv( TEST_PATH / "validate_upstream_ion_1_culling_results_repoint00047_v1.csv" ).to_numpy() - de_ds, _, spin_tbin_edges = setup_repoint_47_data - intervals, _, _ = build_energy_bins() - energy_ranges = get_binned_energy_ranges(intervals) + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data + ) mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) mask[0:2, 0:2] = ( True # This will mark the first 2 energy bins and first 2 spin bins as flagged ) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, - ) + flags = flag_upstream_ion( de_counts_summary, energy_ranges, @@ -978,16 +1011,10 @@ def test_validate_upstream_ion_cull(setup_repoint_47_data): @pytest.mark.external_test_data def test_upstream_ion_cull_invalid_channels(setup_repoint_47_data): """Validate upstream ion error handling.""" - de_ds, _, spin_tbin_edges = setup_repoint_47_data - intervals, _, _ = build_energy_bins() - energy_ranges = get_binned_energy_ranges(intervals) - mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data ) + mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) with pytest.raises( ValueError, match="Channels provided for upstream ion flagging" @@ -1008,20 +1035,13 @@ def test_validate_spectral_cull(setup_repoint_47_data): expected_results = pd.read_csv( TEST_PATH / "validate_spectral_culling_results_repoint00047_v1.csv" ).to_numpy() - de_ds, _, spin_tbin_edges = setup_repoint_47_data - intervals, _, _ = build_energy_bins() - energy_ranges = get_binned_energy_ranges(intervals) + de_datasets, xspin, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data + ) mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) mask[0:2, 0:2] = ( True # This will mark the first 2 energy bins and first 2 spin bins as flagged ) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, - ) - flags = flag_spectral_events( de_counts_summary, energy_ranges, diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 3ef2173129..af6814e0d5 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -254,3 +254,7 @@ class UltraConstants: # When True, applies the scattering rejection mask based on the FWHM thresholds # to the L1C fine energy bin maps. APPLY_SCATTERING_REJECTION_L1C: bool = False + + # When true, only use raw direct events for culling in extendedspin.py. Otherwise, + # refer to the priority specified in the extendedspin-config ancillary file. + L1B_USE_RAW_DE_ONLY = False diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index db8cfe2117..a6fef51f81 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -5,6 +5,7 @@ from numpy.typing import NDArray from imap_processing.ultra.constants import UltraConstants +from imap_processing.ultra.l1b.lookup_utils import ExtendedSpinConfig from imap_processing.ultra.l1b.ultra_l1b_culling import ( count_rejected_events_per_spin, expand_bin_flags_to_spins, @@ -33,7 +34,8 @@ def calculate_extendedspin( dict_datasets: dict[str, xr.Dataset], - de_dataset: xr.Dataset, + de_dataset: dict[str, xr.Dataset], + extendedspin_conf: ExtendedSpinConfig, name: str, instrument_id: int, ) -> xr.Dataset: @@ -46,6 +48,8 @@ def calculate_extendedspin( Dictionary containing all the datasets. de_dataset : xarray.Dataset Dataset containing the direct event data. + extendedspin_conf : ExtendedSpinConfig + Configurations for culling spins. name : str Name of the dataset. instrument_id : int @@ -94,13 +98,13 @@ def calculate_extendedspin( # Get the energy ranges energy_ranges = get_binned_energy_ranges(intervals) energy_bin_flags = get_energy_range_flags(energy_ranges) - # Get valid event counts per energy range and spin bin, shared across all of # the culling steps below. de_counts_summary = get_valid_de_count_summary( de_dataset, energy_ranges, spin_tbin_edges, + extendedspin_conf, instrument_id, ) diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 752f8d01f1..30ce82a329 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -634,9 +634,7 @@ class ExtendedSpinConfig: calibration: str @classmethod - def from_csv( - cls, config_file_path: str, pointing_number: int - ) -> "ExtendedSpinConfig": + def from_csv(cls, config_file_path: str, repointing: str) -> "ExtendedSpinConfig": """ Construct an ExtendedSpinConfig object from inputs. @@ -644,7 +642,7 @@ def from_csv( ---------- config_file_path : str Path to the CSV file containing the configuration data. - pointing_number : int + repointing : str The current pointing number for which to retrieve the configuration. @@ -654,10 +652,18 @@ def from_csv( An instance of ExtendedSpinConfig with the thresholds, date, priority, and calibration values for the specified pointing number. """ + repoint_int = int(repointing.replace("repoint", "")) df = pd.read_csv(config_file_path) # Get the first row where the pointing number is greater or equal # to the pointing number in the config file. - config = df[(df["pointing"] >= pointing_number)].iloc[0] + filtered_df = df[(df["pointing"] >= repoint_int)] + if filtered_df.empty: + raise ValueError( + f"The ancillary file: {config_file_path} contains no " + f"configurations for {repointing}" + ) + config = filtered_df.iloc[0] + # Get the column names that contain "cullThresh" thresh_colnames = [col for col in df.columns if "cullThresh" in col] thresholds = {int(col.split("_")[-1]): config[col] for col in thresh_colnames} date = datetime.datetime.strptime(config["date"], "%m/%d/%y") diff --git a/imap_processing/ultra/l1b/ultra_l1b.py b/imap_processing/ultra/l1b/ultra_l1b.py index d698193145..c15ff00a3b 100644 --- a/imap_processing/ultra/l1b/ultra_l1b.py +++ b/imap_processing/ultra/l1b/ultra_l1b.py @@ -9,12 +9,14 @@ from imap_processing.ultra.l1b.de import calculate_de from imap_processing.ultra.l1b.extendedspin import calculate_extendedspin from imap_processing.ultra.l1b.goodtimes import calculate_goodtimes -from imap_processing.ultra.l1b.lookup_utils import get_de_product_name +from imap_processing.ultra.l1b.lookup_utils import ExtendedSpinConfig logger = logging.getLogger(__name__) -def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: +def ultra_l1b( + data_dict: dict, ancillary_files: dict, repointing: str +) -> list[xr.Dataset]: """ Will process ULTRA L1A data into L1B CDF files at output_filepath. @@ -24,6 +26,9 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: The data itself and its dependent data. ancillary_files : dict Ancillary files. + repointing : str + The repointing ID in the format "repointXXXXX" where XXXXX is the repointing + number. Returns ------- @@ -73,39 +78,23 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict and f"imap_ultra_l1b_{instrument_id}sensor-status" in data_dict ): - # get repoint number - repoint = data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"].attrs.get( - "Repointing", None + # Create dictionary of the de datasets + # For now, only pass in priority 1 and raw de datasets. + de_datasets = { + "p0": data_dict[f"imap_ultra_l1a_{instrument_id}sensor-de"], + "p1": data_dict[f"imap_ultra_l1a_{instrument_id}sensor-priority-1-de"], + } + # Get the extended spin config from ancillary files. + extended_spin_config_anc = ancillary_files[ + f"l1b-{instrument_id}sensor-extendedspin-config" + ] + extendedspin_config = ExtendedSpinConfig.from_csv( + extended_spin_config_anc, repointing ) - if repoint is None: - raise ValueError("Repointing ID attribute is missing from the dataset.") - # Determine which l1b de product to use in calculating the goodtimes - # Will be either the raw de product or a priority 1-4 de product. - de_product_desc = get_de_product_name( - repoint, instrument_id, "l1b", ancillary_files - ) - if de_product_desc not in data_dict: - raise ValueError( - f"Selected L1B DE product '{de_product_desc}' for instrument " - f"{instrument_id} is not present in data_dict. Available L1B DE " - f"products: {data_dict.keys()}" - ) extendedspin_dataset = calculate_extendedspin( - { - f"imap_ultra_l1a_{instrument_id}sensor-aux": data_dict[ - f"imap_ultra_l1a_{instrument_id}sensor-aux" - ], - f"imap_ultra_l1a_{instrument_id}sensor-params": data_dict[ - f"imap_ultra_l1a_{instrument_id}sensor-params" - ], - f"imap_ultra_l1a_{instrument_id}sensor-rates": data_dict[ - f"imap_ultra_l1a_{instrument_id}sensor-rates" - ], - f"imap_ultra_l1b_{instrument_id}sensor-status": data_dict[ - f"imap_ultra_l1b_{instrument_id}sensor-status" - ], - }, - data_dict[de_product_desc], + data_dict, + de_datasets, + extendedspin_config, f"imap_ultra_l1b_{instrument_id}sensor-extendedspin", instrument_id, ) diff --git a/imap_processing/ultra/l1b/ultra_l1b_culling.py b/imap_processing/ultra/l1b/ultra_l1b_culling.py index d8a2e61508..e0b2d31d61 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_culling.py +++ b/imap_processing/ultra/l1b/ultra_l1b_culling.py @@ -24,6 +24,7 @@ from imap_processing.spice.spin import get_spin_data from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import ( + ExtendedSpinConfig, get_scattering_coefficients, get_scattering_thresholds, ) @@ -1061,40 +1062,91 @@ def flag_spectral_events( def get_valid_de_count_summary( - de_dataset: xr.Dataset, + de_datasets: dict[str, xr.Dataset], energy_ranges: NDArray, spin_tbin_edges: NDArray, + spin_config: ExtendedSpinConfig, sensor_id: int = 90, + earth_ang_45: float = UltraConstants.EARTH_ANGLE_45_THRESHOLD, ) -> NDArray: """ Get a summary of valid counts per energy range and spin bin. Parameters ---------- - de_dataset : xr.Dataset - Direct event dataset. + de_datasets : dict[str, xr.Dataset] + Dictionary of raw, and priority 1-4 direct event datasets. energy_ranges : numpy.ndarray Array of energy range edges. spin_tbin_edges : numpy.ndarray Array of spin time bin edges. + spin_config : ExtendedSpinConfig + Spin configuration object containing the direct event priority to use. sensor_id : int Sensor ID (e.g., 45 or 90). + earth_ang_45 : float + Earth angle to use for culling in ULTRA 45. Returns ------- counts : numpy.ndarray A 2D array of counts per energy range and spin bin for valid events. """ - valid_events = get_valid_events_per_energy_range( - de_dataset, energy_ranges, UltraConstants.EARTH_ANGLE_45_THRESHOLD, sensor_id - ) + # Get the specified de priority. E.g. "p0" for raw de or "p1" for priority 1 de. + priority_conf = spin_config.priority + n_energy_ranges = len(energy_ranges) - 1 counts: np.ndarray = np.zeros( - (len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=float + (n_energy_ranges, len(spin_tbin_edges) - 1), dtype=float ) + for i in range(n_energy_ranges): + # By default, use priority 1 de unless it's the last energy bin and the + # config is p0 or if the config is set to use raw de only. + de = de_datasets["p1"] + if UltraConstants.L1B_USE_RAW_DE_ONLY or ( + i == n_energy_ranges - 1 and priority_conf == "p0" + ): + de = de_datasets["p0"] + valid_outliers = de["quality_outliers"].values == 0 + valid_scattering = de["quality_scattering"].values == 0 + # TODO what about species non-proton? For those psets dont cull based on + # High energy? + valid_ebin = np.isin( + de["ebin"].values, UltraConstants.TOFXPH_SPECIES_GROUPS["proton"] + ) + # Exclude events with invalid (fill) event times. + valid_event_times = de["event_times"].values > 0 + energy_mask = ( + (de["energy_spacecraft"].values >= energy_ranges[i]) + & (de["energy_spacecraft"].values < energy_ranges[i + 1]) + & valid_event_times + ) + if not np.any(energy_mask): + continue + # subset the dataset to events within the energy range + de_dataset_subset = de.isel(epoch=energy_mask) + valid_earth_angle: np.ndarray = np.full(np.sum(energy_mask), True, dtype=bool) + # For ultra45, also apply an Earth angle cut to remove times when + # the Earth is in the field of view. ULTRA 90 does not require this since + # Earth is always outside the field of view. + if sensor_id == 45: + valid_earth_angle = get_valid_earth_angle_events( + de_dataset_subset, earth_ang_45 + ) - for i in range(len(energy_ranges) - 1): + # Count events at the valid energy ranges if they meet all the criteria for + # valid events: not flagged as outliers, not flagged as scattering, + # in a valid ebin, and (for ultra45) have a valid Earth angle. + valid_events = np.logical_and.reduce( + [ + valid_outliers[energy_mask], + valid_scattering[energy_mask], + valid_ebin[energy_mask], + valid_earth_angle, + ] + ) counts[i, :], _ = np.histogram( - de_dataset["de_event_met"].values[valid_events[i, :]], bins=spin_tbin_edges + de_dataset_subset["de_event_met"].values[valid_events], + bins=spin_tbin_edges, ) return counts @@ -1132,70 +1184,6 @@ def combine_de_counts_summary( return np.mean(windows, axis=-1) -def get_valid_events_per_energy_range( - de_dataset: xr.Dataset, energy_ranges: NDArray, earth_ang_45: float, sensor_id: int -) -> NDArray: - """ - Get valid events per energy range. - - Parameters - ---------- - de_dataset : xr.Dataset - Direct event dataset. - energy_ranges : numpy.ndarray - Array of energy range edges. - earth_ang_45 : float - Earth angle to use for culling in ULTRA 45. - sensor_id : int - Sensor ID (e.g., 45 or 90). - - Returns - ------- - valid_events_per_range : numpy.ndarray - A boolean array of shape (n_energy_ranges, n_events). - """ - event_energies = de_dataset["energy_spacecraft"].values - valid_events: np.ndarray = np.zeros( - (len(energy_ranges) - 1, len(event_energies)), dtype=bool - ) - valid_outliers = de_dataset["quality_outliers"].values == 0 - valid_scattering = de_dataset["quality_scattering"].values == 0 - # TODO what about species non-proton? For those psets dont cull based on - # High energy? - ebin = de_dataset["ebin"].values - valid_ebin = np.isin(ebin, UltraConstants.TOFXPH_SPECIES_GROUPS["proton"]) - for i in range(len(energy_ranges) - 1): - energy_mask = (event_energies >= energy_ranges[i]) & ( - event_energies < energy_ranges[i + 1] - ) - if not np.any(energy_mask): - continue - # subset the dataset to events within the energy range - de_dataset_subset = de_dataset.isel(epoch=energy_mask) - valid_earth_angle: np.ndarray = np.full(np.sum(energy_mask), True, dtype=bool) - # For ultra45, also apply an Earth angle cut to remove times when - # the Earth is in the field of view. ULTRA 90 does not require this since Earth - # is always outside the field of view. - if sensor_id == 45: - valid_earth_angle = get_valid_earth_angle_events( - de_dataset_subset, earth_ang_45 - ) - - # Flag events at the valid energy ranges if they meet all the criteria for - # valid events: not flagged as outliers, not flagged as scattering, - # in a valid ebin, and (for ultra45) have a valid Earth angle. - valid_events[i, energy_mask] = np.logical_and.reduce( - [ - valid_outliers[energy_mask], - valid_scattering[energy_mask], - valid_ebin[energy_mask], - valid_earth_angle, - ] - ) - - return valid_events - - def get_valid_earth_angle_events( de_dataset_subset: xr.Dataset, earth_ang_45: float = UltraConstants.EARTH_ANGLE_45_THRESHOLD, From 3d88e8a858ce11ef48b22c0289fbc8beceb7e5c1 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Tue, 29 Sep 2026 11:36:01 -0600 Subject: [PATCH 05/13] fix high energy test --- .../ultra/unit/test_ultra_l1b_culling.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 7c02cb1fff..7edd318725 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -760,6 +760,10 @@ def test_flag_high_energy(): assert not np.any(quality_flags[:, 2]) +@mock.patch( + "imap_processing.ultra.l1b.ultra_l1b_culling.UltraConstants.HIGH_ENERGY_CULL_CHANNEL", + 4, +) @pytest.mark.external_test_data def test_validate_high_energy_cull(setup_repoint_47_data): """Validate that high energy spins are correctly flagged""" @@ -769,14 +773,16 @@ def test_validate_high_energy_cull(setup_repoint_47_data): expected_qf = pd.read_csv( TEST_PATH / "validate_high_energy_culling_results_repoint00047_v2.csv" ).to_numpy() - de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( - setup_repoint_47_data + de_datasets, _, spin_tbin_edges, _, _ = setup_repoint_47_data + spin_config = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", + ) + # Get the energy ranges + energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) + de_counts_summary = get_valid_de_count_summary( + de_datasets, energy_ranges, spin_tbin_edges, spin_config ) - # # Get the energy ranges - # energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) - # de_counts_summary = get_valid_de_count_summary( - # de_ds, energy_ranges, spin_tbin_edges - # ) high_energy_combined_spin_bin_radius = 3 e_flags = flag_high_energy( de_counts_summary, From 570a9b553b63495b843c4f26f347a1cb6b4ab46c Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Tue, 29 Sep 2026 11:47:19 -0600 Subject: [PATCH 06/13] move back helper to reduce pr changes --- .../tests/ultra/unit/test_l1c_lookup_utils.py | 81 ------------------- .../tests/ultra/unit/test_lookup_utils.py | 78 ++++++++++++++++++ imap_processing/ultra/l1b/lookup_utils.py | 63 +++++++++++++++ imap_processing/ultra/l1c/l1c_lookup_utils.py | 62 -------------- 4 files changed, 141 insertions(+), 143 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py index 0ee81f2caf..2546ddf150 100644 --- a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py @@ -1,15 +1,11 @@ -from unittest import mock - import astropy_healpix.healpy as hp import numpy as np -import pandas as pd import pytest import xarray as xr from imap_processing import imap_module_directory from imap_processing.ultra.l1c.l1c_lookup_utils import ( calculate_accepted_pixels, - get_de_product_name, get_scattering_thresholds_for_energy, get_spacecraft_pointing_lookup_tables, get_static_deadtime_ratios, @@ -209,80 +205,3 @@ def test_calculate_accepted_pixels_restrict_fov(ancillary_files): expected_accepted_px = in_restricted_fov(mock_theta, mock_phi, 45) expected_accepted_px[:, :, :outside_inds] = False assert np.array_equal(expected_accepted_px, valid_spun_pixels) - - -def test_get_de_product_name_no_repoint(): - """Tests function get_de_product_name when the lookup is missing the repoint.""" - ancillary_files = { - "l1c-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" - } - with mock.patch( - "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" - ) as mock_read_csv: - mock_read_csv.return_value = pd.DataFrame( - { - "repointing_id_start": [1, 2], - "repointing_id_end": [3, 4], - "de_product": [ - "imap_ultra_l1b_45sensor-de", - "imap_ultra_l1b_45sensor-priority-1-de", - ], - } - ) - with pytest.raises(ValueError, match="No DE product found for repoint ID 0"): - get_de_product_name("repoint00000", 45, ancillary_files) - - -def test_get_de_product_name_multiple_products(): - """Tests function get_de_product_name when the lookup is ambiguous.""" - ancillary_files = { - "l1c-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" - } - with mock.patch( - "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" - ) as mock_read_csv: - mock_read_csv.return_value = pd.DataFrame( - { - "repointing_id_start": [2, 2], - "repointing_id_end": [3, 4], - "de_product": [ - "imap_ultra_l1b_45sensor-de", - "imap_ultra_l1b_45sensor-priority-1-de", - ], - } - ) - with pytest.raises(ValueError, match="Multiple DE products found"): - get_de_product_name("repoint00002", 45, ancillary_files) - - -def test_get_de_product_name(): - """Tests function get_de_product_name.""" - ancillary_files = { - "l1c-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" - } - with mock.patch( - "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" - ) as mock_read_csv: - mock_read_csv.return_value = pd.DataFrame( - { - "repointing_id_start": [0, 2, 4], - "repointing_id_end": [1, 4, np.nan], - "de_product": [ - "imap_ultra_l1b_45sensor-de", - "imap_ultra_l1b_45sensor-priority-1-de", - "imap_ultra_l1b_45sensor-priority-2-de", - ], - } - ) - # Test with a repoint in the future. Should return the priority 2 de product - # since the last repoint range does not have an end and should be assumed to - # cover all future repoints. - de_product = get_de_product_name("repoint00100", 45, ancillary_files) - assert de_product == "imap_ultra_l1b_45sensor-priority-2-de" - - # Test with valid repoint that falls in the second range. - de_product = get_de_product_name("repoint00003", 45, ancillary_files) - assert de_product == "imap_ultra_l1b_45sensor-priority-1-de" diff --git a/imap_processing/tests/ultra/unit/test_lookup_utils.py b/imap_processing/tests/ultra/unit/test_lookup_utils.py index b87b2ff1d9..4b47908a4d 100644 --- a/imap_processing/tests/ultra/unit/test_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_lookup_utils.py @@ -11,6 +11,7 @@ ExtendedSpinConfig, get_angular_profiles, get_back_position, + get_de_product_name, get_ebins, get_energy_efficiencies, get_energy_norm, @@ -220,6 +221,83 @@ def test_get_scattering_thresholds(ancillary_files): assert thresholds[(20.0, np.inf)] == 4.0 +def test_get_de_product_name_no_repoint(): + """Tests function get_de_product_name when the lookup is missing the repoint.""" + ancillary_files = { + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [1, 2], + "repointing_id_end": [3, 4], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + ], + } + ) + with pytest.raises(ValueError, match="No DE product found for repoint ID 0"): + get_de_product_name("repoint00000", 45, ancillary_files) + + +def test_get_de_product_name_multiple_products(): + """Tests function get_de_product_name when the lookup is ambiguous.""" + ancillary_files = { + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [2, 2], + "repointing_id_end": [3, 4], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + ], + } + ) + with pytest.raises(ValueError, match="Multiple DE products found"): + get_de_product_name("repoint00002", 45, ancillary_files) + + +def test_get_de_product_name(): + """Tests function get_de_product_name.""" + ancillary_files = { + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [0, 2, 4], + "repointing_id_end": [1, 4, np.nan], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + "imap_ultra_l1b_45sensor-priority-2-de", + ], + } + ) + # Test with a repoint in the future. Should return the priority 2 de product + # since the last repoint range does not have an end and should be assumed to + # cover all future repoints. + de_product = get_de_product_name("repoint00100", 45, ancillary_files) + assert de_product == "imap_ultra_l1b_45sensor-priority-2-de" + + # Test with valid repoint that falls in the second range. + de_product = get_de_product_name("repoint00003", 45, ancillary_files) + assert de_product == "imap_ultra_l1b_45sensor-priority-1-de" + + def test_extended_config_class(): """Tests the ExtendedSpinConfig class.""" config_path = ( diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 30ce82a329..09eeab2b33 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -624,6 +624,69 @@ def get_scattering_thresholds(ancillary_files: dict) -> dict: return threshold_dict +# TODO move to l1c_lookup_utils.py +def get_de_product_name(repoint: str, sensor: int, ancillary_files: dict) -> str: + """ + Get the name of the de product to use for processing. + + This will be either the raw de product or a priority 1-4 de product, depending on + the pointing and data level. + + Note: Currently the lookup tables are identical between ultra45 and ultra90, + but this function accounts for the possibility of them being different in the + future. + + Parameters + ---------- + repoint : str + The repointing ID in the format "repointXXXXX" where XXXXX is the repointing + number. + sensor : int + Sensor number, either 45 or 90. + ancillary_files : dict + Ancillary files containing the lookup tables to determine which DE product + to use based on the repointing ID. + + Returns + ------- + de_product_name : str + Name of the de product to use for processing. + """ + # load the lookup table. + # The lookup table will have columns for repointing_id_start, repointing_id_end, + # and de_product. If repointing_id_end is NaN that indicates that the de_product + # should be used for all repoint IDs greater than or equal to repointing_id_start. + file_name = f"l1c-{sensor}sensor-de-product-lookup" + de_lookup = pd.read_csv(ancillary_files[file_name]) + repoint_id = int(repoint.replace("repoint", "")) + # Filter the dataset to find where the current repoint ID falls within the + # repointing_id_start and repointing_id_end range. OR if repointing_id_end is NaN, + # then just check if repoint_id is greater than or equal to repointing_id_start + repoint_row = de_lookup[ + (de_lookup["repointing_id_start"] <= repoint_id) + & ( + (de_lookup["repointing_id_end"] > repoint_id) + | (pd.isna(de_lookup["repointing_id_end"])) + ) + ] + if repoint_row.empty: + raise ValueError( + f"No DE product found for repoint ID {repoint_id} in {file_name}" + ) + if len(repoint_row) > 1: + raise ValueError( + f"Multiple DE products found for repoint ID {repoint_id} using " + f"ancillary file {file_name}. Check that the " + f"repointing_id_start and repointing_id_end values are correct" + f" and not overlapping." + ) + product = repoint_row["de_product"].values[0] + logger.info( + f"Using DE product {product} for repoint ID {repoint_id} based on lookup table" + ) + return product + + @dataclass class ExtendedSpinConfig: """Pointing dependent l1b culling configurations.""" diff --git a/imap_processing/ultra/l1c/l1c_lookup_utils.py b/imap_processing/ultra/l1c/l1c_lookup_utils.py index e91dc629ab..341c5de8e9 100644 --- a/imap_processing/ultra/l1c/l1c_lookup_utils.py +++ b/imap_processing/ultra/l1c/l1c_lookup_utils.py @@ -513,65 +513,3 @@ def build_energy_bins( energy_bin_geometric_means = np.sqrt(energy_bin_edges[:-1] * energy_bin_edges[1:]) return intervals, energy_midpoints, energy_bin_geometric_means - - -def get_de_product_name(repoint: str, sensor: int, ancillary_files: dict) -> str: - """ - Get the name of the de product to use for processing. - - This will be either the raw de product or a priority 1-4 de product, depending on - the pointing and data level. - - Note: Currently the lookup tables are identical between ultra45 and ultra90, - but this function accounts for the possibility of them being different in the - future. - - Parameters - ---------- - repoint : str - The repointing ID in the format "repointXXXXX" where XXXXX is the repointing - number. - sensor : int - Sensor number, either 45 or 90. - ancillary_files : dict - Ancillary files containing the lookup tables to determine which DE product - to use based on the repointing ID. - - Returns - ------- - de_product_name : str - Name of the de product to use for processing. - """ - # load the lookup table. - # The lookup table will have columns for repointing_id_start, repointing_id_end, - # and de_product. If repointing_id_end is NaN that indicates that the de_product - # should be used for all repoint IDs greater than or equal to repointing_id_start. - file_name = f"l1c-{sensor}sensor-de-product-lookup" - de_lookup = pd.read_csv(ancillary_files[file_name]) - repoint_id = int(repoint.replace("repoint", "")) - # Filter the dataset to find where the current repoint ID falls within the - # repointing_id_start and repointing_id_end range. OR if repointing_id_end is NaN, - # then just check if repoint_id is greater than or equal to repointing_id_start - repoint_row = de_lookup[ - (de_lookup["repointing_id_start"] <= repoint_id) - & ( - (de_lookup["repointing_id_end"] > repoint_id) - | (pd.isna(de_lookup["repointing_id_end"])) - ) - ] - if repoint_row.empty: - raise ValueError( - f"No DE product found for repoint ID {repoint_id} in {file_name}" - ) - if len(repoint_row) > 1: - raise ValueError( - f"Multiple DE products found for repoint ID {repoint_id} using " - f"ancillary file {file_name}. Check that the " - f"repointing_id_start and repointing_id_end values are correct" - f" and not overlapping." - ) - product = repoint_row["de_product"].values[0] - logger.info( - f"Using DE product {product} for repoint ID {repoint_id} based on lookup table" - ) - return product From 1fc56206b9391739615362d2dc222da7aa3af407 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Tue, 29 Sep 2026 14:48:35 -0600 Subject: [PATCH 07/13] more edits from testing --- .../tests/ultra/unit/test_l1c_lookup_utils.py | 3 -- .../tests/ultra/unit/test_lookup_utils.py | 14 ++++----- .../tests/ultra/unit/test_ultra_l1b.py | 31 ++++++++++++------- imap_processing/ultra/l1b/extendedspin.py | 12 ++++--- imap_processing/ultra/l1b/lookup_utils.py | 15 +++++++-- imap_processing/ultra/l1b/ultra_l1b.py | 7 +++-- imap_processing/ultra/l1c/ultra_l1c.py | 2 +- 7 files changed, 51 insertions(+), 33 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py index 2546ddf150..d2149d4ea2 100644 --- a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from imap_processing import imap_module_directory from imap_processing.ultra.l1c.l1c_lookup_utils import ( calculate_accepted_pixels, get_scattering_thresholds_for_energy, @@ -13,8 +12,6 @@ mask_below_fwhm_scattering_threshold, ) -TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1" - @pytest.mark.external_test_data def test_get_spacecraft_pointing_lookup_tables(ancillary_files): diff --git a/imap_processing/tests/ultra/unit/test_lookup_utils.py b/imap_processing/tests/ultra/unit/test_lookup_utils.py index 4b47908a4d..5702b250d4 100644 --- a/imap_processing/tests/ultra/unit/test_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_lookup_utils.py @@ -228,7 +228,7 @@ def test_get_de_product_name_no_repoint(): / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" } with mock.patch( - "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" ) as mock_read_csv: mock_read_csv.return_value = pd.DataFrame( { @@ -251,7 +251,7 @@ def test_get_de_product_name_multiple_products(): / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" } with mock.patch( - "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" ) as mock_read_csv: mock_read_csv.return_value = pd.DataFrame( { @@ -274,7 +274,7 @@ def test_get_de_product_name(): / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" } with mock.patch( - "imap_processing.ultra.l1c.l1c_lookup_utils.pd.read_csv" + "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" ) as mock_read_csv: mock_read_csv.return_value = pd.DataFrame( { @@ -303,9 +303,9 @@ def test_extended_config_class(): config_path = ( TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" ) - config = ExtendedSpinConfig.from_csv(config_path, 128) + config = ExtendedSpinConfig.from_csv(config_path, "repoint00128") assert config.priority == "p0" - assert config.calibration == "c1" - assert config.thresholds == {0: 96.5, 1: 80.5, 2: 8.5, 3: 10.5, 4: 3.5} - assert config.date == datetime.datetime(2026, 1, 22) + assert config.calibration == "c0" + assert config.thresholds == {0: 200, 1: 7.5, 2: 4.5, 3: 3.5, 4: 3.5} + assert config.date == datetime.datetime(2025, 11, 10) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index a6e6794b95..6fcbec06a7 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -159,7 +159,7 @@ def test_cdf_de( use_fake_spin_data_for_time(511000000, 511000000 + 86400 * 5) use_fake_repoint_data_for_time(np.arange(511000000, 511000000 + 86400 * 5, 86400)) - l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") assert ( l1b_de_dataset[0].attrs["Logical_source_description"] @@ -197,7 +197,7 @@ def test_cdf_de_flags( # Use repoint data that will NOT cover the event times to test flag setting use_fake_repoint_data_for_time(np.arange(0, +86400 * 5, 86400)) - l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") # All valid events should be flagged as DURINGREPOINT since the repoint data does # not cover any of the event times valid_events = l1b_de_dataset[0]["event_times"] != FILLVAL_FLOAT32 @@ -220,6 +220,7 @@ def test_ultra_l1b_extendedspin( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -228,10 +229,10 @@ def test_ultra_l1b_extendedspin( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH + "l1b-45sensor-extendedspin-config": TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") assert len(l1b_extendedspin_dataset) == 1 assert ( @@ -255,6 +256,7 @@ def test_cdf_extendedspin( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -263,10 +265,10 @@ def test_cdf_extendedspin( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH + "l1b-45sensor-extendedspin-config": TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") """Tests that CDF file is created and contains same attributes as xarray.""" l1b_extendedspin_dataset[0].attrs["Data_version"] = "999" l1b_extendedspin_dataset[0].attrs["Repointing"] = "repoint99999" @@ -295,6 +297,7 @@ def test_cdf_goodtimes( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -303,14 +306,15 @@ def test_cdf_goodtimes( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH + "l1b-45sensor-extendedspin-config": TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") goodtimes_dataset = ultra_l1b( {"imap_ultra_l1b_45sensor-extendedspin": l1b_extendedspin_dataset[0]}, ancillary_files, + "repoint99999", ) goodtimes_dataset[0].attrs["Data_version"] = "999" goodtimes_dataset[0].attrs["Repointing"] = "repoint99999" @@ -339,6 +343,7 @@ def test_cdf_badtimes( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -347,15 +352,16 @@ def test_cdf_badtimes( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH + "l1b-45sensor-extendedspin-config": TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") ancillary_files = {} goodtimes_dataset = ultra_l1b( {"imap_ultra_l1b_45sensor-extendedspin": l1b_extendedspin_dataset[0]}, ancillary_files, + "repoint99999", ) l1b_badtimes_dataset = ultra_l1b( @@ -364,6 +370,7 @@ def test_cdf_badtimes( "imap_ultra_l1b_45sensor-goodtimes": goodtimes_dataset[0], }, ancillary_files, + "repoint99999", ) l1b_badtimes_dataset[0].attrs["Data_version"] = "999" l1b_badtimes_dataset[0].attrs["Repointing"] = "repoint99999" @@ -385,7 +392,7 @@ def test_ultra_l1b_error(mock_data_l1a_rates_dict): with pytest.raises( ValueError, match="Data dictionary does not contain the expected keys." ): - ultra_l1b(mock_data_l1a_rates_dict, ancillary_files) + ultra_l1b(mock_data_l1a_rates_dict, ancillary_files, "repoint99999") @pytest.mark.external_test_data @@ -410,7 +417,7 @@ def test_ultra_l1b_priority_de( data_dict["imap_ultra_l1a_45sensor-priority-1-de"] = de_dataset data_dict[aux_dataset.attrs["Logical_source"]] = aux_dataset - l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") assert l1b_de_dataset[0] assert ( diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index a6fef51f81..b69348f422 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -34,7 +34,7 @@ def calculate_extendedspin( dict_datasets: dict[str, xr.Dataset], - de_dataset: dict[str, xr.Dataset], + de_datasets: dict[str, xr.Dataset], extendedspin_conf: ExtendedSpinConfig, name: str, instrument_id: int, @@ -46,8 +46,9 @@ def calculate_extendedspin( ---------- dict_datasets : dict Dictionary containing all the datasets. - de_dataset : xarray.Dataset - Dataset containing the direct event data. + de_datasets : dict[str, xarray.Dataset] + Dictionary of direct event datasets keyed by priority, e.g. "p0" for raw + de and "p1" for priority 1 de. extendedspin_conf : ExtendedSpinConfig Configurations for culling spins. name : str @@ -63,6 +64,9 @@ def calculate_extendedspin( aux_dataset = dict_datasets[f"imap_ultra_l1a_{instrument_id}sensor-aux"] rates_dataset = dict_datasets[f"imap_ultra_l1a_{instrument_id}sensor-rates"] status_dataset = dict_datasets[f"imap_ultra_l1b_{instrument_id}sensor-status"] + # Use the raw de for the spin level quantities since it contains all events. + # The energy dependent culling selects its de dataset per energy range. + de_dataset = de_datasets["p0"] extendedspin_dict = {} rates_qf, spin, energy_bin_geometric_mean, n_sigma_per_energy = flag_rates( @@ -101,7 +105,7 @@ def calculate_extendedspin( # Get valid event counts per energy range and spin bin, shared across all of # the culling steps below. de_counts_summary = get_valid_de_count_summary( - de_dataset, + de_datasets, energy_ranges, spin_tbin_edges, extendedspin_conf, diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 09eeab2b33..a71abb7b80 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -717,14 +717,23 @@ def from_csv(cls, config_file_path: str, repointing: str) -> "ExtendedSpinConfig """ repoint_int = int(repointing.replace("repoint", "")) df = pd.read_csv(config_file_path) - # Get the first row where the pointing number is greater or equal - # to the pointing number in the config file. - filtered_df = df[(df["pointing"] >= repoint_int)] + # Each row applies from its pointing number up to (but not including) the + # next row's pointing number. The last row applies to all later pointings. + pointings = np.append(df["pointing"].values, np.inf) + filtered_df = df[ + ((repoint_int < pointings[1:]) & (repoint_int >= pointings[:-1])) + ] if filtered_df.empty: raise ValueError( f"The ancillary file: {config_file_path} contains no " f"configurations for {repointing}" ) + if filtered_df.shape[0] > 1: + raise ValueError( + f"The ancillary file: {config_file_path} contains multiple " + f"configurations for {repointing}. It should only contain one" + f" per pointing." + ) config = filtered_df.iloc[0] # Get the column names that contain "cullThresh" thresh_colnames = [col for col in df.columns if "cullThresh" in col] diff --git a/imap_processing/ultra/l1b/ultra_l1b.py b/imap_processing/ultra/l1b/ultra_l1b.py index c15ff00a3b..f535674f9e 100644 --- a/imap_processing/ultra/l1b/ultra_l1b.py +++ b/imap_processing/ultra/l1b/ultra_l1b.py @@ -70,9 +70,10 @@ def ultra_l1b( ) output_datasets.append(de_dataset) # L1b extended data will be created if L1a hk, rates, - # aux, params, and l1b de data are available + # aux, params, and l1b de (raw and priority 1) data are available elif ( f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict + and f"imap_ultra_l1b_{instrument_id}sensor-priority-1-de" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict @@ -81,8 +82,8 @@ def ultra_l1b( # Create dictionary of the de datasets # For now, only pass in priority 1 and raw de datasets. de_datasets = { - "p0": data_dict[f"imap_ultra_l1a_{instrument_id}sensor-de"], - "p1": data_dict[f"imap_ultra_l1a_{instrument_id}sensor-priority-1-de"], + "p0": data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], + "p1": data_dict[f"imap_ultra_l1b_{instrument_id}sensor-priority-1-de"], } # Get the extended spin config from ancillary files. extended_spin_config_anc = ancillary_files[ diff --git a/imap_processing/ultra/l1c/ultra_l1c.py b/imap_processing/ultra/l1c/ultra_l1c.py index 7fe259e66e..9fb4c386a9 100644 --- a/imap_processing/ultra/l1c/ultra_l1c.py +++ b/imap_processing/ultra/l1c/ultra_l1c.py @@ -50,7 +50,7 @@ def ultra_l1c( # Determine which l1b de product to use in calculating the l1c products. # Will be either the raw de product or a priority 1-4 de product. de_product_desc = get_de_product_name( - repoint, instrument_id, "l1c", ancillary_files + repoint, instrument_id, ancillary_files ) if de_product_desc not in data_dict: raise ValueError( From 368954fda4b50d058e231cd713a13827151e04cf Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Tue, 29 Sep 2026 15:17:14 -0600 Subject: [PATCH 08/13] add descriptors of class attrs --- imap_processing/ultra/l1b/lookup_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index a71abb7b80..622cad50f8 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -691,10 +691,10 @@ def get_de_product_name(repoint: str, sensor: int, ancillary_files: dict) -> str class ExtendedSpinConfig: """Pointing dependent l1b culling configurations.""" - thresholds: dict[int, float] - date: datetime.datetime - priority: str - calibration: str + thresholds: dict[int, float] # energy thresholds for culling + date: datetime.datetime # Date when configuration changed + priority: str # Which de product to use priority 1-4 de or raw de. e.g. p0-p4 + calibration: str # Calibration label @classmethod def from_csv(cls, config_file_path: str, repointing: str) -> "ExtendedSpinConfig": From 920a53bbc68f0dfe3bd48c868dde8bb9e4ff0862 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Wed, 30 Sep 2026 14:57:31 -0600 Subject: [PATCH 09/13] add bobs new changes to config file --- .../tests/ultra/unit/test_lookup_utils.py | 21 ++++++++++++++++++- imap_processing/ultra/l1b/lookup_utils.py | 16 ++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_lookup_utils.py b/imap_processing/tests/ultra/unit/test_lookup_utils.py index 5702b250d4..bd2a5c636b 100644 --- a/imap_processing/tests/ultra/unit/test_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_lookup_utils.py @@ -307,5 +307,24 @@ def test_extended_config_class(): assert config.priority == "p0" assert config.calibration == "c0" - assert config.thresholds == {0: 200, 1: 7.5, 2: 4.5, 3: 3.5, 4: 3.5} + np.testing.assert_array_equal( + config.energy_thresholds, np.array([200, 7.5, 4.5, 3.5, 3.5, 3.5]) + ) assert config.date == datetime.datetime(2025, 11, 10) + assert config.voltage_threshold == 3400 + + +def test_extended_config_class_last_repoint_range(): + """Tests the ExtendedSpinConfig class.""" + config_path = ( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" + ) + config = ExtendedSpinConfig.from_csv(config_path, "repoint00383") + + assert config.priority == "p1" + assert config.calibration == "c4" + np.testing.assert_array_equal( + config.energy_thresholds, np.array([117.5, 73.5, 2.5, 26.5, 8.5, 8.5]) + ) + assert config.date == datetime.datetime(2026, 8, 11) + assert config.voltage_threshold == 2900 diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 622cad50f8..278e98408a 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -691,7 +691,8 @@ def get_de_product_name(repoint: str, sensor: int, ancillary_files: dict) -> str class ExtendedSpinConfig: """Pointing dependent l1b culling configurations.""" - thresholds: dict[int, float] # energy thresholds for culling + energy_thresholds: np.ndarray # energy thresholds for culling + voltage_threshold: float # voltage threshold for culling date: datetime.datetime # Date when configuration changed priority: str # Which de product to use priority 1-4 de or raw de. e.g. p0-p4 calibration: str # Calibration label @@ -736,12 +737,19 @@ def from_csv(cls, config_file_path: str, repointing: str) -> "ExtendedSpinConfig ) config = filtered_df.iloc[0] # Get the column names that contain "cullThresh" - thresh_colnames = [col for col in df.columns if "cullThresh" in col] - thresholds = {int(col.split("_")[-1]): config[col] for col in thresh_colnames} + thresh_colnames = sorted( + [col for col in df.columns if "cullThresh" in col], + key=lambda threshold: int(threshold.split("_")[-1]), + ) + e_thresholds = np.array([config[col] for col in thresh_colnames]) + # The config provides thresholds for bins 0-4; bin 5 (> + # UltraConstands.MAX_ENERGY_THRESHOLD keV) reuses the bin 4 threshold + e_thresholds = np.append(e_thresholds, e_thresholds[-1]) date = datetime.datetime.strptime(config["date"], "%m/%d/%y") return cls( - thresholds=thresholds, + energy_thresholds=e_thresholds, + voltage_threshold=config["deflector_Vthresh"], date=date, priority=config["pri_config"], calibration=config["cal_config"], From c066e7e5188af4b942bffb41dbd94d21c6423bc7 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Wed, 30 Sep 2026 15:30:04 -0600 Subject: [PATCH 10/13] new anc files --- ...ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv | 6 ++++++ ...ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv create mode 100644 imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv new file mode 100644 index 0000000000..0d19e11199 --- /dev/null +++ b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv @@ -0,0 +1,6 @@ +pointing,date,pri_config,cal_config,deflector_Vthresh,cullThresh_0,cullThresh_1,cullThresh_2,cullThresh_3,cullThresh_4 +44,11/10/25,p0,c0,3400,200,7.5,4.5,3.5,3.5 +134,1/22/26,p0,c1,3400,96.5,80.5,8.5,10.5,3.5 +233,4/30/26,p1,c2,3400,200,200,35.5,14.5,8.5 +282,6/17/26,p1,c3,3400,200,200,11.5,26.5,2.5 +337,8/11/26,p1,c4,2900,117.5,73.5,2.5,26.5,8.5 \ No newline at end of file diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv new file mode 100644 index 0000000000..b824c5c3f0 --- /dev/null +++ b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv @@ -0,0 +1,6 @@ +pointing,date,pri_config,cal_config,deflector_Vthresh,cullThresh_0,cullThresh_1,cullThresh_2,cullThresh_3,cullThresh_4 +44,11/10/25,p0,c0,3400,9.5,13.5,4.5,4.5,0.5 +128,1/16/26,p0,c1,3400,200,118.5,78.5,12.5,5.5 +238,5/4/26,p1,c2,3400,200,200,21.5,6.5,5.5 +282,6/17/26,p1,c3,3400,81.5,49.5,15.5,16.5,9.5 +337,8/11/26,p1,c4,3400,200,97.5,82.5,66.5,8.5 \ No newline at end of file From 7c2c2c6d02cbf0b433b5f745a47f4652ef8c1441 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 1 Oct 2026 09:13:43 -0600 Subject: [PATCH 11/13] fix broken tests --- imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 7edd318725..4fe48be0fc 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -610,7 +610,7 @@ def test_get_valid_de_count_summary_valid_events(): } ) spin_config = ExtendedSpinConfig( - thresholds={}, + energy_thresholds={}, date=datetime.datetime(2025, 10, 1), priority="p0", calibration="", @@ -662,7 +662,7 @@ def test_get_valid_de_count_summary_ultra45(mock_spkezr): } ) spin_config = ExtendedSpinConfig( - thresholds={}, + energy_thresholds={}, date=datetime.datetime(2025, 10, 1), priority="p0", calibration="", From 3b6203894f024aa7dbc6a2571f7fc08d1a9bda0b Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 1 Oct 2026 10:28:38 -0600 Subject: [PATCH 12/13] fix tests --- .../ultra/unit/test_ultra_l1b_culling.py | 56 +++++-------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 4fe48be0fc..56a5bf456b 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -1,6 +1,5 @@ """Tests Culling for ULTRA L1b.""" -import datetime from unittest import mock import numpy as np @@ -50,6 +49,11 @@ TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1" +REPOINT_47_SPIN_CONFIG = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", +) + @pytest.fixture def setup_repoint_47_data(): @@ -75,10 +79,6 @@ def setup_repoint_47_data(): ) de_datasets = {"p0": de_ds, "p1": de_ds} - spin_config = ExtendedSpinConfig.from_csv( - TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", - "repoint00047", - ) # Get the energy ranges energy_ranges = get_binned_energy_ranges(build_energy_bins()[0]) @@ -87,7 +87,7 @@ def setup_repoint_47_data(): de_datasets, energy_ranges, spin_tbin_edges, - spin_config, + REPOINT_47_SPIN_CONFIG, 90, ) @@ -609,17 +609,11 @@ def test_get_valid_de_count_summary_valid_events(): "ebin": ("epoch", ebin), } ) - spin_config = ExtendedSpinConfig( - energy_thresholds={}, - date=datetime.datetime(2025, 10, 1), - priority="p0", - calibration="", - ) counts = get_valid_de_count_summary( {"p0": de_dataset, "p1": de_dataset}, energy_range_edges, spin_tbin_edges, - spin_config, + REPOINT_47_SPIN_CONFIG, 90, ) @@ -661,12 +655,7 @@ def test_get_valid_de_count_summary_ultra45(mock_spkezr): "ebin": ("epoch", np.full(len(energy), 10)), } ) - spin_config = ExtendedSpinConfig( - energy_thresholds={}, - date=datetime.datetime(2025, 10, 1), - priority="p0", - calibration="", - ) + # ensure that all events fail the earth angle check by setting a very large # keepout angle keepout_angle = np.radians(360) @@ -674,7 +663,7 @@ def test_get_valid_de_count_summary_ultra45(mock_spkezr): {"p0": de_dataset, "p1": de_dataset}, energy_range_edges, spin_tbin_edges, - spin_config, + REPOINT_47_SPIN_CONFIG, 45, keepout_angle, ) @@ -721,16 +710,11 @@ def test_flag_high_energy(): start=0, stop=len(energy) + 1, step=4 ) # create spin bins of 4 seconds de_datasets = {"p0": de_dataset, "p1": de_dataset} - spin_config = ExtendedSpinConfig.from_csv( - TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", - "repoint00047", - ) - de_counts_summary = get_valid_de_count_summary( de_datasets, energy_range_edges, spin_tbin_edges, - spin_config, + REPOINT_47_SPIN_CONFIG, 90, ) quality_flags = flag_high_energy( @@ -774,14 +758,11 @@ def test_validate_high_energy_cull(setup_repoint_47_data): TEST_PATH / "validate_high_energy_culling_results_repoint00047_v2.csv" ).to_numpy() de_datasets, _, spin_tbin_edges, _, _ = setup_repoint_47_data - spin_config = ExtendedSpinConfig.from_csv( - TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", - "repoint00047", - ) + # Get the energy ranges energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) de_counts_summary = get_valid_de_count_summary( - de_datasets, energy_ranges, spin_tbin_edges, spin_config + de_datasets, energy_ranges, spin_tbin_edges, REPOINT_47_SPIN_CONFIG ) high_energy_combined_spin_bin_radius = 3 e_flags = flag_high_energy( @@ -824,16 +805,11 @@ def test_flag_statistical_outliers(): start=0, stop=len(energy) + 1, step=spin_step ) # create spin bins of 7 seconds de_datasets = {"p0": de_dataset, "p1": de_dataset} - spin_config = ExtendedSpinConfig.from_csv( - TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", - "repoint00047", - ) - de_counts_summary = get_valid_de_count_summary( de_datasets, energy_range_edges, spin_tbin_edges, - spin_config, + REPOINT_47_SPIN_CONFIG, 90, ) quality_flags, convergence, iterations, std_diff = flag_statistical_outliers( @@ -888,16 +864,12 @@ def test_flag_statistical_outliers_invalid_events(): ) # create spin bins of 5 seconds mask = np.ones((len(energy_range_edges) - 1, len(spin_tbin_edges) - 1), dtype=bool) de_datasets = {"p0": de_dataset, "p1": de_dataset} - spin_config = ExtendedSpinConfig.from_csv( - TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", - "repoint00047", - ) de_counts_summary = get_valid_de_count_summary( de_datasets, energy_range_edges, spin_tbin_edges, - spin_config, + REPOINT_47_SPIN_CONFIG, 90, ) quality_flags, convergence, iterations, std_diff = flag_statistical_outliers( From 9015d847d9e55cf4fd8573b4ce40c9fee1d72b8e Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 1 Oct 2026 10:33:04 -0600 Subject: [PATCH 13/13] ignore extendedSpinConfig data class (docs) --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 7909a7acbd..1826dbbd0d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -116,6 +116,7 @@ (r"py:.*", r".*.hit.l0.utils.*"), (r"py:.*", r".*.hit.l0.data_classes.*"), (r"py:.*", r".*.hit.l1a.*"), + (r"py:.*", r".*ExtendedSpinConfig.*"), (r"py:.*", r".*InitVar*"), (r"py:.*", r".*.glows.utils.constants.TimeTuple.*"), (r"py:.*", r".*glows.utils.constants.DirectEvent.*"),