From 15d25fa2af70693ea6dd91c35a288acfae76c2f0 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Wed, 9 Sep 2026 13:13:53 -0600 Subject: [PATCH 1/9] fix logger --- imap_processing/ultra/l1b/ultra_l1b_extended.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 78da8bd24..6583cdb05 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -945,7 +945,7 @@ def get_spin_start_indices( # The time window spans from the first spin start to the end of the last spin. first_spin_start = spin_start_sec[0] # Define the end of the last spin as start time + max duration (15s) - last_spin_end = spin_start_sec[-1] + 15.0 + last_spin_end = spin_start_sec[-1] + 15.0 # TODO use actual duration missing_aux_data_mask = (de_event_met < first_spin_start) | ( de_event_met > last_spin_end ) @@ -954,8 +954,8 @@ def get_spin_start_indices( "Coarse MET time contains events outside aux_dataset time range " f"({first_spin_start} - {last_spin_end}). " f"Found min={de_event_met.min()}, max={de_event_met.max()}. " - f"Found {np.sum(missing_aux_data_mask)} events not covered by aux data. " - f" Trying to fill missing data using universal spin table." + f"Throwing away {np.sum(missing_aux_data_mask)} events not covered " + f"by aux data. " ) # Find the spin_start_sec that started directly before each event. start_inds = ( From 5221b934af3b2939b367168e861b48f220f6f00d Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 10 Sep 2026 11:29:09 -0600 Subject: [PATCH 2/9] update event_time calc --- imap_processing/quality_flags.py | 1 + .../ultra/unit/test_ultra_l1b_extended.py | 12 +++- imap_processing/ultra/l1b/de.py | 3 +- .../ultra/l1b/ultra_l1b_extended.py | 60 ++++++++++--------- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/imap_processing/quality_flags.py b/imap_processing/quality_flags.py index 11d13389b..b6631380c 100644 --- a/imap_processing/quality_flags.py +++ b/imap_processing/quality_flags.py @@ -47,6 +47,7 @@ class ImapDEOutliersUltraFlags(FlagNameMixin): INVALID_ENERGY = 2**3 # bit 3 DURINGREPOINT = 2**4 # bit 4 # event during a repointing BACKTOF = 2**5 # bit 5 # Back TOF outlier + AUXOUTLIER = 2**6 # bit 6 # Event time is outside of aux dataset range. class ImapHkUltraFlags(FlagNameMixin): diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index 75ebe6557..e8e173af0 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -11,6 +11,7 @@ from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import get_angular_profiles from imap_processing.ultra.l1b.ultra_l1b_extended import ( + FILLVAL_FLOAT32, CoinType, StartType, StopType, @@ -524,7 +525,7 @@ def test_get_eventtimes(test_fixture, aux_dataset): """Tests get_eventtimes function.""" df_filt, _, _, de_dataset = test_fixture - event_times, spin_start_times = get_event_times( + event_times, spin_start_times, _ = get_event_times( aux_dataset, de_dataset["shcoarse"].values, de_dataset["phase_angle"].values, @@ -600,7 +601,7 @@ def test_get_event_times_out_of_range( # set spin data that DOES cover the range of coarse_times use_fake_spin_data_for_time(min_time - 1000, min_time + 10000) # This should not raise an error. - event_times, spin_starts = get_event_times( + event_times, spin_starts, quality_flags = get_event_times( aux_dataset, coarse_times, de_dataset["phase_angle"].values, @@ -608,6 +609,13 @@ def test_get_event_times_out_of_range( assert event_times.shape == coarse_times.shape assert spin_starts.shape == coarse_times.shape + # The out-of-range event should be filled and flagged, not silently + # converted to a bogus (but finite) spice time. + assert event_times[0] == FILLVAL_FLOAT32 + assert spin_starts[0] == FILLVAL_FLOAT32 + assert quality_flags[0] == ImapDEOutliersUltraFlags.AUXOUTLIER.value + assert np.all(quality_flags[1:] == ImapDEOutliersUltraFlags.NONE.value) + @pytest.mark.external_test_data def test_interpolate_fwhm(ancillary_files): diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index 0122f756b..801b373a5 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -204,12 +204,13 @@ def calculate_de( start_type[valid_indices] = de_dataset["start_type"].data[valid_indices] spin_ds = get_spin_info(aux_dataset, de_dataset["shcoarse"].data) - (event_times[valid_mask], spin_starts[valid_mask]) = get_event_times( + (event_times[valid_mask], spin_starts[valid_mask], event_time_qf) = get_event_times( aux_dataset, de_dataset["shcoarse"].data[valid_mask], de_dataset["phase_angle"].data[valid_mask], spin_ds.isel(epoch=valid_mask), ) + quality_flags[valid_mask] |= event_time_qf de_dict["spin"] = spin_ds.spin_number.data de_dict["event_times"] = event_times.astype(np.float64) diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 6583cdb05..76a2b850b 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -13,7 +13,6 @@ from scipy.interpolate import LinearNDInterpolator, RegularGridInterpolator from imap_processing.quality_flags import ImapDEOutliersUltraFlags -from imap_processing.spice.spin import interpolate_spin_data from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import ( @@ -973,7 +972,7 @@ def get_event_times( de_event_met: NDArray, phase_angle: NDArray, spin_ds: xr.Dataset | None = None, -) -> tuple[NDArray, NDArray]: +) -> tuple[NDArray, NDArray, NDArray]: """ Get the event times, spin start times. @@ -998,23 +997,43 @@ def get_event_times( Event times in et. spin_start_times: numpy.ndarray Spin start times in et. + quality_flags : numpy.ndarray + Quality flags. Events with missing aux/spin data are flagged with + ``ImapDEOutliersUltraFlags.AUXOUTLIER``. """ # Get or compute spin info if spin_ds is None: spin_ds = get_spin_info(aux_dataset, de_event_met) # spin start with subsecond precision - spin_start_times = spin_ds.spin_starts + (spin_ds.spin_start_subs / 1000.0) + spin_start_times = (spin_ds.spin_starts + (spin_ds.spin_start_subs / 1000.0)).values # add the fractional spin offset - event_times = spin_start_times + (spin_ds.spin_duration / 1000.0) * ( + event_times = spin_start_times + (spin_ds.spin_duration.values / 1000.0) * ( phase_angle / 720.0 ) - return ( - ttj2000ns_to_et(met_to_ttj2000ns(event_times)), - ttj2000ns_to_et(met_to_ttj2000ns(spin_start_times)), + + # Flag and fill events with missing aux/spin data before doing the + # spice time conversion below. + missing_mask = np.isnan(event_times) | np.isnan(spin_start_times) + quality_flags = np.full( + event_times.shape, ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16 + ) + quality_flags[missing_mask] = ImapDEOutliersUltraFlags.AUXOUTLIER.value + + out_event_times = np.full(event_times.shape, FILLVAL_FLOAT32, dtype=np.float64) + out_spin_start_times = np.full( + spin_start_times.shape, FILLVAL_FLOAT32, dtype=np.float64 + ) + out_event_times[~missing_mask] = ttj2000ns_to_et( + met_to_ttj2000ns(event_times[~missing_mask]) + ) + out_spin_start_times[~missing_mask] = ttj2000ns_to_et( + met_to_ttj2000ns(spin_start_times[~missing_mask]) ) + return out_event_times, out_spin_start_times, quality_flags + def get_spin_info(aux_dataset: xr.Dataset, de_event_met: NDArray) -> xr.Dataset: """ @@ -1040,32 +1059,17 @@ def get_spin_info(aux_dataset: xr.Dataset, de_event_met: NDArray) -> xr.Dataset: spin_info_per_event = xr.Dataset() # Create dict of var name lookups var_names = { - "spin_number": ("spinnumber", "spin_number"), - "spin_duration": ("duration", "spin_period_sec"), - "spin_starts": ("timespinstart", "spin_start_sec_sclk"), - "spin_start_subs": ("timespinstartsub", "spin_start_subsec_sclk"), + "spin_number": "spinnumber", + "spin_duration": "duration", + "spin_starts": "timespinstart", + "spin_start_subs": "timespinstartsub", } - # If there is not enough aux data covering an event, query the universal - # spin table using the start time to fill in the missing data. - # This can happen for the first event if the aux data starts after the DE data. - spin_data = ( - interpolate_spin_data(de_event_met[missing_events]) - if np.any(missing_events) - else None - ) - for var, (aux_name, ut_name) in var_names.items(): - init_array = np.zeros_like(de_event_met, dtype=np.float64) - if np.any(missing_events) and spin_data is not None: - # Get data from universal table for events missing aux data - init_array[missing_events] = spin_data[ut_name].values - if ut_name == "spin_start_subsec_sclk": - # Convert from microseconds to milliseconds to match aux data units - init_array[missing_events] /= 1000.0 + for var, aux_name in var_names.items(): + init_array = np.full(de_event_met.shape, np.nan, dtype=np.float64) # Get data from aux dataset for the rest of the events init_array[~missing_events] = aux_dataset[aux_name].values[start_inds] spin_info_per_event[var] = (("epoch",), init_array) - return spin_info_per_event From e352e40de1e34f395401e77efd9401c1850110c8 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Mon, 21 Sep 2026 14:12:52 -0600 Subject: [PATCH 3/9] new flag for aux data outliers # Conflicts: # imap_processing/ultra/l1b/extendedspin.py --- imap_processing/ultra/l1b/de.py | 5 +++- imap_processing/ultra/l1b/extendedspin.py | 28 +++++++++++-------- .../ultra/l1b/quality_flag_filters.py | 1 + .../ultra/l1b/ultra_l1b_extended.py | 4 +-- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index 801b373a5..35be50f69 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -212,7 +212,10 @@ def calculate_de( ) quality_flags[valid_mask] |= event_time_qf - de_dict["spin"] = spin_ds.spin_number.data + spin_number = spin_ds.spin_number.data + spin_missing_mask = np.isnan(spin_number) + spin_number[spin_missing_mask] = FILLVAL_UINT32 + de_dict["spin"] = spin_number.astype(np.uint32) de_dict["event_times"] = event_times.astype(np.float64) # Pulse height ph_result = get_ph_tof_and_back_positions( diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index b03b88e0c..481bbb7a1 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -30,6 +30,7 @@ FILLVAL_UINT16 = 65535 FILLVAL_FLOAT32 = -1.0e31 +FILLVAL_UINT32 = 4294967295 def calculate_extendedspin( @@ -68,20 +69,25 @@ def calculate_extendedspin( # The energy dependent culling selects its de dataset per energy range. priority_1_de_dataset = de_datasets["p1"] + # Events with no aux/spin coverage (AUXOUTLIER, flagged in de.py) have a + # fill-valued "spin" that isn't a real spin number and must be excluded + # from per-spin binning to avoid using an invalid spin. + has_spin_mask = priority_1_de_dataset["spin"].values != FILLVAL_UINT32 + spin_number = priority_1_de_dataset["spin"].values[has_spin_mask] + de_energy = priority_1_de_dataset["energy"].values[has_spin_mask] + extendedspin_dict = {} rates_qf, spin, energy_bin_geometric_mean, n_sigma_per_energy = flag_rates( - priority_1_de_dataset["spin"].values, - priority_1_de_dataset["energy"].values, - ) - count_rates, _, _counts, _ = get_energy_histogram( - priority_1_de_dataset["spin"].values, priority_1_de_dataset["energy"].values + spin_number, + de_energy, ) + count_rates, _, _counts, _ = get_energy_histogram(spin_number, de_energy) attitude_qf, spin_rates, spin_period, spin_starttime = flag_attitude( - priority_1_de_dataset["spin"].values, aux_dataset + spin_number, aux_dataset ) # TODO: We will add to this later - hk_qf = flag_hk(priority_1_de_dataset["spin"].values) - inst_qf = flag_imap_instruments(priority_1_de_dataset["spin"].values) + hk_qf = flag_hk(spin_number) + inst_qf = flag_imap_instruments(spin_number) spin_bin_size = UltraConstants.SPIN_BIN_SIZE spin_tbin_edges = get_binned_spins_edges( @@ -156,9 +162,9 @@ def calculate_extendedspin( # Track rejected events in each spin based on # quality flags in de l1b data. rejected_counts = count_rejected_events_per_spin( - priority_1_de_dataset["spin"].values, - priority_1_de_dataset["quality_scattering"].values, - priority_1_de_dataset["quality_outliers"].values, + spin_number, + priority_1_de_dataset["quality_scattering"].values[has_spin_mask], + priority_1_de_dataset["quality_outliers"].values[has_spin_mask], ) # These will be the coordinates. diff --git a/imap_processing/ultra/l1b/quality_flag_filters.py b/imap_processing/ultra/l1b/quality_flag_filters.py index faa3921a5..5d34c97b2 100644 --- a/imap_processing/ultra/l1b/quality_flag_filters.py +++ b/imap_processing/ultra/l1b/quality_flag_filters.py @@ -34,6 +34,7 @@ ImapDEOutliersUltraFlags.DURINGREPOINT, ImapDEOutliersUltraFlags.COINPH, ImapDEOutliersUltraFlags.BACKTOF, + ImapDEOutliersUltraFlags.AUXOUTLIER, ], "quality_scattering": [ ImapDEScatteringUltraFlags.ABOVE_THRESHOLD, diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 76a2b850b..69df5035b 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -935,8 +935,8 @@ def get_spin_start_indices( start_inds : numpy.ndarray Spin start indices for each event. missing_aux_data_mask : numpy.ndarray - Boolean array indicating where there are events out of the aux data range. The - universal spin table should be used to fill in missing data for these events. + Boolean array indicating where there are events out of the aux data range. + These events are dropped/flagged rather than filled in. """ # Get Spin Start Time in seconds spin_start_sec = aux_dataset["timespinstart"].values From c9493ee9899053c048e447d1473469cc549b92f1 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Mon, 21 Sep 2026 14:46:10 -0600 Subject: [PATCH 4/9] new flag --- imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py | 4 ++-- imap_processing/ultra/constants.py | 2 ++ imap_processing/ultra/l1b/extendedspin.py | 2 +- imap_processing/ultra/l1b/ultra_l1b_extended.py | 6 ++++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index e8e173af0..e9876f4a3 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -609,8 +609,8 @@ def test_get_event_times_out_of_range( assert event_times.shape == coarse_times.shape assert spin_starts.shape == coarse_times.shape - # The out-of-range event should be filled and flagged, not silently - # converted to a bogus (but finite) spice time. + # Check events that dont have aux data coverage. These should be fill vals + # and the quality flag array should indicate an AUXOUTLIER flag. assert event_times[0] == FILLVAL_FLOAT32 assert spin_starts[0] == FILLVAL_FLOAT32 assert quality_flags[0] == ImapDEOutliersUltraFlags.AUXOUTLIER.value diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 5f432d805..7320c8888 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -43,6 +43,8 @@ class UltraConstants: SSD-specific correction to DMIN for time-of-flight normalization """ + NOMINAL_SPIN_PERIOD_SEC: float = 15.0 + D_SLIT_FOIL: float = 3.39 SLIT_Z: float = 44.89 YF_ESTIMATE_LEFT: float = 40.0 diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index 481bbb7a1..b202232f7 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -69,7 +69,7 @@ def calculate_extendedspin( # The energy dependent culling selects its de dataset per energy range. priority_1_de_dataset = de_datasets["p1"] - # Events with no aux/spin coverage (AUXOUTLIER, flagged in de.py) have a + # Events with no aux data coverage (AUXOUTLIER, flagged in de.py) have a # fill-valued "spin" that isn't a real spin number and must be excluded # from per-spin binning to avoid using an invalid spin. has_spin_mask = priority_1_de_dataset["spin"].values != FILLVAL_UINT32 diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 69df5035b..43015ea37 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -943,8 +943,10 @@ def get_spin_start_indices( # Check that all events fall within the aux dataset time range. # The time window spans from the first spin start to the end of the last spin. first_spin_start = spin_start_sec[0] - # Define the end of the last spin as start time + max duration (15s) - last_spin_end = spin_start_sec[-1] + 15.0 # TODO use actual duration + # Define the end of the last spin as start time + nominal spin duration + last_spin_end = ( + spin_start_sec[-1] + UltraConstants.NOMINAL_SPIN_PERIOD_SEC + ) # TODO ask ultra team missing_aux_data_mask = (de_event_met < first_spin_start) | ( de_event_met > last_spin_end ) From 0447e2ce816296a7561224cfb7feba6495a72045 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Mon, 21 Sep 2026 15:24:47 -0600 Subject: [PATCH 5/9] add check # Conflicts: # imap_processing/_version.py --- imap_processing/ultra/l1b/extendedspin.py | 7 +++++++ imap_processing/ultra/l1c/ultra_l1c_pset_bins.py | 1 + 2 files changed, 8 insertions(+) diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index b202232f7..d434557e6 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -76,6 +76,13 @@ def calculate_extendedspin( spin_number = priority_1_de_dataset["spin"].values[has_spin_mask] de_energy = priority_1_de_dataset["energy"].values[has_spin_mask] + # check if there are no valid spins. + if spin_number.size == 0: + raise ValueError( + "All Spins are invalid. Please ensure that the l1a aux dataset " + "has the correct spin information." + ) + extendedspin_dict = {} rates_qf, spin, energy_bin_geometric_mean, n_sigma_per_energy = flag_rates( spin_number, diff --git a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py index cc33e0259..57dce2af6 100644 --- a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +++ b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py @@ -304,6 +304,7 @@ def get_deadtime_ratios_by_spin_phase( # Get timestamps at the start of each spin (sector 0) spin_start_indices = np.where(sector_indices == 0)[0] met_time = sectored_rates["shcoarse"].values[spin_start_indices] + print("any warnings?") spin_ds = get_spin_info(aux_dataset, met_time) # Repeat the spin duration for each of the 15 sectors. # Sectors are all within a spin so each one corresponds to the same spin From 517e586f602d8d0d9dc031811b8cdd3084a0ed86 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Tue, 22 Sep 2026 13:46:01 -0600 Subject: [PATCH 6/9] raise value error at the source when there is no aux data covering the de events --- .../tests/ultra/unit/test_ultra_l1c_pset_bins.py | 3 ++- imap_processing/ultra/l1b/ultra_l1b_extended.py | 15 +++++++++++++++ imap_processing/ultra/l1c/ultra_l1c_pset_bins.py | 1 - 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py index 9fedf82fa..edfadff9a 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py @@ -208,9 +208,10 @@ def test_get_deadtime_interpolator(use_fake_spin_data_for_time, aux_dataset): deadtime_ratios = xr.DataArray( np.random.uniform(0.1, 1.0, num_deadtimes), dims=["epoch"] ) + met_in_range = aux_dataset["timespinstart"].values[0] sectored_rates_ds = xr.Dataset( {"epoch": ("epoch", np.ones_like(deadtime_ratios))}, - {"shcoarse": ("epoch", np.ones_like(deadtime_ratios))}, + {"shcoarse": ("epoch", np.full_like(deadtime_ratios, met_in_range))}, ) with mock.patch( "imap_processing.ultra.l1c.ultra_l1c_pset_bins.get_deadtime_ratios", diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 43015ea37..610199f6a 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -1055,8 +1055,23 @@ def get_spin_info(aux_dataset: xr.Dataset, de_event_met: NDArray) -> xr.Dataset: ------- spin_info_per_event : xarray.Dataset Spin information for each event. + + Raises + ------ + ValueError + If none of the events fall within the aux dataset's time range. This + indicates a mismatched or missing aux dataset rather than a handful of + boundary events, so it is not safe to silently proceed. """ start_inds, missing_events = get_spin_start_indices(aux_dataset, de_event_met) + if de_event_met.size > 0 and np.all(missing_events): + raise ValueError( + "No events fall within the aux_dataset time range " + f"({aux_dataset['timespinstart'].values[0]} - " + f"{aux_dataset['timespinstart'].values[-1]}" + f" + {UltraConstants.NOMINAL_SPIN_PERIOD_SEC}). " + "Please check the aux dataset and direct event MET values." + ) # Initialize spin info dataset spin_info_per_event = xr.Dataset() # Create dict of var name lookups diff --git a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py index 57dce2af6..cc33e0259 100644 --- a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +++ b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py @@ -304,7 +304,6 @@ def get_deadtime_ratios_by_spin_phase( # Get timestamps at the start of each spin (sector 0) spin_start_indices = np.where(sector_indices == 0)[0] met_time = sectored_rates["shcoarse"].values[spin_start_indices] - print("any warnings?") spin_ds = get_spin_info(aux_dataset, met_time) # Repeat the spin duration for each of the 15 sectors. # Sectors are all within a spin so each one corresponds to the same spin From 1dc9554ec91615c92404f40a30ba48d67914ff54 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 24 Sep 2026 09:34:58 -0600 Subject: [PATCH 7/9] move fillvals to constants.py --- .../tests/ultra/unit/test_ultra_l1b.py | 3 +- .../ultra/unit/test_ultra_l1b_extended.py | 5 +- imap_processing/ultra/constants.py | 6 ++ imap_processing/ultra/l1b/badtimes.py | 40 ++++----- imap_processing/ultra/l1b/de.py | 83 +++++++++++-------- imap_processing/ultra/l1b/extendedspin.py | 20 +++-- imap_processing/ultra/l1b/goodtimes.py | 58 +++++++------ .../ultra/l1b/ultra_l1b_extended.py | 36 ++++---- .../ultra/l1c/ultra_l1c_pset_bins.py | 1 - imap_processing/ultra/l2/ultra_l2.py | 10 +-- 10 files changed, 146 insertions(+), 116 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index 6fcbec06a..9df548847 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -8,7 +8,6 @@ from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.quality_flags import ImapDEOutliersUltraFlags from imap_processing.ultra.constants import UltraConstants -from imap_processing.ultra.l1b.de import FILLVAL_FLOAT32 from imap_processing.ultra.l1b.ultra_l1b import ultra_l1b from imap_processing.ultra.utils.ultra_l1_utils import create_dataset @@ -200,7 +199,7 @@ def test_cdf_de_flags( l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") # All valid events should be flagged as DURINGREPOINT since the repoint data does # not cover any of the event times - valid_events = l1b_de_dataset[0]["event_times"] != FILLVAL_FLOAT32 + valid_events = l1b_de_dataset[0]["event_times"] != UltraConstants.FILLVAL_FLOAT flags = l1b_de_dataset[0]["quality_outliers"].values[valid_events] assert np.all((flags & ImapDEOutliersUltraFlags.DURINGREPOINT.value) != 0) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index e9876f4a3..93c4f4fbd 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -11,7 +11,6 @@ from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import get_angular_profiles from imap_processing.ultra.l1b.ultra_l1b_extended import ( - FILLVAL_FLOAT32, CoinType, StartType, StopType, @@ -611,8 +610,8 @@ def test_get_event_times_out_of_range( # Check events that dont have aux data coverage. These should be fill vals # and the quality flag array should indicate an AUXOUTLIER flag. - assert event_times[0] == FILLVAL_FLOAT32 - assert spin_starts[0] == FILLVAL_FLOAT32 + assert event_times[0] == UltraConstants.FILLVAL_FLOAT + assert spin_starts[0] == UltraConstants.FILLVAL_FLOAT assert quality_flags[0] == ImapDEOutliersUltraFlags.AUXOUTLIER.value assert np.all(quality_flags[1:] == ImapDEOutliersUltraFlags.NONE.value) diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 7320c8888..2a70972dd 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -43,6 +43,12 @@ class UltraConstants: SSD-specific correction to DMIN for time-of-flight normalization """ + # Define fillvals + FILLVAL_UINT8 = 255 + FILLVAL_UINT16 = 65535 + FILLVAL_UINT32 = 4294967295 + FILLVAL_FLOAT = -1.0e31 + NOMINAL_SPIN_PERIOD_SEC: float = 15.0 D_SLIT_FOIL: float = 3.39 diff --git a/imap_processing/ultra/l1b/badtimes.py b/imap_processing/ultra/l1b/badtimes.py index c78520a49..3fe1ca4d6 100644 --- a/imap_processing/ultra/l1b/badtimes.py +++ b/imap_processing/ultra/l1b/badtimes.py @@ -4,13 +4,9 @@ import xarray as xr from numpy.typing import NDArray +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict -FILLVAL_UINT16 = 65535 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_FLOAT64 = -1.0e31 -FILLVAL_UINT32 = 4294967295 - def calculate_badtimes( extendedspin_dataset: xr.Dataset, @@ -46,54 +42,60 @@ def calculate_badtimes( if badtimes_dataset["spin_number"].size == 0: badtimes_dataset = badtimes_dataset.drop_dims("spin_number") - badtimes_dataset = badtimes_dataset.expand_dims(spin_number=[FILLVAL_UINT32]) + badtimes_dataset = badtimes_dataset.expand_dims( + spin_number=[UltraConstants.FILLVAL_UINT32] + ) badtimes_dataset["spin_start_time"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) badtimes_dataset["spin_period"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) badtimes_dataset["spin_rate"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) badtimes_dataset["start_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) badtimes_dataset["stop_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) badtimes_dataset["coin_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) badtimes_dataset["rejected_events_per_spin"] = xr.DataArray( - np.array([FILLVAL_UINT32], dtype="uint32"), + np.array([UltraConstants.FILLVAL_UINT32], dtype="uint32"), dims=["spin_number"], ) badtimes_dataset["quality_attitude"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) badtimes_dataset["quality_hk"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) badtimes_dataset["quality_instruments"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) badtimes_dataset["quality_ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_UINT16, dtype="uint16"), + np.full((n_bins, 1), UltraConstants.FILLVAL_UINT16, dtype="uint16"), ) badtimes_dataset["ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT64, dtype="float64"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float64"), ) badtimes_dataset["ena_rates_threshold"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT32, dtype="float32"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float32"), ) return badtimes_dataset diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index 35be50f69..9c7a9bff1 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -13,6 +13,7 @@ from imap_processing.spice.time import ( et_to_met, ) +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import get_geometric_factor from imap_processing.ultra.l1b.ultra_l1b_annotated import ( get_annotated_particle_velocity, @@ -45,10 +46,6 @@ ) from imap_processing.ultra.utils.ultra_l1_utils import create_dataset -FILLVAL_UINT8 = 255 -FILLVAL_UINT32 = 4294967295 -FILLVAL_FLOAT32 = -1.0e31 - def calculate_de( de_dataset: xr.Dataset, aux_dataset: xr.Dataset, name: str, ancillary_files: dict @@ -106,7 +103,7 @@ def calculate_de( for key, dataset_key in zip(keys, dataset_keys, strict=False) } ) - valid_mask = de_dataset["start_type"].data != FILLVAL_UINT8 + valid_mask = de_dataset["start_type"].data != UltraConstants.FILLVAL_UINT8 ph_mask = np.isin( de_dataset["stop_type"].data, [StopType.Top.value, StopType.Bottom.value] ) @@ -117,72 +114,86 @@ def calculate_de( ssd_indices = np.nonzero(valid_mask & ssd_mask)[0] # Instantiate arrays xf: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) yf: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) xb: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) yb: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) xc: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + d: np.ndarray = np.full( + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) + r: np.ndarray = np.full( + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) - d: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) - r: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) phi: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) theta: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) tof: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) etof: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) ctof: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) tof_energy: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) magnitude_v: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) energy: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + e_bin: np.ndarray = np.full( + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) - e_bin: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) e_bin_l1a: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) species_bin: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) t2: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) event_times: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64 ) spin_starts: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64 ) shape = (len(de_dataset["epoch"]), 3) - sc_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - sc_dps_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - helio_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - velocities: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - v_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - r_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + sc_velocity: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + sc_dps_velocity: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + helio_velocity: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + velocities: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + v_hat: np.ndarray = np.full(shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32) + r_hat: np.ndarray = np.full(shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32) start_type: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) quality_flags = np.full( de_dataset["epoch"].shape, ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16 @@ -214,7 +225,7 @@ def calculate_de( spin_number = spin_ds.spin_number.data spin_missing_mask = np.isnan(spin_number) - spin_number[spin_missing_mask] = FILLVAL_UINT32 + spin_number[spin_missing_mask] = UltraConstants.FILLVAL_UINT32 de_dict["spin"] = spin_number.astype(np.uint32) de_dict["event_times"] = event_times.astype(np.float64) # Pulse height @@ -361,7 +372,7 @@ def calculate_de( de_dict["tof_energy"] = tof_energy de_dict["energy"] = energy de_dict["computed_ebin"] = e_bin - valid_ebin = de_dataset["bin"].values != FILLVAL_UINT32 + valid_ebin = de_dataset["bin"].values != UltraConstants.FILLVAL_UINT32 e_bin_l1a[valid_ebin] = de_dataset["bin"].values[valid_ebin] de_dict["ebin"] = e_bin_l1a de_dict["species"] = species_bin @@ -370,7 +381,7 @@ def calculate_de( ultra_frame = getattr(SpiceFrame, f"IMAP_ULTRA_{sensor}") # Account for counts=0 (event times have FILL value) - valid_events = (event_times != FILLVAL_FLOAT32).copy() + valid_events = (event_times != UltraConstants.FILLVAL_FLOAT).copy() if repoint_id is not None: # Check all valid event times to see which are in the pointing in_pointing = calculate_events_in_pointing( diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index d434557e6..59b94c4e6 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -28,10 +28,6 @@ from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins from imap_processing.ultra.utils.ultra_l1_utils import create_dataset -FILLVAL_UINT16 = 65535 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_UINT32 = 4294967295 - def calculate_extendedspin( dict_datasets: dict[str, xr.Dataset], @@ -190,9 +186,15 @@ def calculate_extendedspin( # Validate that the spin values match valid = (idx < pulses.unique_spins.size) & (pulses.unique_spins[idx] == spin) - start_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) - stop_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) - coin_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) + start_per_spin: np.ndarray = np.full( + len(spin), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + stop_per_spin: np.ndarray = np.full( + len(spin), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + coin_per_spin: np.ndarray = np.full( + len(spin), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) # Fill only the valid ones start_per_spin[valid] = pulses.start_per_spin[idx[valid]] @@ -266,7 +268,9 @@ def calculate_extendedspin( # energy ranges. Set the length to be the max number of energy bins we expect to # use for culling. The number of edges is one more than the number of bins (17). ranges: np.ndarray = np.full( - (UltraConstants.MAX_ENERGY_RANGE_EDGES,), FILLVAL_FLOAT32, dtype=np.float32 + (UltraConstants.MAX_ENERGY_RANGE_EDGES,), + UltraConstants.FILLVAL_FLOAT, + dtype=np.float32, ) ranges[: len(energy_ranges)] = energy_ranges extendedspin_dict["energy_range_edges"] = ranges diff --git a/imap_processing/ultra/l1b/goodtimes.py b/imap_processing/ultra/l1b/goodtimes.py index 5a57191e5..7daaf71ff 100644 --- a/imap_processing/ultra/l1b/goodtimes.py +++ b/imap_processing/ultra/l1b/goodtimes.py @@ -3,14 +3,10 @@ import numpy as np import xarray as xr +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.quality_flag_filters import SPIN_QUALITY_FLAG_FILTERS from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict -FILLVAL_UINT16 = 65535 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_FLOAT64 = -1.0e31 -FILLVAL_UINT32 = 4294967295 - def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Dataset: """ @@ -57,72 +53,84 @@ def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Datas goodtimes_dataset = create_dataset(data_dict, name, "l1b") if goodtimes_dataset["spin_number"].size == 0: goodtimes_dataset = goodtimes_dataset.drop_dims("spin_number") - goodtimes_dataset = goodtimes_dataset.expand_dims(spin_number=[FILLVAL_UINT32]) + goodtimes_dataset = goodtimes_dataset.expand_dims( + spin_number=[UltraConstants.FILLVAL_UINT32] + ) goodtimes_dataset["spin_start_time"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) goodtimes_dataset["spin_period"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) goodtimes_dataset["spin_rate"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) goodtimes_dataset["start_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) goodtimes_dataset["stop_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) goodtimes_dataset["coin_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) goodtimes_dataset["rejected_events_per_spin"] = xr.DataArray( - np.array([FILLVAL_UINT32], dtype="uint32"), + np.array([UltraConstants.FILLVAL_UINT32], dtype="uint32"), dims=["spin_number"], ) goodtimes_dataset["quality_attitude"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_low_voltage"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_high_energy"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_upstream_ion_1"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_upstream_ion_2"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_spectral"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_statistics"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_hk"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) goodtimes_dataset["quality_instruments"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) goodtimes_dataset["quality_ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_UINT16, dtype="uint16"), + np.full((n_bins, 1), UltraConstants.FILLVAL_UINT16, dtype="uint16"), ) goodtimes_dataset["ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT64, dtype="float64"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float64"), ) goodtimes_dataset["ena_rates_threshold"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT32, dtype="float32"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float32"), ) return goodtimes_dataset diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 610199f6a..932841339 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -29,10 +29,6 @@ logger = logging.getLogger(__name__) -FILLVAL_UINT8 = 255 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_FLOAT64 = -1.0e31 - class StartType(Enum): """Start Type: 1=Left, 2=Right.""" @@ -544,9 +540,9 @@ def get_de_velocity( v_y = -delta_v[:, 1] / tof * 1e3 v_z = -delta_v[:, 2] / tof * 1e3 - v_x[tof < 0] = FILLVAL_FLOAT32 # used as fillvals - v_y[tof < 0] = FILLVAL_FLOAT32 - v_z[tof < 0] = FILLVAL_FLOAT32 + v_x[tof < 0] = UltraConstants.FILLVAL_FLOAT # used as fillvals + v_y[tof < 0] = UltraConstants.FILLVAL_FLOAT + v_z[tof < 0] = UltraConstants.FILLVAL_FLOAT velocities = np.vstack((v_x, v_y, v_z)).T @@ -645,7 +641,7 @@ def get_de_energy_kev( valid_velocity = np.isfinite(v2) valid_mask = index_hydrogen & valid_velocity - energy = np.full_like(v2, FILLVAL_FLOAT32) + energy = np.full_like(v2, UltraConstants.FILLVAL_FLOAT) # TODO: we will calculate the energies of the different species here. # 1/2 mv^2 in Joules, convert to keV @@ -1023,9 +1019,11 @@ def get_event_times( ) quality_flags[missing_mask] = ImapDEOutliersUltraFlags.AUXOUTLIER.value - out_event_times = np.full(event_times.shape, FILLVAL_FLOAT32, dtype=np.float64) + out_event_times = np.full( + event_times.shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) out_spin_start_times = np.full( - spin_start_times.shape, FILLVAL_FLOAT32, dtype=np.float64 + spin_start_times.shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float64 ) out_event_times[~missing_mask] = ttj2000ns_to_et( met_to_ttj2000ns(event_times[~missing_mask]) @@ -1130,8 +1128,10 @@ def interpolate_fwhm( phi_vals = interp_phi((energy, phi_inst)) theta_vals = interp_theta((energy, theta_inst)) - phi_interp = np.where(np.isnan(phi_vals), FILLVAL_FLOAT32, phi_vals) - theta_interp = np.where(np.isnan(theta_vals), FILLVAL_FLOAT32, theta_vals) + phi_interp = np.where(np.isnan(phi_vals), UltraConstants.FILLVAL_FLOAT, phi_vals) + theta_interp = np.where( + np.isnan(theta_vals), UltraConstants.FILLVAL_FLOAT, theta_vals + ) return phi_interp, theta_interp @@ -1169,8 +1169,10 @@ def get_fwhm( theta_interp : NDArray Interpolated theta FWHM values. """ - phi_interp = np.full_like(phi_inst, FILLVAL_FLOAT64, dtype=np.float64) - theta_interp = np.full_like(theta_inst, FILLVAL_FLOAT64, dtype=np.float64) + phi_interp = np.full_like(phi_inst, UltraConstants.FILLVAL_FLOAT, dtype=np.float64) + theta_interp = np.full_like( + theta_inst, UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) lt_table = get_angular_profiles("left", sensor, ancillary_files) rt_table = get_angular_profiles("right", sensor, ancillary_files) @@ -1234,7 +1236,7 @@ def get_efficiency_interpolator( (theta_vals, phi_vals, energy_vals), efficiency_grid, bounds_error=False, - fill_value=FILLVAL_FLOAT32, + fill_value=UltraConstants.FILLVAL_FLOAT, ) return interpolator, theta_min_max, phi_min_max, energy_min_max @@ -1323,7 +1325,7 @@ def determine_ebin_pulse_height( # PH event TOF normalization to Z axis ctof, _ = get_ctof(tof, path_length, type="PH") - ebins = np.full(path_length.shape, FILLVAL_UINT8, dtype=np.uint8) + ebins = np.full(path_length.shape, UltraConstants.FILLVAL_UINT8, dtype=np.uint8) valid = backtofvalid & coinphvalid ebins[valid] = get_ebins( "l1b-tofxph", energy[valid], ctof[valid], ebins[valid], ancillary_files @@ -1376,7 +1378,7 @@ def determine_ebin_ssd( # SSD event TOF normalization to Z axis ctof, _ = get_ctof(tof, path_length, type="SSD") - ebins = np.full(path_length.shape, FILLVAL_UINT8, dtype=np.uint8) + ebins = np.full(path_length.shape, UltraConstants.FILLVAL_UINT8, dtype=np.uint8) steep_path_length = get_image_params("PathSteepThresh", sensor, ancillary_files) medium_path_length = get_image_params("PathMediumThresh", sensor, ancillary_files) diff --git a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py index cc33e0259..67cbd5798 100644 --- a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +++ b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py @@ -34,7 +34,6 @@ ) # TODO: add species binning. -FILLVAL_FLOAT32 = -1.0e31 logger = logging.getLogger(__name__) diff --git a/imap_processing/ultra/l2/ultra_l2.py b/imap_processing/ultra/l2/ultra_l2.py index 2080ad6aa..4cc2325e6 100644 --- a/imap_processing/ultra/l2/ultra_l2.py +++ b/imap_processing/ultra/l2/ultra_l2.py @@ -24,10 +24,7 @@ from imap_processing.quality_flags import ImapPSETUltraFlags from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins -from imap_processing.ultra.l1c.ultra_l1c_pset_bins import ( - FILLVAL_FLOAT32, - get_energy_delta_minus_plus, -) +from imap_processing.ultra.l1c.ultra_l1c_pset_bins import get_energy_delta_minus_plus logger = logging.getLogger(__name__) @@ -252,7 +249,10 @@ def bin_pset_energy_bins( ) # Count number of pixels non_zero_pixels_per_group = ( - ((pset[vars_to_average] != 0) & (pset[vars_to_average] != FILLVAL_FLOAT32)) + ( + (pset[vars_to_average] != 0) + & (pset[vars_to_average] != UltraConstants.FILLVAL_FLOAT) + ) .astype(int) .groupby("energy_bin_index") .sum() From c0a833473334d3482f08a12eb007d7f108dd57cb Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Mon, 5 Oct 2026 08:42:25 -0600 Subject: [PATCH 8/9] fix constant reference --- imap_processing/ultra/l1b/extendedspin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index 59b94c4e6..82211fc2f 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -68,7 +68,9 @@ def calculate_extendedspin( # Events with no aux data coverage (AUXOUTLIER, flagged in de.py) have a # fill-valued "spin" that isn't a real spin number and must be excluded # from per-spin binning to avoid using an invalid spin. - has_spin_mask = priority_1_de_dataset["spin"].values != FILLVAL_UINT32 + has_spin_mask = ( + priority_1_de_dataset["spin"].values != UltraConstants.FILLVAL_UINT32 + ) spin_number = priority_1_de_dataset["spin"].values[has_spin_mask] de_energy = priority_1_de_dataset["energy"].values[has_spin_mask] From d6f05dbfd2e20a5d04e31adcc5e1719d02eaf36e Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Mon, 5 Oct 2026 08:49:37 -0600 Subject: [PATCH 9/9] remove unused constant --- imap_processing/ultra/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 2a70972dd..db00a29d8 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -203,7 +203,6 @@ class UltraConstants: DEFAULT_EARTH_CULLING_RADIUS = EARTH_RADIUS_KM * N_RE # L1b extended spin culling parameters - LOW_VOLTAGE_CULL_THRESHOLD = 3400.0 SPIN_BIN_SIZE = 20 # Number of energy bins to use in energy dependent culling N_CULL_EBINS = 8