diff --git a/imap_processing/quality_flags.py b/imap_processing/quality_flags.py index 11d13389b..b6631380c 100644 --- a/imap_processing/quality_flags.py +++ b/imap_processing/quality_flags.py @@ -47,6 +47,7 @@ class ImapDEOutliersUltraFlags(FlagNameMixin): INVALID_ENERGY = 2**3 # bit 3 DURINGREPOINT = 2**4 # bit 4 # event during a repointing BACKTOF = 2**5 # bit 5 # Back TOF outlier + AUXOUTLIER = 2**6 # bit 6 # Event time is outside of aux dataset range. class ImapHkUltraFlags(FlagNameMixin): diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index 6fcbec06a..9df548847 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -8,7 +8,6 @@ from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.quality_flags import ImapDEOutliersUltraFlags from imap_processing.ultra.constants import UltraConstants -from imap_processing.ultra.l1b.de import FILLVAL_FLOAT32 from imap_processing.ultra.l1b.ultra_l1b import ultra_l1b from imap_processing.ultra.utils.ultra_l1_utils import create_dataset @@ -200,7 +199,7 @@ def test_cdf_de_flags( l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") # All valid events should be flagged as DURINGREPOINT since the repoint data does # not cover any of the event times - valid_events = l1b_de_dataset[0]["event_times"] != FILLVAL_FLOAT32 + valid_events = l1b_de_dataset[0]["event_times"] != UltraConstants.FILLVAL_FLOAT flags = l1b_de_dataset[0]["quality_outliers"].values[valid_events] assert np.all((flags & ImapDEOutliersUltraFlags.DURINGREPOINT.value) != 0) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index 75ebe6557..93c4f4fbd 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -524,7 +524,7 @@ def test_get_eventtimes(test_fixture, aux_dataset): """Tests get_eventtimes function.""" df_filt, _, _, de_dataset = test_fixture - event_times, spin_start_times = get_event_times( + event_times, spin_start_times, _ = get_event_times( aux_dataset, de_dataset["shcoarse"].values, de_dataset["phase_angle"].values, @@ -600,7 +600,7 @@ def test_get_event_times_out_of_range( # set spin data that DOES cover the range of coarse_times use_fake_spin_data_for_time(min_time - 1000, min_time + 10000) # This should not raise an error. - event_times, spin_starts = get_event_times( + event_times, spin_starts, quality_flags = get_event_times( aux_dataset, coarse_times, de_dataset["phase_angle"].values, @@ -608,6 +608,13 @@ def test_get_event_times_out_of_range( assert event_times.shape == coarse_times.shape assert spin_starts.shape == coarse_times.shape + # Check events that dont have aux data coverage. These should be fill vals + # and the quality flag array should indicate an AUXOUTLIER flag. + assert event_times[0] == UltraConstants.FILLVAL_FLOAT + assert spin_starts[0] == UltraConstants.FILLVAL_FLOAT + assert quality_flags[0] == ImapDEOutliersUltraFlags.AUXOUTLIER.value + assert np.all(quality_flags[1:] == ImapDEOutliersUltraFlags.NONE.value) + @pytest.mark.external_test_data def test_interpolate_fwhm(ancillary_files): diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py index 9fedf82fa..edfadff9a 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py @@ -208,9 +208,10 @@ def test_get_deadtime_interpolator(use_fake_spin_data_for_time, aux_dataset): deadtime_ratios = xr.DataArray( np.random.uniform(0.1, 1.0, num_deadtimes), dims=["epoch"] ) + met_in_range = aux_dataset["timespinstart"].values[0] sectored_rates_ds = xr.Dataset( {"epoch": ("epoch", np.ones_like(deadtime_ratios))}, - {"shcoarse": ("epoch", np.ones_like(deadtime_ratios))}, + {"shcoarse": ("epoch", np.full_like(deadtime_ratios, met_in_range))}, ) with mock.patch( "imap_processing.ultra.l1c.ultra_l1c_pset_bins.get_deadtime_ratios", diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 5f432d805..db00a29d8 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -43,6 +43,14 @@ class UltraConstants: SSD-specific correction to DMIN for time-of-flight normalization """ + # Define fillvals + FILLVAL_UINT8 = 255 + FILLVAL_UINT16 = 65535 + FILLVAL_UINT32 = 4294967295 + FILLVAL_FLOAT = -1.0e31 + + NOMINAL_SPIN_PERIOD_SEC: float = 15.0 + D_SLIT_FOIL: float = 3.39 SLIT_Z: float = 44.89 YF_ESTIMATE_LEFT: float = 40.0 @@ -195,7 +203,6 @@ class UltraConstants: DEFAULT_EARTH_CULLING_RADIUS = EARTH_RADIUS_KM * N_RE # L1b extended spin culling parameters - LOW_VOLTAGE_CULL_THRESHOLD = 3400.0 SPIN_BIN_SIZE = 20 # Number of energy bins to use in energy dependent culling N_CULL_EBINS = 8 diff --git a/imap_processing/ultra/l1b/badtimes.py b/imap_processing/ultra/l1b/badtimes.py index c78520a49..3fe1ca4d6 100644 --- a/imap_processing/ultra/l1b/badtimes.py +++ b/imap_processing/ultra/l1b/badtimes.py @@ -4,13 +4,9 @@ import xarray as xr from numpy.typing import NDArray +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict -FILLVAL_UINT16 = 65535 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_FLOAT64 = -1.0e31 -FILLVAL_UINT32 = 4294967295 - def calculate_badtimes( extendedspin_dataset: xr.Dataset, @@ -46,54 +42,60 @@ def calculate_badtimes( if badtimes_dataset["spin_number"].size == 0: badtimes_dataset = badtimes_dataset.drop_dims("spin_number") - badtimes_dataset = badtimes_dataset.expand_dims(spin_number=[FILLVAL_UINT32]) + badtimes_dataset = badtimes_dataset.expand_dims( + spin_number=[UltraConstants.FILLVAL_UINT32] + ) badtimes_dataset["spin_start_time"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) badtimes_dataset["spin_period"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) badtimes_dataset["spin_rate"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) badtimes_dataset["start_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) badtimes_dataset["stop_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) badtimes_dataset["coin_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) badtimes_dataset["rejected_events_per_spin"] = xr.DataArray( - np.array([FILLVAL_UINT32], dtype="uint32"), + np.array([UltraConstants.FILLVAL_UINT32], dtype="uint32"), dims=["spin_number"], ) badtimes_dataset["quality_attitude"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) badtimes_dataset["quality_hk"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) badtimes_dataset["quality_instruments"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) badtimes_dataset["quality_ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_UINT16, dtype="uint16"), + np.full((n_bins, 1), UltraConstants.FILLVAL_UINT16, dtype="uint16"), ) badtimes_dataset["ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT64, dtype="float64"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float64"), ) badtimes_dataset["ena_rates_threshold"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT32, dtype="float32"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float32"), ) return badtimes_dataset diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index 0122f756b..9c7a9bff1 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -13,6 +13,7 @@ from imap_processing.spice.time import ( et_to_met, ) +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import get_geometric_factor from imap_processing.ultra.l1b.ultra_l1b_annotated import ( get_annotated_particle_velocity, @@ -45,10 +46,6 @@ ) from imap_processing.ultra.utils.ultra_l1_utils import create_dataset -FILLVAL_UINT8 = 255 -FILLVAL_UINT32 = 4294967295 -FILLVAL_FLOAT32 = -1.0e31 - def calculate_de( de_dataset: xr.Dataset, aux_dataset: xr.Dataset, name: str, ancillary_files: dict @@ -106,7 +103,7 @@ def calculate_de( for key, dataset_key in zip(keys, dataset_keys, strict=False) } ) - valid_mask = de_dataset["start_type"].data != FILLVAL_UINT8 + valid_mask = de_dataset["start_type"].data != UltraConstants.FILLVAL_UINT8 ph_mask = np.isin( de_dataset["stop_type"].data, [StopType.Top.value, StopType.Bottom.value] ) @@ -117,72 +114,86 @@ def calculate_de( ssd_indices = np.nonzero(valid_mask & ssd_mask)[0] # Instantiate arrays xf: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) yf: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) xb: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) yb: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) xc: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + d: np.ndarray = np.full( + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) + r: np.ndarray = np.full( + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) - d: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) - r: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) phi: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) theta: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) tof: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) etof: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) ctof: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) tof_energy: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) magnitude_v: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) energy: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + e_bin: np.ndarray = np.full( + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) - e_bin: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) e_bin_l1a: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) species_bin: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) t2: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 ) event_times: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64 ) spin_starts: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64 ) shape = (len(de_dataset["epoch"]), 3) - sc_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - sc_dps_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - helio_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - velocities: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - v_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - r_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + sc_velocity: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + sc_dps_velocity: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + helio_velocity: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + velocities: np.ndarray = np.full( + shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + v_hat: np.ndarray = np.full(shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32) + r_hat: np.ndarray = np.full(shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32) start_type: np.ndarray = np.full( - len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8 ) quality_flags = np.full( de_dataset["epoch"].shape, ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16 @@ -204,14 +215,18 @@ def calculate_de( start_type[valid_indices] = de_dataset["start_type"].data[valid_indices] spin_ds = get_spin_info(aux_dataset, de_dataset["shcoarse"].data) - (event_times[valid_mask], spin_starts[valid_mask]) = get_event_times( + (event_times[valid_mask], spin_starts[valid_mask], event_time_qf) = get_event_times( aux_dataset, de_dataset["shcoarse"].data[valid_mask], de_dataset["phase_angle"].data[valid_mask], spin_ds.isel(epoch=valid_mask), ) + quality_flags[valid_mask] |= event_time_qf - de_dict["spin"] = spin_ds.spin_number.data + spin_number = spin_ds.spin_number.data + spin_missing_mask = np.isnan(spin_number) + spin_number[spin_missing_mask] = UltraConstants.FILLVAL_UINT32 + de_dict["spin"] = spin_number.astype(np.uint32) de_dict["event_times"] = event_times.astype(np.float64) # Pulse height ph_result = get_ph_tof_and_back_positions( @@ -357,7 +372,7 @@ def calculate_de( de_dict["tof_energy"] = tof_energy de_dict["energy"] = energy de_dict["computed_ebin"] = e_bin - valid_ebin = de_dataset["bin"].values != FILLVAL_UINT32 + valid_ebin = de_dataset["bin"].values != UltraConstants.FILLVAL_UINT32 e_bin_l1a[valid_ebin] = de_dataset["bin"].values[valid_ebin] de_dict["ebin"] = e_bin_l1a de_dict["species"] = species_bin @@ -366,7 +381,7 @@ def calculate_de( ultra_frame = getattr(SpiceFrame, f"IMAP_ULTRA_{sensor}") # Account for counts=0 (event times have FILL value) - valid_events = (event_times != FILLVAL_FLOAT32).copy() + valid_events = (event_times != UltraConstants.FILLVAL_FLOAT).copy() if repoint_id is not None: # Check all valid event times to see which are in the pointing in_pointing = calculate_events_in_pointing( diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index b03b88e0c..82211fc2f 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -28,9 +28,6 @@ from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins from imap_processing.ultra.utils.ultra_l1_utils import create_dataset -FILLVAL_UINT16 = 65535 -FILLVAL_FLOAT32 = -1.0e31 - def calculate_extendedspin( dict_datasets: dict[str, xr.Dataset], @@ -68,20 +65,34 @@ def calculate_extendedspin( # The energy dependent culling selects its de dataset per energy range. priority_1_de_dataset = de_datasets["p1"] + # Events with no aux data coverage (AUXOUTLIER, flagged in de.py) have a + # fill-valued "spin" that isn't a real spin number and must be excluded + # from per-spin binning to avoid using an invalid spin. + has_spin_mask = ( + priority_1_de_dataset["spin"].values != UltraConstants.FILLVAL_UINT32 + ) + spin_number = priority_1_de_dataset["spin"].values[has_spin_mask] + de_energy = priority_1_de_dataset["energy"].values[has_spin_mask] + + # check if there are no valid spins. + if spin_number.size == 0: + raise ValueError( + "All Spins are invalid. Please ensure that the l1a aux dataset " + "has the correct spin information." + ) + extendedspin_dict = {} rates_qf, spin, energy_bin_geometric_mean, n_sigma_per_energy = flag_rates( - priority_1_de_dataset["spin"].values, - priority_1_de_dataset["energy"].values, - ) - count_rates, _, _counts, _ = get_energy_histogram( - priority_1_de_dataset["spin"].values, priority_1_de_dataset["energy"].values + spin_number, + de_energy, ) + count_rates, _, _counts, _ = get_energy_histogram(spin_number, de_energy) attitude_qf, spin_rates, spin_period, spin_starttime = flag_attitude( - priority_1_de_dataset["spin"].values, aux_dataset + spin_number, aux_dataset ) # TODO: We will add to this later - hk_qf = flag_hk(priority_1_de_dataset["spin"].values) - inst_qf = flag_imap_instruments(priority_1_de_dataset["spin"].values) + hk_qf = flag_hk(spin_number) + inst_qf = flag_imap_instruments(spin_number) spin_bin_size = UltraConstants.SPIN_BIN_SIZE spin_tbin_edges = get_binned_spins_edges( @@ -156,9 +167,9 @@ def calculate_extendedspin( # Track rejected events in each spin based on # quality flags in de l1b data. rejected_counts = count_rejected_events_per_spin( - priority_1_de_dataset["spin"].values, - priority_1_de_dataset["quality_scattering"].values, - priority_1_de_dataset["quality_outliers"].values, + spin_number, + priority_1_de_dataset["quality_scattering"].values[has_spin_mask], + priority_1_de_dataset["quality_outliers"].values[has_spin_mask], ) # These will be the coordinates. @@ -177,9 +188,15 @@ def calculate_extendedspin( # Validate that the spin values match valid = (idx < pulses.unique_spins.size) & (pulses.unique_spins[idx] == spin) - start_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) - stop_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) - coin_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) + start_per_spin: np.ndarray = np.full( + len(spin), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + stop_per_spin: np.ndarray = np.full( + len(spin), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) + coin_per_spin: np.ndarray = np.full( + len(spin), UltraConstants.FILLVAL_FLOAT, dtype=np.float32 + ) # Fill only the valid ones start_per_spin[valid] = pulses.start_per_spin[idx[valid]] @@ -253,7 +270,9 @@ def calculate_extendedspin( # energy ranges. Set the length to be the max number of energy bins we expect to # use for culling. The number of edges is one more than the number of bins (17). ranges: np.ndarray = np.full( - (UltraConstants.MAX_ENERGY_RANGE_EDGES,), FILLVAL_FLOAT32, dtype=np.float32 + (UltraConstants.MAX_ENERGY_RANGE_EDGES,), + UltraConstants.FILLVAL_FLOAT, + dtype=np.float32, ) ranges[: len(energy_ranges)] = energy_ranges extendedspin_dict["energy_range_edges"] = ranges diff --git a/imap_processing/ultra/l1b/goodtimes.py b/imap_processing/ultra/l1b/goodtimes.py index 5a57191e5..7daaf71ff 100644 --- a/imap_processing/ultra/l1b/goodtimes.py +++ b/imap_processing/ultra/l1b/goodtimes.py @@ -3,14 +3,10 @@ import numpy as np import xarray as xr +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.quality_flag_filters import SPIN_QUALITY_FLAG_FILTERS from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict -FILLVAL_UINT16 = 65535 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_FLOAT64 = -1.0e31 -FILLVAL_UINT32 = 4294967295 - def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Dataset: """ @@ -57,72 +53,84 @@ def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Datas goodtimes_dataset = create_dataset(data_dict, name, "l1b") if goodtimes_dataset["spin_number"].size == 0: goodtimes_dataset = goodtimes_dataset.drop_dims("spin_number") - goodtimes_dataset = goodtimes_dataset.expand_dims(spin_number=[FILLVAL_UINT32]) + goodtimes_dataset = goodtimes_dataset.expand_dims( + spin_number=[UltraConstants.FILLVAL_UINT32] + ) goodtimes_dataset["spin_start_time"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) goodtimes_dataset["spin_period"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) goodtimes_dataset["spin_rate"] = xr.DataArray( - np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"), + dims=["spin_number"], ) goodtimes_dataset["start_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) goodtimes_dataset["stop_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) goodtimes_dataset["coin_pulses_per_spin"] = xr.DataArray( - np.array([FILLVAL_FLOAT32], dtype="float32"), + np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"), dims=["spin_number"], ) goodtimes_dataset["rejected_events_per_spin"] = xr.DataArray( - np.array([FILLVAL_UINT32], dtype="uint32"), + np.array([UltraConstants.FILLVAL_UINT32], dtype="uint32"), dims=["spin_number"], ) goodtimes_dataset["quality_attitude"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_low_voltage"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_high_energy"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_upstream_ion_1"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_upstream_ion_2"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_spectral"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_statistics"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), + dims=["spin_number"], ) goodtimes_dataset["quality_hk"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) goodtimes_dataset["quality_instruments"] = xr.DataArray( - np.array([FILLVAL_UINT16], dtype="uint16"), + np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"], ) goodtimes_dataset["quality_ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_UINT16, dtype="uint16"), + np.full((n_bins, 1), UltraConstants.FILLVAL_UINT16, dtype="uint16"), ) goodtimes_dataset["ena_rates"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT64, dtype="float64"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float64"), ) goodtimes_dataset["ena_rates_threshold"] = ( ("energy_bin_geometric_mean", "spin_number"), - np.full((n_bins, 1), FILLVAL_FLOAT32, dtype="float32"), + np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float32"), ) return goodtimes_dataset diff --git a/imap_processing/ultra/l1b/quality_flag_filters.py b/imap_processing/ultra/l1b/quality_flag_filters.py index faa3921a5..5d34c97b2 100644 --- a/imap_processing/ultra/l1b/quality_flag_filters.py +++ b/imap_processing/ultra/l1b/quality_flag_filters.py @@ -34,6 +34,7 @@ ImapDEOutliersUltraFlags.DURINGREPOINT, ImapDEOutliersUltraFlags.COINPH, ImapDEOutliersUltraFlags.BACKTOF, + ImapDEOutliersUltraFlags.AUXOUTLIER, ], "quality_scattering": [ ImapDEScatteringUltraFlags.ABOVE_THRESHOLD, diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 78da8bd24..932841339 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -13,7 +13,6 @@ from scipy.interpolate import LinearNDInterpolator, RegularGridInterpolator from imap_processing.quality_flags import ImapDEOutliersUltraFlags -from imap_processing.spice.spin import interpolate_spin_data from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import ( @@ -30,10 +29,6 @@ logger = logging.getLogger(__name__) -FILLVAL_UINT8 = 255 -FILLVAL_FLOAT32 = -1.0e31 -FILLVAL_FLOAT64 = -1.0e31 - class StartType(Enum): """Start Type: 1=Left, 2=Right.""" @@ -545,9 +540,9 @@ def get_de_velocity( v_y = -delta_v[:, 1] / tof * 1e3 v_z = -delta_v[:, 2] / tof * 1e3 - v_x[tof < 0] = FILLVAL_FLOAT32 # used as fillvals - v_y[tof < 0] = FILLVAL_FLOAT32 - v_z[tof < 0] = FILLVAL_FLOAT32 + v_x[tof < 0] = UltraConstants.FILLVAL_FLOAT # used as fillvals + v_y[tof < 0] = UltraConstants.FILLVAL_FLOAT + v_z[tof < 0] = UltraConstants.FILLVAL_FLOAT velocities = np.vstack((v_x, v_y, v_z)).T @@ -646,7 +641,7 @@ def get_de_energy_kev( valid_velocity = np.isfinite(v2) valid_mask = index_hydrogen & valid_velocity - energy = np.full_like(v2, FILLVAL_FLOAT32) + energy = np.full_like(v2, UltraConstants.FILLVAL_FLOAT) # TODO: we will calculate the energies of the different species here. # 1/2 mv^2 in Joules, convert to keV @@ -936,16 +931,18 @@ def get_spin_start_indices( start_inds : numpy.ndarray Spin start indices for each event. missing_aux_data_mask : numpy.ndarray - Boolean array indicating where there are events out of the aux data range. The - universal spin table should be used to fill in missing data for these events. + Boolean array indicating where there are events out of the aux data range. + These events are dropped/flagged rather than filled in. """ # Get Spin Start Time in seconds spin_start_sec = aux_dataset["timespinstart"].values # Check that all events fall within the aux dataset time range. # The time window spans from the first spin start to the end of the last spin. first_spin_start = spin_start_sec[0] - # Define the end of the last spin as start time + max duration (15s) - last_spin_end = spin_start_sec[-1] + 15.0 + # Define the end of the last spin as start time + nominal spin duration + last_spin_end = ( + spin_start_sec[-1] + UltraConstants.NOMINAL_SPIN_PERIOD_SEC + ) # TODO ask ultra team missing_aux_data_mask = (de_event_met < first_spin_start) | ( de_event_met > last_spin_end ) @@ -954,8 +951,8 @@ def get_spin_start_indices( "Coarse MET time contains events outside aux_dataset time range " f"({first_spin_start} - {last_spin_end}). " f"Found min={de_event_met.min()}, max={de_event_met.max()}. " - f"Found {np.sum(missing_aux_data_mask)} events not covered by aux data. " - f" Trying to fill missing data using universal spin table." + f"Throwing away {np.sum(missing_aux_data_mask)} events not covered " + f"by aux data. " ) # Find the spin_start_sec that started directly before each event. start_inds = ( @@ -973,7 +970,7 @@ def get_event_times( de_event_met: NDArray, phase_angle: NDArray, spin_ds: xr.Dataset | None = None, -) -> tuple[NDArray, NDArray]: +) -> tuple[NDArray, NDArray, NDArray]: """ Get the event times, spin start times. @@ -998,22 +995,44 @@ def get_event_times( Event times in et. spin_start_times: numpy.ndarray Spin start times in et. + quality_flags : numpy.ndarray + Quality flags. Events with missing aux/spin data are flagged with + ``ImapDEOutliersUltraFlags.AUXOUTLIER``. """ # Get or compute spin info if spin_ds is None: spin_ds = get_spin_info(aux_dataset, de_event_met) # spin start with subsecond precision - spin_start_times = spin_ds.spin_starts + (spin_ds.spin_start_subs / 1000.0) + spin_start_times = (spin_ds.spin_starts + (spin_ds.spin_start_subs / 1000.0)).values # add the fractional spin offset - event_times = spin_start_times + (spin_ds.spin_duration / 1000.0) * ( + event_times = spin_start_times + (spin_ds.spin_duration.values / 1000.0) * ( phase_angle / 720.0 ) - return ( - ttj2000ns_to_et(met_to_ttj2000ns(event_times)), - ttj2000ns_to_et(met_to_ttj2000ns(spin_start_times)), + + # Flag and fill events with missing aux/spin data before doing the + # spice time conversion below. + missing_mask = np.isnan(event_times) | np.isnan(spin_start_times) + quality_flags = np.full( + event_times.shape, ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16 + ) + quality_flags[missing_mask] = ImapDEOutliersUltraFlags.AUXOUTLIER.value + + out_event_times = np.full( + event_times.shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) + out_spin_start_times = np.full( + spin_start_times.shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) + out_event_times[~missing_mask] = ttj2000ns_to_et( + met_to_ttj2000ns(event_times[~missing_mask]) ) + out_spin_start_times[~missing_mask] = ttj2000ns_to_et( + met_to_ttj2000ns(spin_start_times[~missing_mask]) + ) + + return out_event_times, out_spin_start_times, quality_flags def get_spin_info(aux_dataset: xr.Dataset, de_event_met: NDArray) -> xr.Dataset: @@ -1034,38 +1053,38 @@ def get_spin_info(aux_dataset: xr.Dataset, de_event_met: NDArray) -> xr.Dataset: ------- spin_info_per_event : xarray.Dataset Spin information for each event. + + Raises + ------ + ValueError + If none of the events fall within the aux dataset's time range. This + indicates a mismatched or missing aux dataset rather than a handful of + boundary events, so it is not safe to silently proceed. """ start_inds, missing_events = get_spin_start_indices(aux_dataset, de_event_met) + if de_event_met.size > 0 and np.all(missing_events): + raise ValueError( + "No events fall within the aux_dataset time range " + f"({aux_dataset['timespinstart'].values[0]} - " + f"{aux_dataset['timespinstart'].values[-1]}" + f" + {UltraConstants.NOMINAL_SPIN_PERIOD_SEC}). " + "Please check the aux dataset and direct event MET values." + ) # Initialize spin info dataset spin_info_per_event = xr.Dataset() # Create dict of var name lookups var_names = { - "spin_number": ("spinnumber", "spin_number"), - "spin_duration": ("duration", "spin_period_sec"), - "spin_starts": ("timespinstart", "spin_start_sec_sclk"), - "spin_start_subs": ("timespinstartsub", "spin_start_subsec_sclk"), + "spin_number": "spinnumber", + "spin_duration": "duration", + "spin_starts": "timespinstart", + "spin_start_subs": "timespinstartsub", } - # If there is not enough aux data covering an event, query the universal - # spin table using the start time to fill in the missing data. - # This can happen for the first event if the aux data starts after the DE data. - spin_data = ( - interpolate_spin_data(de_event_met[missing_events]) - if np.any(missing_events) - else None - ) - for var, (aux_name, ut_name) in var_names.items(): - init_array = np.zeros_like(de_event_met, dtype=np.float64) - if np.any(missing_events) and spin_data is not None: - # Get data from universal table for events missing aux data - init_array[missing_events] = spin_data[ut_name].values - if ut_name == "spin_start_subsec_sclk": - # Convert from microseconds to milliseconds to match aux data units - init_array[missing_events] /= 1000.0 + for var, aux_name in var_names.items(): + init_array = np.full(de_event_met.shape, np.nan, dtype=np.float64) # Get data from aux dataset for the rest of the events init_array[~missing_events] = aux_dataset[aux_name].values[start_inds] spin_info_per_event[var] = (("epoch",), init_array) - return spin_info_per_event @@ -1109,8 +1128,10 @@ def interpolate_fwhm( phi_vals = interp_phi((energy, phi_inst)) theta_vals = interp_theta((energy, theta_inst)) - phi_interp = np.where(np.isnan(phi_vals), FILLVAL_FLOAT32, phi_vals) - theta_interp = np.where(np.isnan(theta_vals), FILLVAL_FLOAT32, theta_vals) + phi_interp = np.where(np.isnan(phi_vals), UltraConstants.FILLVAL_FLOAT, phi_vals) + theta_interp = np.where( + np.isnan(theta_vals), UltraConstants.FILLVAL_FLOAT, theta_vals + ) return phi_interp, theta_interp @@ -1148,8 +1169,10 @@ def get_fwhm( theta_interp : NDArray Interpolated theta FWHM values. """ - phi_interp = np.full_like(phi_inst, FILLVAL_FLOAT64, dtype=np.float64) - theta_interp = np.full_like(theta_inst, FILLVAL_FLOAT64, dtype=np.float64) + phi_interp = np.full_like(phi_inst, UltraConstants.FILLVAL_FLOAT, dtype=np.float64) + theta_interp = np.full_like( + theta_inst, UltraConstants.FILLVAL_FLOAT, dtype=np.float64 + ) lt_table = get_angular_profiles("left", sensor, ancillary_files) rt_table = get_angular_profiles("right", sensor, ancillary_files) @@ -1213,7 +1236,7 @@ def get_efficiency_interpolator( (theta_vals, phi_vals, energy_vals), efficiency_grid, bounds_error=False, - fill_value=FILLVAL_FLOAT32, + fill_value=UltraConstants.FILLVAL_FLOAT, ) return interpolator, theta_min_max, phi_min_max, energy_min_max @@ -1302,7 +1325,7 @@ def determine_ebin_pulse_height( # PH event TOF normalization to Z axis ctof, _ = get_ctof(tof, path_length, type="PH") - ebins = np.full(path_length.shape, FILLVAL_UINT8, dtype=np.uint8) + ebins = np.full(path_length.shape, UltraConstants.FILLVAL_UINT8, dtype=np.uint8) valid = backtofvalid & coinphvalid ebins[valid] = get_ebins( "l1b-tofxph", energy[valid], ctof[valid], ebins[valid], ancillary_files @@ -1355,7 +1378,7 @@ def determine_ebin_ssd( # SSD event TOF normalization to Z axis ctof, _ = get_ctof(tof, path_length, type="SSD") - ebins = np.full(path_length.shape, FILLVAL_UINT8, dtype=np.uint8) + ebins = np.full(path_length.shape, UltraConstants.FILLVAL_UINT8, dtype=np.uint8) steep_path_length = get_image_params("PathSteepThresh", sensor, ancillary_files) medium_path_length = get_image_params("PathMediumThresh", sensor, ancillary_files) diff --git a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py index cc33e0259..67cbd5798 100644 --- a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +++ b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py @@ -34,7 +34,6 @@ ) # TODO: add species binning. -FILLVAL_FLOAT32 = -1.0e31 logger = logging.getLogger(__name__) diff --git a/imap_processing/ultra/l2/ultra_l2.py b/imap_processing/ultra/l2/ultra_l2.py index 2080ad6aa..4cc2325e6 100644 --- a/imap_processing/ultra/l2/ultra_l2.py +++ b/imap_processing/ultra/l2/ultra_l2.py @@ -24,10 +24,7 @@ from imap_processing.quality_flags import ImapPSETUltraFlags from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins -from imap_processing.ultra.l1c.ultra_l1c_pset_bins import ( - FILLVAL_FLOAT32, - get_energy_delta_minus_plus, -) +from imap_processing.ultra.l1c.ultra_l1c_pset_bins import get_energy_delta_minus_plus logger = logging.getLogger(__name__) @@ -252,7 +249,10 @@ def bin_pset_energy_bins( ) # Count number of pixels non_zero_pixels_per_group = ( - ((pset[vars_to_average] != 0) & (pset[vars_to_average] != FILLVAL_FLOAT32)) + ( + (pset[vars_to_average] != 0) + & (pset[vars_to_average] != UltraConstants.FILLVAL_FLOAT) + ) .astype(int) .groupby("energy_bin_index") .sum()