Skip to content

ULTRA l1b: update de event time calculation #3480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lacoak21 merged 9 commits into from
Oct 6, 2026
Merged

ULTRA l1b: update de event time calculation #3480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
15d25fa
fix logger
lacoak21 Sep 9, 2026
5221b93
update event_time calc
lacoak21 Sep 10, 2026
e352e40
new flag for aux data outliers
lacoak21 Sep 21, 2026
c9493ee
new flag
lacoak21 Sep 21, 2026
0447e2c
add check
lacoak21 Sep 21, 2026
517e586
raise value error at the source when there is no aux data covering th…
lacoak21 Sep 22, 2026
1dc9554
move fillvals to constants.py
lacoak21 Sep 24, 2026
c0a8334
fix constant reference
lacoak21 Oct 5, 2026
d6f05db
remove unused constant
lacoak21 Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions imap_processing/quality_flags.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class ImapDEOutliersUltraFlags(FlagNameMixin):
INVALID_ENERGY = 2**3 # bit 3
DURINGREPOINT = 2**4 # bit 4 # event during a repointing
BACKTOF = 2**5 # bit 5 # Back TOF outlier
AUXOUTLIER = 2**6 # bit 6 # Event time is outside of aux dataset range.


class ImapHkUltraFlags(FlagNameMixin):
Expand Down
3 changes: 1 addition & 2 deletions imap_processing/tests/ultra/unit/test_ultra_l1b.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from imap_processing.cdf.utils import load_cdf, write_cdf
from imap_processing.quality_flags import ImapDEOutliersUltraFlags
from imap_processing.ultra.constants import UltraConstants
from imap_processing.ultra.l1b.de import FILLVAL_FLOAT32
from imap_processing.ultra.l1b.ultra_l1b import ultra_l1b
from imap_processing.ultra.utils.ultra_l1_utils import create_dataset

Expand Down Expand Up @@ -200,7 +199,7 @@ def test_cdf_de_flags(
l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999")
# All valid events should be flagged as DURINGREPOINT since the repoint data does
# not cover any of the event times
valid_events = l1b_de_dataset[0]["event_times"] != FILLVAL_FLOAT32
valid_events = l1b_de_dataset[0]["event_times"] != UltraConstants.FILLVAL_FLOAT
flags = l1b_de_dataset[0]["quality_outliers"].values[valid_events]
assert np.all((flags & ImapDEOutliersUltraFlags.DURINGREPOINT.value) != 0)

Expand Down
11 changes: 9 additions & 2 deletions imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,7 @@ def test_get_eventtimes(test_fixture, aux_dataset):
"""Tests get_eventtimes function."""
df_filt, _, _, de_dataset = test_fixture

event_times, spin_start_times = get_event_times(
event_times, spin_start_times, _ = get_event_times(
aux_dataset,
de_dataset["shcoarse"].values,
de_dataset["phase_angle"].values,
Expand Down Expand Up @@ -600,14 +600,21 @@ def test_get_event_times_out_of_range(
# set spin data that DOES cover the range of coarse_times
use_fake_spin_data_for_time(min_time - 1000, min_time + 10000)
# This should not raise an error.
event_times, spin_starts = get_event_times(
event_times, spin_starts, quality_flags = get_event_times(
aux_dataset,
coarse_times,
de_dataset["phase_angle"].values,
)
assert event_times.shape == coarse_times.shape
assert spin_starts.shape == coarse_times.shape

# Check events that dont have aux data coverage. These should be fill vals
# and the quality flag array should indicate an AUXOUTLIER flag.
assert event_times[0] == UltraConstants.FILLVAL_FLOAT
assert spin_starts[0] == UltraConstants.FILLVAL_FLOAT
assert quality_flags[0] == ImapDEOutliersUltraFlags.AUXOUTLIER.value
assert np.all(quality_flags[1:] == ImapDEOutliersUltraFlags.NONE.value)


@pytest.mark.external_test_data
def test_interpolate_fwhm(ancillary_files):
Expand Down
3 changes: 2 additions & 1 deletion imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,10 @@ def test_get_deadtime_interpolator(use_fake_spin_data_for_time, aux_dataset):
deadtime_ratios = xr.DataArray(
np.random.uniform(0.1, 1.0, num_deadtimes), dims=["epoch"]
)
met_in_range = aux_dataset["timespinstart"].values[0]
sectored_rates_ds = xr.Dataset(
{"epoch": ("epoch", np.ones_like(deadtime_ratios))},
{"shcoarse": ("epoch", np.ones_like(deadtime_ratios))},
{"shcoarse": ("epoch", np.full_like(deadtime_ratios, met_in_range))},
)
with mock.patch(
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.get_deadtime_ratios",
Expand Down
9 changes: 8 additions & 1 deletion imap_processing/ultra/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ class UltraConstants:
SSD-specific correction to DMIN for time-of-flight normalization
"""

# Define fillvals
FILLVAL_UINT8 = 255
FILLVAL_UINT16 = 65535
FILLVAL_UINT32 = 4294967295
FILLVAL_FLOAT = -1.0e31

NOMINAL_SPIN_PERIOD_SEC: float = 15.0

D_SLIT_FOIL: float = 3.39
SLIT_Z: float = 44.89
YF_ESTIMATE_LEFT: float = 40.0
Expand Down Expand Up @@ -195,7 +203,6 @@ class UltraConstants:
DEFAULT_EARTH_CULLING_RADIUS = EARTH_RADIUS_KM * N_RE

# L1b extended spin culling parameters
LOW_VOLTAGE_CULL_THRESHOLD = 3400.0
SPIN_BIN_SIZE = 20
# Number of energy bins to use in energy dependent culling
N_CULL_EBINS = 8
Expand Down
40 changes: 21 additions & 19 deletions imap_processing/ultra/l1b/badtimes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,9 @@
import xarray as xr
from numpy.typing import NDArray

from imap_processing.ultra.constants import UltraConstants
from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict

FILLVAL_UINT16 = 65535
FILLVAL_FLOAT32 = -1.0e31
FILLVAL_FLOAT64 = -1.0e31
FILLVAL_UINT32 = 4294967295


def calculate_badtimes(
extendedspin_dataset: xr.Dataset,
Expand Down Expand Up @@ -46,54 +42,60 @@ def calculate_badtimes(

if badtimes_dataset["spin_number"].size == 0:
badtimes_dataset = badtimes_dataset.drop_dims("spin_number")
badtimes_dataset = badtimes_dataset.expand_dims(spin_number=[FILLVAL_UINT32])
badtimes_dataset = badtimes_dataset.expand_dims(
spin_number=[UltraConstants.FILLVAL_UINT32]
)
badtimes_dataset["spin_start_time"] = xr.DataArray(
np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"]
np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"),
dims=["spin_number"],
)
badtimes_dataset["spin_period"] = xr.DataArray(
np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"]
np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"),
dims=["spin_number"],
)
badtimes_dataset["spin_rate"] = xr.DataArray(
np.array([FILLVAL_FLOAT64], dtype="float64"), dims=["spin_number"]
np.array([UltraConstants.FILLVAL_FLOAT], dtype="float64"),
dims=["spin_number"],
)
badtimes_dataset["start_pulses_per_spin"] = xr.DataArray(
np.array([FILLVAL_FLOAT32], dtype="float32"),
np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"),
dims=["spin_number"],
)
badtimes_dataset["stop_pulses_per_spin"] = xr.DataArray(
np.array([FILLVAL_FLOAT32], dtype="float32"),
np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"),
dims=["spin_number"],
)
badtimes_dataset["coin_pulses_per_spin"] = xr.DataArray(
np.array([FILLVAL_FLOAT32], dtype="float32"),
np.array([UltraConstants.FILLVAL_FLOAT], dtype="float32"),
dims=["spin_number"],
)
badtimes_dataset["rejected_events_per_spin"] = xr.DataArray(
np.array([FILLVAL_UINT32], dtype="uint32"),
np.array([UltraConstants.FILLVAL_UINT32], dtype="uint32"),
dims=["spin_number"],
)
badtimes_dataset["quality_attitude"] = xr.DataArray(
np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"]
np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"),
dims=["spin_number"],
)
badtimes_dataset["quality_hk"] = xr.DataArray(
np.array([FILLVAL_UINT16], dtype="uint16"),
np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"),
dims=["spin_number"],
)
badtimes_dataset["quality_instruments"] = xr.DataArray(
np.array([FILLVAL_UINT16], dtype="uint16"),
np.array([UltraConstants.FILLVAL_UINT16], dtype="uint16"),
dims=["spin_number"],
)
badtimes_dataset["quality_ena_rates"] = (
("energy_bin_geometric_mean", "spin_number"),
np.full((n_bins, 1), FILLVAL_UINT16, dtype="uint16"),
np.full((n_bins, 1), UltraConstants.FILLVAL_UINT16, dtype="uint16"),
)
badtimes_dataset["ena_rates"] = (
("energy_bin_geometric_mean", "spin_number"),
np.full((n_bins, 1), FILLVAL_FLOAT64, dtype="float64"),
np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float64"),
)
badtimes_dataset["ena_rates_threshold"] = (
("energy_bin_geometric_mean", "spin_number"),
np.full((n_bins, 1), FILLVAL_FLOAT32, dtype="float32"),
np.full((n_bins, 1), UltraConstants.FILLVAL_FLOAT, dtype="float32"),
)

return badtimes_dataset
89 changes: 52 additions & 37 deletions imap_processing/ultra/l1b/de.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from imap_processing.spice.time import (
et_to_met,
)
from imap_processing.ultra.constants import UltraConstants
from imap_processing.ultra.l1b.lookup_utils import get_geometric_factor
from imap_processing.ultra.l1b.ultra_l1b_annotated import (
get_annotated_particle_velocity,
Expand Down Expand Up @@ -45,10 +46,6 @@
)
from imap_processing.ultra.utils.ultra_l1_utils import create_dataset

FILLVAL_UINT8 = 255
FILLVAL_UINT32 = 4294967295
FILLVAL_FLOAT32 = -1.0e31


def calculate_de(
de_dataset: xr.Dataset, aux_dataset: xr.Dataset, name: str, ancillary_files: dict
Expand Down Expand Up @@ -106,7 +103,7 @@ def calculate_de(
for key, dataset_key in zip(keys, dataset_keys, strict=False)
}
)
valid_mask = de_dataset["start_type"].data != FILLVAL_UINT8
valid_mask = de_dataset["start_type"].data != UltraConstants.FILLVAL_UINT8
ph_mask = np.isin(
de_dataset["stop_type"].data, [StopType.Top.value, StopType.Bottom.value]
)
Expand All @@ -117,72 +114,86 @@ def calculate_de(
ssd_indices = np.nonzero(valid_mask & ssd_mask)[0]
# Instantiate arrays
xf: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
yf: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
xb: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
yb: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
xc: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
d: np.ndarray = np.full(
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64
)
r: np.ndarray = np.full(
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
d: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64)
r: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32)
phi: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
theta: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
tof: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
etof: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
ctof: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
tof_energy: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
magnitude_v: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
energy: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
e_bin: np.ndarray = np.full(
len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8
)
e_bin: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8)
e_bin_l1a: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8
len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8
)
species_bin: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8
len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8
)
t2: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
event_times: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64
)
spin_starts: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64
len(de_dataset["epoch"]), UltraConstants.FILLVAL_FLOAT, dtype=np.float64
)
shape = (len(de_dataset["epoch"]), 3)
sc_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
sc_dps_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
helio_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
velocities: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
v_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
r_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
sc_velocity: np.ndarray = np.full(
shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
sc_dps_velocity: np.ndarray = np.full(
shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
helio_velocity: np.ndarray = np.full(
shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
velocities: np.ndarray = np.full(
shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32
)
v_hat: np.ndarray = np.full(shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32)
r_hat: np.ndarray = np.full(shape, UltraConstants.FILLVAL_FLOAT, dtype=np.float32)

start_type: np.ndarray = np.full(
len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8
len(de_dataset["epoch"]), UltraConstants.FILLVAL_UINT8, dtype=np.uint8
)
quality_flags = np.full(
de_dataset["epoch"].shape, ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16
Expand All @@ -204,14 +215,18 @@ def calculate_de(
start_type[valid_indices] = de_dataset["start_type"].data[valid_indices]
spin_ds = get_spin_info(aux_dataset, de_dataset["shcoarse"].data)

(event_times[valid_mask], spin_starts[valid_mask]) = get_event_times(
(event_times[valid_mask], spin_starts[valid_mask], event_time_qf) = get_event_times(
aux_dataset,
de_dataset["shcoarse"].data[valid_mask],
de_dataset["phase_angle"].data[valid_mask],
spin_ds.isel(epoch=valid_mask),
)
quality_flags[valid_mask] |= event_time_qf

de_dict["spin"] = spin_ds.spin_number.data
spin_number = spin_ds.spin_number.data
spin_missing_mask = np.isnan(spin_number)
spin_number[spin_missing_mask] = UltraConstants.FILLVAL_UINT32
de_dict["spin"] = spin_number.astype(np.uint32)
de_dict["event_times"] = event_times.astype(np.float64)
# Pulse height
ph_result = get_ph_tof_and_back_positions(
Expand Down Expand Up @@ -357,7 +372,7 @@ def calculate_de(
de_dict["tof_energy"] = tof_energy
de_dict["energy"] = energy
de_dict["computed_ebin"] = e_bin
valid_ebin = de_dataset["bin"].values != FILLVAL_UINT32
valid_ebin = de_dataset["bin"].values != UltraConstants.FILLVAL_UINT32
e_bin_l1a[valid_ebin] = de_dataset["bin"].values[valid_ebin]
de_dict["ebin"] = e_bin_l1a
de_dict["species"] = species_bin
Expand All @@ -366,7 +381,7 @@ def calculate_de(
ultra_frame = getattr(SpiceFrame, f"IMAP_ULTRA_{sensor}")

# Account for counts=0 (event times have FILL value)
valid_events = (event_times != FILLVAL_FLOAT32).copy()
valid_events = (event_times != UltraConstants.FILLVAL_FLOAT).copy()
if repoint_id is not None:
# Check all valid event times to see which are in the pointing
in_pointing = calculate_events_in_pointing(
Expand Down
Loading
Loading